zstandard: vendor python-zstandard 0.12

The upstream source distribution from PyPI was extracted. Unwanted
files were removed.

The clang-format ignore list was updated to reflect the new source
of files.

test-repo-compengines.t was updated to reflect a change in behavior
of the zstd library.

The project contains a vendored copy of zstandard 1.4.3. The old
version was 1.3.8. This should result in some minor performance wins.

# no-check-commit because 3rd party code has different style guidelines

Differential Revision: https://phab.mercurial-scm.org/D6858

contrib/clang-format-ignorelist

@@ -1,99 +1,103 b''
 # Files that just need to be migrated to the formatter.
 # Do not add new files here!
 mercurial/cext/dirs.c
 mercurial/cext/manifest.c
 mercurial/cext/osutil.c
 # Vendored code that we should never format:
 contrib/python-zstandard/c-ext/bufferutil.c
 contrib/python-zstandard/c-ext/compressionchunker.c
 contrib/python-zstandard/c-ext/compressiondict.c
 contrib/python-zstandard/c-ext/compressionparams.c
 contrib/python-zstandard/c-ext/compressionreader.c
 contrib/python-zstandard/c-ext/compressionwriter.c
 contrib/python-zstandard/c-ext/compressobj.c
 contrib/python-zstandard/c-ext/compressor.c
 contrib/python-zstandard/c-ext/compressoriterator.c
 contrib/python-zstandard/c-ext/constants.c
 contrib/python-zstandard/c-ext/decompressionreader.c
 contrib/python-zstandard/c-ext/decompressionwriter.c
 contrib/python-zstandard/c-ext/decompressobj.c
 contrib/python-zstandard/c-ext/decompressor.c
 contrib/python-zstandard/c-ext/decompressoriterator.c
 contrib/python-zstandard/c-ext/frameparams.c
 contrib/python-zstandard/c-ext/python-zstandard.h
 contrib/python-zstandard/zstd.c
 contrib/python-zstandard/zstd/common/bitstream.h
 contrib/python-zstandard/zstd/common/compiler.h
 contrib/python-zstandard/zstd/common/cpu.h
 contrib/python-zstandard/zstd/common/debug.c
 contrib/python-zstandard/zstd/common/debug.h
 contrib/python-zstandard/zstd/common/entropy_common.c
 contrib/python-zstandard/zstd/common/error_private.c
 contrib/python-zstandard/zstd/common/error_private.h
 contrib/python-zstandard/zstd/common/fse_decompress.c
 contrib/python-zstandard/zstd/common/fse.h
 contrib/python-zstandard/zstd/common/huf.h
 contrib/python-zstandard/zstd/common/mem.h
 contrib/python-zstandard/zstd/common/pool.c
 contrib/python-zstandard/zstd/common/pool.h
 contrib/python-zstandard/zstd/common/threading.c
 contrib/python-zstandard/zstd/common/threading.h
 contrib/python-zstandard/zstd/common/xxhash.c
 contrib/python-zstandard/zstd/common/xxhash.h
 contrib/python-zstandard/zstd/common/zstd_common.c
 contrib/python-zstandard/zstd/common/zstd_errors.h
 contrib/python-zstandard/zstd/common/zstd_internal.h
 contrib/python-zstandard/zstd/compress/fse_compress.c
 contrib/python-zstandard/zstd/compress/hist.c
 contrib/python-zstandard/zstd/compress/hist.h
 contrib/python-zstandard/zstd/compress/huf_compress.c
 contrib/python-zstandard/zstd/compress/zstd_compress.c
 contrib/python-zstandard/zstd/compress/zstd_compress_internal.h
+contrib/python-zstandard/zstd/compress/zstd_compress_literals.c
+contrib/python-zstandard/zstd/compress/zstd_compress_literals.h
+contrib/python-zstandard/zstd/compress/zstd_compress_sequences.c
+contrib/python-zstandard/zstd/compress/zstd_compress_sequences.h
 contrib/python-zstandard/zstd/compress/zstd_double_fast.c
 contrib/python-zstandard/zstd/compress/zstd_double_fast.h
 contrib/python-zstandard/zstd/compress/zstd_fast.c
 contrib/python-zstandard/zstd/compress/zstd_fast.h
 contrib/python-zstandard/zstd/compress/zstd_lazy.c
 contrib/python-zstandard/zstd/compress/zstd_lazy.h
 contrib/python-zstandard/zstd/compress/zstd_ldm.c
 contrib/python-zstandard/zstd/compress/zstd_ldm.h
 contrib/python-zstandard/zstd/compress/zstdmt_compress.c
 contrib/python-zstandard/zstd/compress/zstdmt_compress.h
 contrib/python-zstandard/zstd/compress/zstd_opt.c
 contrib/python-zstandard/zstd/compress/zstd_opt.h
 contrib/python-zstandard/zstd/decompress/huf_decompress.c
 contrib/python-zstandard/zstd/decompress/zstd_ddict.c
 contrib/python-zstandard/zstd/decompress/zstd_ddict.h
 contrib/python-zstandard/zstd/decompress/zstd_decompress_block.c
 contrib/python-zstandard/zstd/decompress/zstd_decompress_block.h
 contrib/python-zstandard/zstd/decompress/zstd_decompress_internal.h
 contrib/python-zstandard/zstd/decompress/zstd_decompress.c
 contrib/python-zstandard/zstd/deprecated/zbuff_common.c
 contrib/python-zstandard/zstd/deprecated/zbuff_compress.c
 contrib/python-zstandard/zstd/deprecated/zbuff_decompress.c
 contrib/python-zstandard/zstd/deprecated/zbuff.h
 contrib/python-zstandard/zstd/dictBuilder/cover.c
 contrib/python-zstandard/zstd/dictBuilder/cover.h
 contrib/python-zstandard/zstd/dictBuilder/divsufsort.c
 contrib/python-zstandard/zstd/dictBuilder/divsufsort.h
 contrib/python-zstandard/zstd/dictBuilder/fastcover.c
 contrib/python-zstandard/zstd/dictBuilder/zdict.c
 contrib/python-zstandard/zstd/dictBuilder/zdict.h
 contrib/python-zstandard/zstd/zstd.h
 hgext/fsmonitor/pywatchman/bser.c
 mercurial/thirdparty/xdiff/xdiff.h
 mercurial/thirdparty/xdiff/xdiffi.c
 mercurial/thirdparty/xdiff/xdiffi.h
 mercurial/thirdparty/xdiff/xemit.c
 mercurial/thirdparty/xdiff/xemit.h
 mercurial/thirdparty/xdiff/xhistogram.c
 mercurial/thirdparty/xdiff/xinclude.h
 mercurial/thirdparty/xdiff/xmacros.h
 mercurial/thirdparty/xdiff/xmerge.c
 mercurial/thirdparty/xdiff/xpatience.c
 mercurial/thirdparty/xdiff/xprepare.c
 mercurial/thirdparty/xdiff/xprepare.h
 mercurial/thirdparty/xdiff/xtypes.h
 mercurial/thirdparty/xdiff/xutils.c
 mercurial/thirdparty/xdiff/xutils.h
 mercurial/thirdparty/zope/interface/_zope_interface_coptimizations.c

contrib/python-zstandard/NEWS.rst

@@ -1,672 +1,702 b''
 ===============
 Version History
 ===============
 
 1.0.0 (not yet released)
 ========================
 
 Actions Blocking Release
 ------------------------
 
 * compression and decompression APIs that support ``io.RawIOBase`` interface
   (#13).
 * ``stream_writer()`` APIs should support ``io.RawIOBase`` interface.
 * Properly handle non-blocking I/O and partial writes for objects implementing
   ``io.RawIOBase``.
 * Make ``write_return_read=True`` the default for objects implementing
   ``io.RawIOBase``.
 * Audit for consistent and proper behavior of ``flush()`` and ``close()`` for
   all objects implementing ``io.RawIOBase``. Is calling ``close()`` on
   wrapped stream acceptable, should ``__exit__`` always call ``close()``,
   should ``close()`` imply ``flush()``, etc.
 * Consider making reads across frames configurable behavior.
 * Refactor module names so C and CFFI extensions live under ``zstandard``
   package.
 * Overall API design review.
 * Use Python allocator where possible.
 * Figure out what to do about experimental APIs not implemented by CFFI.
 * APIs for auto adjusting compression parameters based on input size. e.g.
   clamping the window log so it isn't too large for input.
 * Consider allowing compressor and decompressor instances to be thread safe,
   support concurrent operations. Or track when an operation is in progress and
   refuse to let concurrent operations use the same instance.
 * Support for magic-less frames for all decompression operations (``decompress()``
   doesn't work due to sniffing the content size and the lack of a ZSTD API to
   sniff magic-less frames - this should be fixed in 1.3.5.).
 * Audit for complete flushing when ending compression streams.
 * Deprecate legacy APIs.
 * Audit for ability to control read/write sizes on all APIs.
 * Detect memory leaks via bench.py.
 * Remove low-level compression parameters from ``ZstdCompressor.__init__`` and
   require use of ``CompressionParameters``.
 * Expose ``ZSTD_getFrameProgression()`` from more compressor types.
 * Support modifying compression parameters mid operation when supported by
   zstd API.
 * Expose ``ZSTD_CLEVEL_DEFAULT`` constant.
 * Support ``ZSTD_p_forceAttachDict`` compression parameter.
+* Support ``ZSTD_c_literalCompressionMode `` compression parameter.
 * Use ``ZSTD_CCtx_getParameter()``/``ZSTD_CCtxParam_getParameter()`` for retrieving
   compression parameters.
 * Consider exposing ``ZSTDMT_toFlushNow()``.
 * Expose ``ZDICT_trainFromBuffer_fastCover()``,
   ``ZDICT_optimizeTrainFromBuffer_fastCover``.
 * Expose and enforce ``ZSTD_minCLevel()`` for minimum compression level.
 * Consider a ``chunker()`` API for decompression.
 * Consider stats for ``chunker()`` API, including finding the last consumed
   offset of input data.
 * Consider exposing ``ZSTD_cParam_getBounds()`` and
   ``ZSTD_dParam_getBounds()`` APIs.
 * Consider controls over resetting compression contexts (session only, parameters,
   or session and parameters).
 * Actually use the CFFI backend in fuzzing tests.
 
 Other Actions Not Blocking Release
 ---------------------------------------
 
 * Support for block compression APIs.
 * API for ensuring max memory ceiling isn't exceeded.
 * Move off nose for testing.
 
+0.12.0 (released 2019-09-15)
+============================
+
+Backwards Compatibility Notes
+-----------------------------
+
+* Support for Python 3.4 has been dropped since Python 3.4 is no longer
+  a supported Python version upstream. (But it will likely continue to
+  work until Python 2.7 support is dropped and we port to Python 3.5+
+  APIs.)
+
+Bug Fixes
+---------
+
+* Fix ``ZstdDecompressor.__init__`` on 64-bit big-endian systems (#91).
+* Fix memory leak in ``ZstdDecompressionReader.seek()`` (#82).
+
+Changes
+-------
+
+* CI transitioned to Azure Pipelines (from AppVeyor and Travis CI).
+* Switched to ``pytest`` for running tests (from ``nose``).
+* Bundled zstandard library upgraded from 1.3.8 to 1.4.3.
+
+0.11.1 (released 2019-05-14)
+============================
+
+* Fix memory leak in ``ZstdDecompressionReader.seek()`` (#82).
+
 0.11.0 (released 2019-02-24)
 ============================
 
-Backwards Compatibility Nodes
+Backwards Compatibility Notes
 -----------------------------
 
 * ``ZstdDecompressor.read()`` now allows reading sizes of ``-1`` or ``0``
   and defaults to ``-1``, per the documented behavior of
   ``io.RawIOBase.read()``. Previously, we required an argument that was
   a positive value.
 * The ``readline()``, ``readlines()``, ``__iter__``, and ``__next__`` methods
   of ``ZstdDecompressionReader()`` now raise ``io.UnsupportedOperation``
   instead of ``NotImplementedError``.
 * ``ZstdDecompressor.stream_reader()`` now accepts a ``read_across_frames``
   argument. The default value will likely be changed in a future release
   and consumers are advised to pass the argument to avoid unwanted change
   of behavior in the future.
 * ``setup.py`` now always disables the CFFI backend if the installed
   CFFI package does not meet the minimum version requirements. Before, it was
   possible for the CFFI backend to be generated and a run-time error to
   occur.
 * In the CFFI backend, ``CompressionReader`` and ``DecompressionReader``
   were renamed to ``ZstdCompressionReader`` and ``ZstdDecompressionReader``,
   respectively so naming is identical to the C extension. This should have
   no meaningful end-user impact, as instances aren't meant to be
   constructed directly.
 * ``ZstdDecompressor.stream_writer()`` now accepts a ``write_return_read``
   argument to control whether ``write()`` returns the number of bytes
   read from the source / written to the decompressor. It defaults to off,
   which preserves the existing behavior of returning the number of bytes
   emitted from the decompressor. The default will change in a future release
   so behavior aligns with the specified behavior of ``io.RawIOBase``.
 * ``ZstdDecompressionWriter.__exit__`` now calls ``self.close()``. This
   will result in that stream plus the underlying stream being closed as
   well. If this behavior is not desirable, do not use instances as
   context managers.
 * ``ZstdCompressor.stream_writer()`` now accepts a ``write_return_read``
   argument to control whether ``write()`` returns the number of bytes read
   from the source / written to the compressor. It defaults to off, which
   preserves the existing behavior of returning the number of bytes emitted
   from the compressor. The default will change in a future release so
   behavior aligns with the specified behavior of ``io.RawIOBase``.
 * ``ZstdCompressionWriter.__exit__`` now calls ``self.close()``. This will
   result in that stream plus any underlying stream being closed as well. If
   this behavior is not desirable, do not use instances as context managers.
 * ``ZstdDecompressionWriter`` no longer requires being used as a context
   manager (#57).
 * ``ZstdCompressionWriter`` no longer requires being used as a context
   manager (#57).
 * The ``overlap_size_log`` attribute on ``CompressionParameters`` instances
   has been deprecated and will be removed in a future release. The
   ``overlap_log`` attribute should be used instead.
 * The ``overlap_size_log`` argument to ``CompressionParameters`` has been
   deprecated and will be removed in a future release. The ``overlap_log``
   argument should be used instead.
 * The ``ldm_hash_every_log`` attribute on ``CompressionParameters`` instances
   has been deprecated and will be removed in a future release. The
   ``ldm_hash_rate_log`` attribute should be used instead.
 * The ``ldm_hash_every_log`` argument to ``CompressionParameters`` has been
   deprecated and will be removed in a future release. The ``ldm_hash_rate_log``
   argument should be used instead.
 * The ``compression_strategy`` argument to ``CompressionParameters`` has been
   deprecated and will be removed in a future release. The ``strategy``
   argument should be used instead.
 * The ``SEARCHLENGTH_MIN`` and ``SEARCHLENGTH_MAX`` constants are deprecated
   and will be removed in a future release. Use ``MINMATCH_MIN`` and
   ``MINMATCH_MAX`` instead.
 * The ``zstd_cffi`` module has been renamed to ``zstandard.cffi``. As had
   been documented in the ``README`` file since the ``0.9.0`` release, the
   module should not be imported directly at its new location. Instead,
   ``import zstandard`` to cause an appropriate backend module to be loaded
   automatically.
 
 Bug Fixes
 ---------
 
 * CFFI backend could encounter a failure when sending an empty chunk into
   ``ZstdDecompressionObj.decompress()``. The issue has been fixed.
 * CFFI backend could encounter an error when calling
   ``ZstdDecompressionReader.read()`` if there was data remaining in an
   internal buffer. The issue has been fixed. (#71)
 
 Changes
 -------
 
 * ``ZstDecompressionObj.decompress()`` now properly handles empty inputs in
   the CFFI backend.
 * ``ZstdCompressionReader`` now implements ``read1()`` and ``readinto1()``.
   These are part of the ``io.BufferedIOBase`` interface.
 * ``ZstdCompressionReader`` has gained a ``readinto(b)`` method for reading
   compressed output into an existing buffer.
 * ``ZstdCompressionReader.read()`` now defaults to ``size=-1`` and accepts
   read sizes of ``-1`` and ``0``. The new behavior aligns with the documented
   behavior of ``io.RawIOBase``.
 * ``ZstdCompressionReader`` now implements ``readall()``. Previously, this
   method raised ``NotImplementedError``.
 * ``ZstdDecompressionReader`` now implements ``read1()`` and ``readinto1()``.
   These are part of the ``io.BufferedIOBase`` interface.
 * ``ZstdDecompressionReader.read()`` now defaults to ``size=-1`` and accepts
   read sizes of ``-1`` and ``0``. The new behavior aligns with the documented
   behavior of ``io.RawIOBase``.
 * ``ZstdDecompressionReader()`` now implements ``readall()``. Previously, this
   method raised ``NotImplementedError``.
 * The ``readline()``, ``readlines()``, ``__iter__``, and ``__next__`` methods
   of ``ZstdDecompressionReader()`` now raise ``io.UnsupportedOperation``
   instead of ``NotImplementedError``. This reflects a decision to never
   implement text-based I/O on (de)compressors and keep the low-level API
   operating in the binary domain. (#13)
 * ``README.rst`` now documented how to achieve linewise iteration using
   an ``io.TextIOWrapper`` with a ``ZstdDecompressionReader``.
 * ``ZstdDecompressionReader`` has gained a ``readinto(b)`` method for
   reading decompressed output into an existing buffer. This allows chaining
   to an ``io.TextIOWrapper`` on Python 3 without using an ``io.BufferedReader``.
 * ``ZstdDecompressor.stream_reader()`` now accepts a ``read_across_frames``
   argument to control behavior when the input data has multiple zstd
   *frames*. When ``False`` (the default for backwards compatibility), a
   ``read()`` will stop when the end of a zstd *frame* is encountered. When
   ``True``, ``read()`` can potentially return data spanning multiple zstd
   *frames*. The default will likely be changed to ``True`` in a future
   release.
 * ``setup.py`` now performs CFFI version sniffing and disables the CFFI
   backend if CFFI is too old. Previously, we only used ``install_requires``
   to enforce the CFFI version and not all build modes would properly enforce
   the minimum CFFI version. (#69)
 * CFFI's ``ZstdDecompressionReader.read()`` now properly handles data
   remaining in any internal buffer. Before, repeated ``read()`` could
   result in *random* errors. (#71)
 * Upgraded various Python packages in CI environment.
 * Upgrade to hypothesis 4.5.11.
 * In the CFFI backend, ``CompressionReader`` and ``DecompressionReader``
   were renamed to ``ZstdCompressionReader`` and ``ZstdDecompressionReader``,
   respectively.
 * ``ZstdDecompressor.stream_writer()`` now accepts a ``write_return_read``
   argument to control whether ``write()`` returns the number of bytes read
   from the source. It defaults to ``False`` to preserve backwards
   compatibility.
 * ``ZstdDecompressor.stream_writer()`` now implements the ``io.RawIOBase``
   interface and behaves as a proper stream object.
 * ``ZstdCompressor.stream_writer()`` now accepts a ``write_return_read``
   argument to control whether ``write()`` returns the number of bytes read
   from the source. It defaults to ``False`` to preserve backwards
   compatibility.
 * ``ZstdCompressionWriter`` now implements the ``io.RawIOBase`` interface and
   behaves as a proper stream object. ``close()`` will now close the stream
   and the underlying stream (if possible). ``__exit__`` will now call
   ``close()``. Methods like ``writable()`` and ``fileno()`` are implemented.
 * ``ZstdDecompressionWriter`` no longer must be used as a context manager.
 * ``ZstdCompressionWriter`` no longer must be used as a context manager.
   When not using as a context manager, it is important to call
   ``flush(FRAME_FRAME)`` or the compression stream won't be properly
   terminated and decoders may complain about malformed input.
 * ``ZstdCompressionWriter.flush()`` (what is returned from
   ``ZstdCompressor.stream_writer()``) now accepts an argument controlling the
   flush behavior. Its value can be one of the new constants
   ``FLUSH_BLOCK`` or ``FLUSH_FRAME``.
 * ``ZstdDecompressionObj`` instances now have a ``flush([length=None])`` method.
   This provides parity with standard library equivalent types. (#65)
 * ``CompressionParameters`` no longer redundantly store individual compression
   parameters on each instance. Instead, compression parameters are stored inside
   the underlying ``ZSTD_CCtx_params`` instance. Attributes for obtaining
   parameters are now properties rather than instance variables.
 * Exposed the ``STRATEGY_BTULTRA2`` constant.
 * ``CompressionParameters`` instances now expose an ``overlap_log`` attribute.
   This behaves identically to the ``overlap_size_log`` attribute.
 * ``CompressionParameters()`` now accepts an ``overlap_log`` argument that
   behaves identically to the ``overlap_size_log`` argument. An error will be
   raised if both arguments are specified.
 * ``CompressionParameters`` instances now expose an ``ldm_hash_rate_log``
   attribute. This behaves identically to the ``ldm_hash_every_log`` attribute.
 * ``CompressionParameters()`` now accepts a ``ldm_hash_rate_log`` argument that
   behaves identically to the ``ldm_hash_every_log`` argument. An error will be
   raised if both arguments are specified.
 * ``CompressionParameters()`` now accepts a ``strategy`` argument that behaves
   identically to the ``compression_strategy`` argument. An error will be raised
   if both arguments are specified.
 * The ``MINMATCH_MIN`` and ``MINMATCH_MAX`` constants were added. They are
   semantically equivalent to the old ``SEARCHLENGTH_MIN`` and
   ``SEARCHLENGTH_MAX`` constants.
 * Bundled zstandard library upgraded from 1.3.7 to 1.3.8.
 * ``setup.py`` denotes support for Python 3.7 (Python 3.7 was supported and
   tested in the 0.10 release).
 * ``zstd_cffi`` module has been renamed to ``zstandard.cffi``.
 * ``ZstdCompressor.stream_writer()`` now reuses a buffer in order to avoid
   allocating a new buffer for every operation. This should result in faster
   performance in cases where ``write()`` or ``flush()`` are being called
   frequently. (#62)
 * Bundled zstandard library upgraded from 1.3.6 to 1.3.7.
 
 0.10.2 (released 2018-11-03)
 ============================
 
 Bug Fixes
 ---------
 
 * ``zstd_cffi.py`` added to ``setup.py`` (#60).
 
 Changes
 -------
 
 * Change some integer casts to avoid ``ssize_t`` (#61).
 
 0.10.1 (released 2018-10-08)
 ============================
 
 Backwards Compatibility Notes
 -----------------------------
 
 * ``ZstdCompressor.stream_reader().closed`` is now a property instead of a
   method (#58).
 * ``ZstdDecompressor.stream_reader().closed`` is now a property instead of a
   method (#58).
 
 Changes
 -------
 
 * Stop attempting to package Python 3.6 for Miniconda. The latest version of
   Miniconda is using Python 3.7. The Python 3.6 Miniconda packages were a lie
   since this were built against Python 3.7.
 * ``ZstdCompressor.stream_reader()``'s and ``ZstdDecompressor.stream_reader()``'s
   ``closed`` attribute is now a read-only property instead of a method. This now
   properly matches the ``IOBase`` API and allows instances to be used in more
   places that accept ``IOBase`` instances.
 
 0.10.0 (released 2018-10-08)
 ============================
 
 Backwards Compatibility Notes
 -----------------------------
 
 * ``ZstdDecompressor.stream_reader().read()`` now consistently requires an
   argument in both the C and CFFI backends. Before, the CFFI implementation
   would assume a default value of ``-1``, which was later rejected.
 * The ``compress_literals`` argument and attribute has been removed from
   ``zstd.ZstdCompressionParameters`` because it was removed by the zstd 1.3.5
   API.
 * ``ZSTD_CCtx_setParametersUsingCCtxParams()`` is no longer called on every
   operation performed against ``ZstdCompressor`` instances. The reason for this
   change is that the zstd 1.3.5 API no longer allows this without calling
   ``ZSTD_CCtx_resetParameters()`` first. But if we called
   ``ZSTD_CCtx_resetParameters()`` on every operation, we'd have to redo
   potentially expensive setup when using dictionaries. We now call
   ``ZSTD_CCtx_reset()`` on every operation and don't attempt to change
   compression parameters.
 * Objects returned by ``ZstdCompressor.stream_reader()`` no longer need to be
   used as a context manager. The context manager interface still exists and its
   behavior is unchanged.
 * Objects returned by ``ZstdDecompressor.stream_reader()`` no longer need to be
   used as a context manager. The context manager interface still exists and its
   behavior is unchanged.
 
 Bug Fixes
 ---------
 
 * ``ZstdDecompressor.decompressobj().decompress()`` should now return all data
   from internal buffers in more scenarios. Before, it was possible for data to
   remain in internal buffers. This data would be emitted on a subsequent call
   to ``decompress()``. The overall output stream would still be valid. But if
   callers were expecting input data to exactly map to output data (say the
   producer had used ``flush(COMPRESSOBJ_FLUSH_BLOCK)`` and was attempting to
   map input chunks to output chunks), then the previous behavior would be
   wrong. The new behavior is such that output from
   ``flush(COMPRESSOBJ_FLUSH_BLOCK)`` fed into ``decompressobj().decompress()``
   should produce all available compressed input.
 * ``ZstdDecompressor.stream_reader().read()`` should no longer segfault after
   a previous context manager resulted in error (#56).
 * ``ZstdCompressor.compressobj().flush(COMPRESSOBJ_FLUSH_BLOCK)`` now returns
   all data necessary to flush a block. Before, it was possible for the
   ``flush()`` to not emit all data necessary to fully represent a block. This
   would mean decompressors wouldn't be able to decompress all data that had been
   fed into the compressor and ``flush()``ed. (#55).
 
 New Features
 ------------
 
 * New module constants ``BLOCKSIZELOG_MAX``, ``BLOCKSIZE_MAX``,
   ``TARGETLENGTH_MAX`` that expose constants from libzstd.
 * New ``ZstdCompressor.chunker()`` API for manually feeding data into a
   compressor and emitting chunks of a fixed size. Like ``compressobj()``, the
   API doesn't impose restrictions on the input or output types for the
   data streams. Unlike ``compressobj()``, it ensures output chunks are of a
   fixed size. This makes this API useful when the compressed output is being
   fed into an I/O layer, where uniform write sizes are useful.
 * ``ZstdCompressor.stream_reader()`` no longer needs to be used as a context
   manager (#34).
 * ``ZstdDecompressor.stream_reader()`` no longer needs to be used as a context
   manager (#34).
 * Bundled zstandard library upgraded from 1.3.4 to 1.3.6.
 
 Changes
 -------
 
 * Added ``zstd_cffi.py`` and ``NEWS.rst`` to ``MANIFEST.in``.
 * ``zstandard.__version__`` is now defined (#50).
 * Upgrade pip, setuptools, wheel, and cibuildwheel packages to latest versions.
 * Upgrade various packages used in CI to latest versions. Notably tox (in
   order to support Python 3.7).
 * Use relative paths in setup.py to appease Python 3.7 (#51).
 * Added CI for Python 3.7.
 
 0.9.1 (released 2018-06-04)
 ===========================
 
 * Debian packaging support.
 * Fix typo in setup.py (#44).
 * Support building with mingw compiler (#46).
 
 0.9.0 (released 2018-04-08)
 ===========================
 
 Backwards Compatibility Notes
 -----------------------------
 
 * CFFI 1.11 or newer is now required (previous requirement was 1.8).
 * The primary module is now ``zstandard``. Please change imports of ``zstd``
   and ``zstd_cffi`` to ``import zstandard``. See the README for more. Support
   for importing the old names will be dropped in the next release.
 * ``ZstdCompressor.read_from()`` and ``ZstdDecompressor.read_from()`` have
   been renamed to ``read_to_iter()``. ``read_from()`` is aliased to the new
   name and will be deleted in a future release.
 * Support for Python 2.6 has been removed.
 * Support for Python 3.3 has been removed.
 * The ``selectivity`` argument to ``train_dictionary()`` has been removed, as
   the feature disappeared from zstd 1.3.
 * Support for legacy dictionaries has been removed. Cover dictionaries are now
   the default. ``train_cover_dictionary()`` has effectively been renamed to
   ``train_dictionary()``.
 * The ``allow_empty`` argument from ``ZstdCompressor.compress()`` has been
   deleted and the method now allows empty inputs to be compressed by default.
 * ``estimate_compression_context_size()`` has been removed. Use
   ``CompressionParameters.estimated_compression_context_size()`` instead.
 * ``get_compression_parameters()`` has been removed. Use
   ``CompressionParameters.from_level()`` instead.
 * The arguments to ``CompressionParameters.__init__()`` have changed. If you
   were using positional arguments before, the positions now map to different
   arguments. It is recommended to use keyword arguments to construct
   ``CompressionParameters`` instances.
 * ``TARGETLENGTH_MAX`` constant has been removed (it disappeared from zstandard
   1.3.4).
 * ``ZstdCompressor.write_to()`` and ``ZstdDecompressor.write_to()`` have been
   renamed to ``ZstdCompressor.stream_writer()`` and
   ``ZstdDecompressor.stream_writer()``, respectively. The old names are still
   aliased, but will be removed in the next major release.
 * Content sizes are written into frame headers by default
   (``ZstdCompressor(write_content_size=True)`` is now the default).
 * ``CompressionParameters`` has been renamed to ``ZstdCompressionParameters``
   for consistency with other types. The old name is an alias and will be removed
   in the next major release.
 
 Bug Fixes
 ---------
 
 * Fixed memory leak in ``ZstdCompressor.copy_stream()`` (#40) (from 0.8.2).
 * Fixed memory leak in ``ZstdDecompressor.copy_stream()`` (#35) (from 0.8.2).
 * Fixed memory leak of ``ZSTD_DDict`` instances in CFFI's ``ZstdDecompressor``.
 
 New Features
 ------------
 
 * Bundled zstandard library upgraded from 1.1.3 to 1.3.4. This delivers various
   bug fixes and performance improvements. It also gives us access to newer
   features.
 * Support for negative compression levels.
 * Support for *long distance matching* (facilitates compression ratios that approach
   LZMA).
 * Supporting for reading empty zstandard frames (with an embedded content size
   of 0).
 * Support for writing and partial support for reading zstandard frames without a
   magic header.
 * New ``stream_reader()`` API that exposes the ``io.RawIOBase`` interface (allows
   you to ``.read()`` from a file-like object).
 * Several minor features, bug fixes, and performance enhancements.
 * Wheels for Linux and macOS are now provided with releases.
 
 Changes
 -------
 
 * Functions accepting bytes data now use the buffer protocol and can accept
   more types (like ``memoryview`` and ``bytearray``) (#26).
 * Add #includes so compilation on OS X and BSDs works (#20).
 * New ``ZstdDecompressor.stream_reader()`` API to obtain a read-only i/o stream
   of decompressed data for a source.
 * New ``ZstdCompressor.stream_reader()`` API to obtain a read-only i/o stream of
   compressed data for a source.
 * Renamed ``ZstdDecompressor.read_from()`` to ``ZstdDecompressor.read_to_iter()``.
   The old name is still available.
 * Renamed ``ZstdCompressor.read_from()`` to ``ZstdCompressor.read_to_iter()``.
   ``read_from()`` is still available at its old location.
 * Introduce the ``zstandard`` module to import and re-export the C or CFFI
   *backend* as appropriate. Behavior can be controlled via the
   ``PYTHON_ZSTANDARD_IMPORT_POLICY`` environment variable. See README for
   usage info.
 * Vendored version of zstd upgraded to 1.3.4.
 * Added module constants ``CONTENTSIZE_UNKNOWN`` and ``CONTENTSIZE_ERROR``.
 * Add ``STRATEGY_BTULTRA`` compression strategy constant.
 * Switch from deprecated ``ZSTD_getDecompressedSize()`` to
   ``ZSTD_getFrameContentSize()`` replacement.
 * ``ZstdCompressor.compress()`` can now compress empty inputs without requiring
   special handling.
 * ``ZstdCompressor`` and ``ZstdDecompressor`` now have a ``memory_size()``
   method for determining the current memory utilization of the underlying zstd
   primitive.
 * ``train_dictionary()`` has new arguments and functionality for trying multiple
   variations of COVER parameters and selecting the best one.
 * Added module constants ``LDM_MINMATCH_MIN``, ``LDM_MINMATCH_MAX``, and
   ``LDM_BUCKETSIZELOG_MAX``.
 * Converted all consumers to the zstandard *new advanced API*, which uses
   ``ZSTD_compress_generic()``
 * ``CompressionParameters.__init__`` now accepts several more arguments,
   including support for *long distance matching*.
 * ``ZstdCompressionDict.__init__`` now accepts a ``dict_type`` argument that
   controls how the dictionary should be interpreted. This can be used to
   force the use of *content-only* dictionaries or to require the presence
   of the dictionary magic header.
 * ``ZstdCompressionDict.precompute_compress()`` can be used to precompute the
   compression dictionary so it can efficiently be used with multiple
   ``ZstdCompressor`` instances.
 * Digested dictionaries are now stored in ``ZstdCompressionDict`` instances,
   created automatically on first use, and automatically reused by all
   ``ZstdDecompressor`` instances bound to that dictionary.
 * All meaningful functions now accept keyword arguments.
 * ``ZstdDecompressor.decompressobj()`` now accepts a ``write_size`` argument
   to control how much work to perform on every decompressor invocation.
 * ``ZstdCompressor.write_to()`` now exposes a ``tell()``, which exposes the
   total number of bytes written so far.
 * ``ZstdDecompressor.stream_reader()`` now supports ``seek()`` when moving
   forward in the stream.
 * Removed ``TARGETLENGTH_MAX`` constant.
 * Added ``frame_header_size(data)`` function.
 * Added ``frame_content_size(data)`` function.
 * Consumers of ``ZSTD_decompress*`` have been switched to the new *advanced
   decompression* API.
 * ``ZstdCompressor`` and ``ZstdCompressionParams`` can now be constructed with
   negative compression levels.
 * ``ZstdDecompressor`` now accepts a ``max_window_size`` argument to limit the
   amount of memory required for decompression operations.
 * ``FORMAT_ZSTD1`` and ``FORMAT_ZSTD1_MAGICLESS`` constants to be used with
   the ``format`` compression parameter to control whether the frame magic
   header is written.
 * ``ZstdDecompressor`` now accepts a ``format`` argument to control the
   expected frame format.
 * ``ZstdCompressor`` now has a ``frame_progression()`` method to return
   information about the current compression operation.
 * Error messages in CFFI no longer have ``b''`` literals.
 * Compiler warnings and underlying overflow issues on 32-bit platforms have been
   fixed.
 * Builds in CI now build with compiler warnings as errors. This should hopefully
   fix new compiler warnings from being introduced.
 * Make ``ZstdCompressor(write_content_size=True)`` and
   ``CompressionParameters(write_content_size=True)`` the default.
 * ``CompressionParameters`` has been renamed to ``ZstdCompressionParameters``.
 
 0.8.2 (released 2018-02-22)
 ---------------------------
 
 * Fixed memory leak in ``ZstdCompressor.copy_stream()`` (#40).
 * Fixed memory leak in ``ZstdDecompressor.copy_stream()`` (#35).
 
 0.8.1 (released 2017-04-08)
 ---------------------------
 
 * Add #includes so compilation on OS X and BSDs works (#20).
 
 0.8.0 (released 2017-03-08)
 ===========================
 
 * CompressionParameters now has a estimated_compression_context_size() method.
   zstd.estimate_compression_context_size() is now deprecated and slated for
   removal.
 * Implemented a lot of fuzzing tests.
 * CompressionParameters instances now perform extra validation by calling
   ZSTD_checkCParams() at construction time.
 * multi_compress_to_buffer() API for compressing multiple inputs as a
   single operation, as efficiently as possible.
 * ZSTD_CStream instances are now used across multiple operations on
   ZstdCompressor instances, resulting in much better performance for
   APIs that do streaming.
 * ZSTD_DStream instances are now used across multiple operations on
   ZstdDecompressor instances, resulting in much better performance for
   APIs that do streaming.
 * train_dictionary() now releases the GIL.
 * Support for training dictionaries using the COVER algorithm.
 * multi_decompress_to_buffer() API for decompressing multiple frames as a
   single operation, as efficiently as possible.
 * Support for multi-threaded compression.
 * Disable deprecation warnings when compiling CFFI module.
 * Fixed memory leak in train_dictionary().
 * Removed DictParameters type.
 * train_dictionary() now accepts keyword arguments instead of a
   DictParameters instance to control dictionary generation.
 
 0.7.0 (released 2017-02-07)
 ===========================
 
 * Added zstd.get_frame_parameters() to obtain info about a zstd frame.
 * Added ZstdDecompressor.decompress_content_dict_chain() for efficient
   decompression of *content-only dictionary chains*.
 * CFFI module fully implemented; all tests run against both C extension and
   CFFI implementation.
 * Vendored version of zstd updated to 1.1.3.
 * Use ZstdDecompressor.decompress() now uses ZSTD_createDDict_byReference()
   to avoid extra memory allocation of dict data.
 * Add function names to error messages (by using ":name" in PyArg_Parse*
   functions).
 * Reuse decompression context across operations. Previously, we created a
   new ZSTD_DCtx for each decompress(). This was measured to slow down
   decompression by 40-200MB/s. The API guarantees say ZstdDecompressor
   is not thread safe. So we reuse the ZSTD_DCtx across operations and make
   things faster in the process.
 * ZstdCompressor.write_to()'s compress() and flush() methods now return number
   of bytes written.
 * ZstdDecompressor.write_to()'s write() method now returns the number of bytes
   written to the underlying output object.
 * CompressionParameters instances now expose their values as attributes.
 * CompressionParameters instances no longer are subscriptable nor behave
   as tuples (backwards incompatible). Use attributes to obtain values.
 * DictParameters instances now expose their values as attributes.
 
 0.6.0 (released 2017-01-14)
 ===========================
 
 * Support for legacy zstd protocols (build time opt in feature).
 * Automation improvements to test against Python 3.6, latest versions
   of Tox, more deterministic AppVeyor behavior.
 * CFFI "parser" improved to use a compiler preprocessor instead of rewriting
   source code manually.
 * Vendored version of zstd updated to 1.1.2.
 * Documentation improvements.
 * Introduce a bench.py script for performing (crude) benchmarks.
 * ZSTD_CCtx instances are now reused across multiple compress() operations.
 * ZstdCompressor.write_to() now has a flush() method.
 * ZstdCompressor.compressobj()'s flush() method now accepts an argument to
   flush a block (as opposed to ending the stream).
 * Disallow compress(b'') when writing content sizes by default (issue #11).
 
 0.5.2 (released 2016-11-12)
 ===========================
 
 * more packaging fixes for source distribution
 
 0.5.1 (released 2016-11-12)
 ===========================
 
 * setup_zstd.py is included in the source distribution
 
 0.5.0 (released 2016-11-10)
 ===========================
 
 * Vendored version of zstd updated to 1.1.1.
 * Continuous integration for Python 3.6 and 3.7
 * Continuous integration for Conda
 * Added compression and decompression APIs providing similar interfaces
   to the standard library ``zlib`` and ``bz2`` modules. This allows
   coding to a common interface.
 * ``zstd.__version__` is now defined.
 * ``read_from()`` on various APIs now accepts objects implementing the buffer
   protocol.
 * ``read_from()`` has gained a ``skip_bytes`` argument. This allows callers
   to pass in an existing buffer with a header without having to create a
   slice or a new object.
 * Implemented ``ZstdCompressionDict.as_bytes()``.
 * Python's memory allocator is now used instead of ``malloc()``.
 * Low-level zstd data structures are reused in more instances, cutting down
   on overhead for certain operations.
 * ``distutils`` boilerplate for obtaining an ``Extension`` instance
   has now been refactored into a standalone ``setup_zstd.py`` file. This
   allows other projects with ``setup.py`` files to reuse the
   ``distutils`` code for this project without copying code.
 * The monolithic ``zstd.c`` file has been split into a header file defining
   types and separate ``.c`` source files for the implementation.
 
 Older History
 =============
 
 2016-08-31 - Zstandard 1.0.0 is released and Gregory starts hacking on a
 Python extension for use by the Mercurial project. A very hacky prototype
 is sent to the mercurial-devel list for RFC.
 
 2016-09-03 - Most functionality from Zstandard C API implemented. Source
 code published on https://github.com/indygreg/python-zstandard. Travis-CI
 automation configured. 0.0.1 release on PyPI.
 
 2016-09-05 - After the API was rounded out a bit and support for Python
 2.6 and 2.7 was added, version 0.1 was released to PyPI.
 
 2016-09-05 - After the compressor and decompressor APIs were changed, 0.2
 was released to PyPI.
 
 2016-09-10 - 0.3 is released with a bunch of new features. ZstdCompressor
 now accepts arguments controlling frame parameters. The source size can now
 be declared when performing streaming compression. ZstdDecompressor.decompress()
 is implemented. Compression dictionaries are now cached when using the simple
 compression and decompression APIs. Memory size APIs added.
 ZstdCompressor.read_from() and ZstdDecompressor.read_from() have been
 implemented. This rounds out the major compression/decompression APIs planned
 by the author.
 
 2016-10-02 - 0.3.3 is released with a bug fix for read_from not fully
 decoding a zstd frame (issue #2).
 
 2016-10-02 - 0.4.0 is released with zstd 1.1.0, support for custom read and
 write buffer sizes, and a few bug fixes involving failure to read/write
 all data when buffer sizes were too small to hold remaining data.
 
 2016-11-10 - 0.5.0 is released with zstd 1.1.1 and other enhancements.

contrib/python-zstandard/README.rst

@@ -1,1606 +1,1602 b''
 ================
 python-zstandard
 ================
 
 This project provides Python bindings for interfacing with the
 `Zstandard <http://www.zstd.net>`_ compression library. A C extension
 and CFFI interface are provided.
 
 The primary goal of the project is to provide a rich interface to the
 underlying C API through a Pythonic interface while not sacrificing
 performance. This means exposing most of the features and flexibility
 of the C API while not sacrificing usability or safety that Python provides.
 
 The canonical home for this project lives in a Mercurial repository run by
 the author. For convenience, that repository is frequently synchronized to
 https://github.com/indygreg/python-zstandard.
 
-|  |ci-status| |win-ci-status|
+|  |ci-status|
 
 Requirements
 ============
 
 This extension is designed to run with Python 2.7, 3.4, 3.5, 3.6, and 3.7
 on common platforms (Linux, Windows, and OS X). On PyPy (both PyPy2 and PyPy3) we support version 6.0.0 and above. 
 x86 and x86_64 are well-tested on Windows. Only x86_64 is well-tested on Linux and macOS.
 
 Installing
 ==========
 
 This package is uploaded to PyPI at https://pypi.python.org/pypi/zstandard.
 So, to install this package::
 
    $ pip install zstandard
 
 Binary wheels are made available for some platforms. If you need to
 install from a source distribution, all you should need is a working C
 compiler and the Python development headers/libraries. On many Linux
 distributions, you can install a ``python-dev`` or ``python-devel``
 package to provide these dependencies.
 
 Packages are also uploaded to Anaconda Cloud at
 https://anaconda.org/indygreg/zstandard. See that URL for how to install
 this package with ``conda``.
 
 Performance
 ===========
 
 zstandard is a highly tunable compression algorithm. In its default settings
 (compression level 3), it will be faster at compression and decompression and
 will have better compression ratios than zlib on most data sets. When tuned
 for speed, it approaches lz4's speed and ratios. When tuned for compression
 ratio, it approaches lzma ratios and compression speed, but decompression
 speed is much faster. See the official zstandard documentation for more.
 
 zstandard and this library support multi-threaded compression. There is a
 mechanism to compress large inputs using multiple threads.
 
 The performance of this library is usually very similar to what the zstandard
 C API can deliver. Overhead in this library is due to general Python overhead
 and can't easily be avoided by *any* zstandard Python binding. This library
 exposes multiple APIs for performing compression and decompression so callers
 can pick an API suitable for their need. Contrast with the compression
 modules in Python's standard library (like ``zlib``), which only offer limited
 mechanisms for performing operations. The API flexibility means consumers can
 choose to use APIs that facilitate zero copying or minimize Python object
 creation and garbage collection overhead.
 
 This library is capable of single-threaded throughputs well over 1 GB/s. For
 exact numbers, measure yourself. The source code repository has a ``bench.py``
 script that can be used to measure things.
 
 API
 ===
 
 To interface with Zstandard, simply import the ``zstandard`` module::
 
    import zstandard
 
 It is a popular convention to alias the module as a different name for
 brevity::
 
    import zstandard as zstd
 
 This module attempts to import and use either the C extension or CFFI
 implementation. On Python platforms known to support C extensions (like
 CPython), it raises an ImportError if the C extension cannot be imported.
 On Python platforms known to not support C extensions (like PyPy), it only
 attempts to import the CFFI implementation and raises ImportError if that
 can't be done. On other platforms, it first tries to import the C extension
 then falls back to CFFI if that fails and raises ImportError if CFFI fails.
 
 To change the module import behavior, a ``PYTHON_ZSTANDARD_IMPORT_POLICY``
 environment variable can be set. The following values are accepted:
 
 default
    The behavior described above.
 cffi_fallback
    Always try to import the C extension then fall back to CFFI if that
    fails.
 cext
    Only attempt to import the C extension.
 cffi
    Only attempt to import the CFFI implementation.
 
 In addition, the ``zstandard`` module exports a ``backend`` attribute
 containing the string name of the backend being used. It will be one
 of ``cext`` or ``cffi`` (for *C extension* and *cffi*, respectively).
 
 The types, functions, and attributes exposed by the ``zstandard`` module
 are documented in the sections below.
 
 .. note::
 
    The documentation in this section makes references to various zstd
    concepts and functionality. The source repository contains a
    ``docs/concepts.rst`` file explaining these in more detail.
 
 ZstdCompressor
 --------------
 
 The ``ZstdCompressor`` class provides an interface for performing
 compression operations. Each instance is essentially a wrapper around a
 ``ZSTD_CCtx`` from the C API.
 
 Each instance is associated with parameters that control compression
 behavior. These come from the following named arguments (all optional):
 
 level
    Integer compression level. Valid values are between 1 and 22.
 dict_data
    Compression dictionary to use.
 
    Note: When using dictionary data and ``compress()`` is called multiple
    times, the ``ZstdCompressionParameters`` derived from an integer
    compression ``level`` and the first compressed data's size will be reused
    for all subsequent operations. This may not be desirable if source data
    size varies significantly.
 compression_params
    A ``ZstdCompressionParameters`` instance defining compression settings.
 write_checksum
    Whether a 4 byte checksum should be written with the compressed data.
    Defaults to False. If True, the decompressor can verify that decompressed
    data matches the original input data.
 write_content_size
    Whether the size of the uncompressed data will be written into the
    header of compressed data. Defaults to True. The data will only be
    written if the compressor knows the size of the input data. This is
    often not true for streaming compression.
 write_dict_id
    Whether to write the dictionary ID into the compressed data.
    Defaults to True. The dictionary ID is only written if a dictionary
    is being used.
 threads
    Enables and sets the number of threads to use for multi-threaded compression
    operations. Defaults to 0, which means to use single-threaded compression.
    Negative values will resolve to the number of logical CPUs in the system.
    Read below for more info on multi-threaded compression. This argument only
    controls thread count for operations that operate on individual pieces of
    data. APIs that spawn multiple threads for working on multiple pieces of
    data have their own ``threads`` argument.
 
 ``compression_params`` is mutually exclusive with ``level``, ``write_checksum``,
 ``write_content_size``, ``write_dict_id``, and ``threads``.
 
 Unless specified otherwise, assume that no two methods of ``ZstdCompressor``
 instances can be called from multiple Python threads simultaneously. In other
 words, assume instances are not thread safe unless stated otherwise.
 
 Utility Methods
 ^^^^^^^^^^^^^^^
 
 ``frame_progression()`` returns a 3-tuple containing the number of bytes
 ingested, consumed, and produced by the current compression operation.
 
 ``memory_size()`` obtains the memory utilization of the underlying zstd
 compression context, in bytes.::
 
     cctx = zstd.ZstdCompressor()
     memory = cctx.memory_size()
 
 Simple API
 ^^^^^^^^^^
 
 ``compress(data)`` compresses and returns data as a one-shot operation.::
 
    cctx = zstd.ZstdCompressor()
    compressed = cctx.compress(b'data to compress')
 
 The ``data`` argument can be any object that implements the *buffer protocol*.
 
 Stream Reader API
 ^^^^^^^^^^^^^^^^^
 
 ``stream_reader(source)`` can be used to obtain an object conforming to the
 ``io.RawIOBase`` interface for reading compressed output as a stream::
 
    with open(path, 'rb') as fh:
        cctx = zstd.ZstdCompressor()
        reader = cctx.stream_reader(fh)
        while True:
            chunk = reader.read(16384)
            if not chunk:
                break
 
            # Do something with compressed chunk.
 
 Instances can also be used as context managers::
 
    with open(path, 'rb') as fh:
        with cctx.stream_reader(fh) as reader:
            while True:
                chunk = reader.read(16384)
                if not chunk:
                    break
 
                # Do something with compressed chunk.
 
 When the context manager exits or ``close()`` is called, the stream is closed,
 underlying resources are released, and future operations against the compression
 stream will fail.
 
 The ``source`` argument to ``stream_reader()`` can be any object with a
 ``read(size)`` method or any object implementing the *buffer protocol*.
 
 ``stream_reader()`` accepts a ``size`` argument specifying how large the input
 stream is. This is used to adjust compression parameters so they are
 tailored to the source size.::
 
    with open(path, 'rb') as fh:
        cctx = zstd.ZstdCompressor()
        with cctx.stream_reader(fh, size=os.stat(path).st_size) as reader:
            ...
 
 If the ``source`` is a stream, you can specify how large ``read()`` requests
 to that stream should be via the ``read_size`` argument. It defaults to
 ``zstandard.COMPRESSION_RECOMMENDED_INPUT_SIZE``.::
 
    with open(path, 'rb') as fh:
        cctx = zstd.ZstdCompressor()
        # Will perform fh.read(8192) when obtaining data to feed into the
        # compressor.
        with cctx.stream_reader(fh, read_size=8192) as reader:
            ...
 
 The stream returned by ``stream_reader()`` is neither writable nor seekable
 (even if the underlying source is seekable). ``readline()`` and
 ``readlines()`` are not implemented because they don't make sense for
 compressed data. ``tell()`` returns the number of compressed bytes
 emitted so far.
 
 Streaming Input API
 ^^^^^^^^^^^^^^^^^^^
 
 ``stream_writer(fh)`` allows you to *stream* data into a compressor.
 
 Returned instances implement the ``io.RawIOBase`` interface. Only methods
 that involve writing will do useful things.
 
 The argument to ``stream_writer()`` must have a ``write(data)`` method. As
 compressed data is available, ``write()`` will be called with the compressed
 data as its argument. Many common Python types implement ``write()``, including
 open file handles and ``io.BytesIO``.
 
 The ``write(data)`` method is used to feed data into the compressor.
 
 The ``flush([flush_mode=FLUSH_BLOCK])`` method can be called to evict whatever
 data remains within the compressor's internal state into the output object. This
 may result in 0 or more ``write()`` calls to the output object. This method
 accepts an optional ``flush_mode`` argument to control the flushing behavior.
 Its value can be any of the ``FLUSH_*`` constants.
 
 Both ``write()`` and ``flush()`` return the number of bytes written to the
 object's ``write()``. In many cases, small inputs do not accumulate enough
 data to cause a write and ``write()`` will return ``0``.
 
 Calling ``close()`` will mark the stream as closed and subsequent I/O
 operations will raise ``ValueError`` (per the documented behavior of
 ``io.RawIOBase``). ``close()`` will also call ``close()`` on the underlying
 stream if such a method exists.
 
 Typically usage is as follows::
 
    cctx = zstd.ZstdCompressor(level=10)
    compressor = cctx.stream_writer(fh)
 
    compressor.write(b'chunk 0\n')
    compressor.write(b'chunk 1\n')
    compressor.flush()
    # Receiver will be able to decode ``chunk 0\nchunk 1\n`` at this point.
    # Receiver is also expecting more data in the zstd *frame*.
 
    compressor.write(b'chunk 2\n')
    compressor.flush(zstd.FLUSH_FRAME)
    # Receiver will be able to decode ``chunk 0\nchunk 1\nchunk 2``.
    # Receiver is expecting no more data, as the zstd frame is closed.
    # Any future calls to ``write()`` at this point will construct a new
    # zstd frame.
 
 Instances can be used as context managers. Exiting the context manager is
 the equivalent of calling ``close()``, which is equivalent to calling
 ``flush(zstd.FLUSH_FRAME)``::
 
    cctx = zstd.ZstdCompressor(level=10)
    with cctx.stream_writer(fh) as compressor:
        compressor.write(b'chunk 0')
        compressor.write(b'chunk 1')
        ...
 
 .. important::
 
    If ``flush(FLUSH_FRAME)`` is not called, emitted data doesn't constitute
    a full zstd *frame* and consumers of this data may complain about malformed
    input. It is recommended to use instances as a context manager to ensure
    *frames* are properly finished.
 
 If the size of the data being fed to this streaming compressor is known,
 you can declare it before compression begins::
 
    cctx = zstd.ZstdCompressor()
    with cctx.stream_writer(fh, size=data_len) as compressor:
        compressor.write(chunk0)
        compressor.write(chunk1)
        ...
 
 Declaring the size of the source data allows compression parameters to
 be tuned. And if ``write_content_size`` is used, it also results in the
 content size being written into the frame header of the output data.
 
 The size of chunks being ``write()`` to the destination can be specified::
 
     cctx = zstd.ZstdCompressor()
     with cctx.stream_writer(fh, write_size=32768) as compressor:
         ...
 
 To see how much memory is being used by the streaming compressor::
 
     cctx = zstd.ZstdCompressor()
     with cctx.stream_writer(fh) as compressor:
         ...
         byte_size = compressor.memory_size()
 
 Thte total number of bytes written so far are exposed via ``tell()``::
 
     cctx = zstd.ZstdCompressor()
     with cctx.stream_writer(fh) as compressor:
         ...
         total_written = compressor.tell()
 
 ``stream_writer()`` accepts a ``write_return_read`` boolean argument to control
 the return value of ``write()``. When ``False`` (the default), ``write()`` returns
 the number of bytes that were ``write()``en to the underlying object. When
 ``True``, ``write()`` returns the number of bytes read from the input that
 were subsequently written to the compressor. ``True`` is the *proper* behavior
 for ``write()`` as specified by the ``io.RawIOBase`` interface and will become
 the default value in a future release.
 
 Streaming Output API
 ^^^^^^^^^^^^^^^^^^^^
 
 ``read_to_iter(reader)`` provides a mechanism to stream data out of a
 compressor as an iterator of data chunks.::
 
    cctx = zstd.ZstdCompressor()
    for chunk in cctx.read_to_iter(fh):
         # Do something with emitted data.
 
 ``read_to_iter()`` accepts an object that has a ``read(size)`` method or
 conforms to the buffer protocol.
 
 Uncompressed data is fetched from the source either by calling ``read(size)``
 or by fetching a slice of data from the object directly (in the case where
 the buffer protocol is being used). The returned iterator consists of chunks
 of compressed data.
 
 If reading from the source via ``read()``, ``read()`` will be called until
 it raises or returns an empty bytes (``b''``). It is perfectly valid for
 the source to deliver fewer bytes than were what requested by ``read(size)``.
 
 Like ``stream_writer()``, ``read_to_iter()`` also accepts a ``size`` argument
 declaring the size of the input stream::
 
     cctx = zstd.ZstdCompressor()
     for chunk in cctx.read_to_iter(fh, size=some_int):
         pass
 
 You can also control the size that data is ``read()`` from the source and
 the ideal size of output chunks::
 
     cctx = zstd.ZstdCompressor()
     for chunk in cctx.read_to_iter(fh, read_size=16384, write_size=8192):
         pass
 
 Unlike ``stream_writer()``, ``read_to_iter()`` does not give direct control
 over the sizes of chunks fed into the compressor. Instead, chunk sizes will
 be whatever the object being read from delivers. These will often be of a
 uniform size.
 
 Stream Copying API
 ^^^^^^^^^^^^^^^^^^
 
 ``copy_stream(ifh, ofh)`` can be used to copy data between 2 streams while
 compressing it.::
 
    cctx = zstd.ZstdCompressor()
    cctx.copy_stream(ifh, ofh)
 
 For example, say you wish to compress a file::
 
    cctx = zstd.ZstdCompressor()
    with open(input_path, 'rb') as ifh, open(output_path, 'wb') as ofh:
        cctx.copy_stream(ifh, ofh)
 
 It is also possible to declare the size of the source stream::
 
    cctx = zstd.ZstdCompressor()
    cctx.copy_stream(ifh, ofh, size=len_of_input)
 
 You can also specify how large the chunks that are ``read()`` and ``write()``
 from and to the streams::
 
    cctx = zstd.ZstdCompressor()
    cctx.copy_stream(ifh, ofh, read_size=32768, write_size=16384)
 
 The stream copier returns a 2-tuple of bytes read and written::
 
    cctx = zstd.ZstdCompressor()
    read_count, write_count = cctx.copy_stream(ifh, ofh)
 
 Compressor API
 ^^^^^^^^^^^^^^
 
 ``compressobj()`` returns an object that exposes ``compress(data)`` and
 ``flush()`` methods. Each returns compressed data or an empty bytes.
 
 The purpose of ``compressobj()`` is to provide an API-compatible interface
 with ``zlib.compressobj``, ``bz2.BZ2Compressor``, etc. This allows callers to
 swap in different compressor objects while using the same API.
 
 ``flush()`` accepts an optional argument indicating how to end the stream.
 ``zstd.COMPRESSOBJ_FLUSH_FINISH`` (the default) ends the compression stream.
 Once this type of flush is performed, ``compress()`` and ``flush()`` can
 no longer be called. This type of flush **must** be called to end the
 compression context. If not called, returned data may be incomplete.
 
 A ``zstd.COMPRESSOBJ_FLUSH_BLOCK`` argument to ``flush()`` will flush a
 zstd block. Flushes of this type can be performed multiple times. The next
 call to ``compress()`` will begin a new zstd block.
 
 Here is how this API should be used::
 
    cctx = zstd.ZstdCompressor()
    cobj = cctx.compressobj()
    data = cobj.compress(b'raw input 0')
    data = cobj.compress(b'raw input 1')
    data = cobj.flush()
 
 Or to flush blocks::
 
    cctx.zstd.ZstdCompressor()
    cobj = cctx.compressobj()
    data = cobj.compress(b'chunk in first block')
    data = cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK)
    data = cobj.compress(b'chunk in second block')
    data = cobj.flush()
 
 For best performance results, keep input chunks under 256KB. This avoids
 extra allocations for a large output object.
 
 It is possible to declare the input size of the data that will be fed into
 the compressor::
 
    cctx = zstd.ZstdCompressor()
    cobj = cctx.compressobj(size=6)
    data = cobj.compress(b'foobar')
    data = cobj.flush()
 
 Chunker API
 ^^^^^^^^^^^
 
 ``chunker(size=None, chunk_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE)`` returns
 an object that can be used to iteratively feed chunks of data into a compressor
 and produce output chunks of a uniform size.
 
 The object returned by ``chunker()`` exposes the following methods:
 
 ``compress(data)``
    Feeds new input data into the compressor.
 
 ``flush()``
    Flushes all data currently in the compressor.
 
 ``finish()``
    Signals the end of input data. No new data can be compressed after this
    method is called.
 
 ``compress()``, ``flush()``, and ``finish()`` all return an iterator of
 ``bytes`` instances holding compressed data. The iterator may be empty. Callers
 MUST iterate through all elements of the returned iterator before performing
 another operation on the object.
 
 All chunks emitted by ``compress()`` will have a length of ``chunk_size``.
 
 ``flush()`` and ``finish()`` may return a final chunk smaller than
 ``chunk_size``.
 
 Here is how the API should be used::
 
    cctx = zstd.ZstdCompressor()
    chunker = cctx.chunker(chunk_size=32768)
 
    with open(path, 'rb') as fh:
        while True:
            in_chunk = fh.read(32768)
            if not in_chunk:
                break
 
            for out_chunk in chunker.compress(in_chunk):
                # Do something with output chunk of size 32768.
 
        for out_chunk in chunker.finish():
            # Do something with output chunks that finalize the zstd frame.
 
 The ``chunker()`` API is often a better alternative to ``compressobj()``.
 
 ``compressobj()`` will emit output data as it is available. This results in a
 *stream* of output chunks of varying sizes. The consistency of the output chunk
 size with ``chunker()`` is more appropriate for many usages, such as sending
 compressed data to a socket.
 
 ``compressobj()`` may also perform extra memory reallocations in order to
 dynamically adjust the sizes of the output chunks. Since ``chunker()`` output
 chunks are all the same size (except for flushed or final chunks), there is
 less memory allocation overhead.
 
 Batch Compression API
 ^^^^^^^^^^^^^^^^^^^^^
 
 (Experimental. Not yet supported in CFFI bindings.)
 
 ``multi_compress_to_buffer(data, [threads=0])`` performs compression of multiple
 inputs as a single operation.
 
 Data to be compressed can be passed as a ``BufferWithSegmentsCollection``, a
 ``BufferWithSegments``, or a list containing byte like objects. Each element of
 the container will be compressed individually using the configured parameters
 on the ``ZstdCompressor`` instance.
 
 The ``threads`` argument controls how many threads to use for compression. The
 default is ``0`` which means to use a single thread. Negative values use the
 number of logical CPUs in the machine.
 
 The function returns a ``BufferWithSegmentsCollection``. This type represents
 N discrete memory allocations, eaching holding 1 or more compressed frames.
 
 Output data is written to shared memory buffers. This means that unlike
 regular Python objects, a reference to *any* object within the collection
 keeps the shared buffer and therefore memory backing it alive. This can have
 undesirable effects on process memory usage.
 
 The API and behavior of this function is experimental and will likely change.
 Known deficiencies include:
 
 * If asked to use multiple threads, it will always spawn that many threads,
   even if the input is too small to use them. It should automatically lower
   the thread count when the extra threads would just add overhead.
 * The buffer allocation strategy is fixed. There is room to make it dynamic,
   perhaps even to allow one output buffer per input, facilitating a variation
   of the API to return a list without the adverse effects of shared memory
   buffers.
 
 ZstdDecompressor
 ----------------
 
 The ``ZstdDecompressor`` class provides an interface for performing
 decompression. It is effectively a wrapper around the ``ZSTD_DCtx`` type from
 the C API.
 
 Each instance is associated with parameters that control decompression. These
 come from the following named arguments (all optional):
 
 dict_data
    Compression dictionary to use.
 max_window_size
    Sets an uppet limit on the window size for decompression operations in
    kibibytes. This setting can be used to prevent large memory allocations
    for inputs using large compression windows.
 format
    Set the format of data for the decoder. By default, this is
    ``zstd.FORMAT_ZSTD1``. It can be set to ``zstd.FORMAT_ZSTD1_MAGICLESS`` to
    allow decoding frames without the 4 byte magic header. Not all decompression
    APIs support this mode.
 
 The interface of this class is very similar to ``ZstdCompressor`` (by design).
 
 Unless specified otherwise, assume that no two methods of ``ZstdDecompressor``
 instances can be called from multiple Python threads simultaneously. In other
 words, assume instances are not thread safe unless stated otherwise.
 
 Utility Methods
 ^^^^^^^^^^^^^^^
 
 ``memory_size()`` obtains the size of the underlying zstd decompression context,
 in bytes.::
 
     dctx = zstd.ZstdDecompressor()
     size = dctx.memory_size()
 
 Simple API
 ^^^^^^^^^^
 
 ``decompress(data)`` can be used to decompress an entire compressed zstd
 frame in a single operation.::
 
     dctx = zstd.ZstdDecompressor()
     decompressed = dctx.decompress(data)
 
 By default, ``decompress(data)`` will only work on data written with the content
 size encoded in its header (this is the default behavior of
 ``ZstdCompressor().compress()`` but may not be true for streaming compression). If
 compressed data without an embedded content size is seen, ``zstd.ZstdError`` will
 be raised.
 
 If the compressed data doesn't have its content size embedded within it,
 decompression can be attempted by specifying the ``max_output_size``
 argument.::
 
     dctx = zstd.ZstdDecompressor()
     uncompressed = dctx.decompress(data, max_output_size=1048576)
 
 Ideally, ``max_output_size`` will be identical to the decompressed output
 size.
 
 If ``max_output_size`` is too small to hold the decompressed data,
 ``zstd.ZstdError`` will be raised.
 
 If ``max_output_size`` is larger than the decompressed data, the allocated
 output buffer will be resized to only use the space required.
 
 Please note that an allocation of the requested ``max_output_size`` will be
 performed every time the method is called. Setting to a very large value could
 result in a lot of work for the memory allocator and may result in
 ``MemoryError`` being raised if the allocation fails.
 
 .. important::
 
    If the exact size of decompressed data is unknown (not passed in explicitly
    and not stored in the zstandard frame), for performance reasons it is
    encouraged to use a streaming API.
 
 Stream Reader API
 ^^^^^^^^^^^^^^^^^
 
 ``stream_reader(source)`` can be used to obtain an object conforming to the
 ``io.RawIOBase`` interface for reading decompressed output as a stream::
 
    with open(path, 'rb') as fh:
        dctx = zstd.ZstdDecompressor()
        reader = dctx.stream_reader(fh)
        while True:
            chunk = reader.read(16384)
             if not chunk:
                 break
 
             # Do something with decompressed chunk.
 
 The stream can also be used as a context manager::
 
    with open(path, 'rb') as fh:
        dctx = zstd.ZstdDecompressor()
        with dctx.stream_reader(fh) as reader:
            ...
 
 When used as a context manager, the stream is closed and the underlying
 resources are released when the context manager exits. Future operations against
 the stream will fail.
 
 The ``source`` argument to ``stream_reader()`` can be any object with a
 ``read(size)`` method or any object implementing the *buffer protocol*.
 
 If the ``source`` is a stream, you can specify how large ``read()`` requests
 to that stream should be via the ``read_size`` argument. It defaults to
 ``zstandard.DECOMPRESSION_RECOMMENDED_INPUT_SIZE``.::
 
    with open(path, 'rb') as fh:
        dctx = zstd.ZstdDecompressor()
        # Will perform fh.read(8192) when obtaining data for the decompressor.
        with dctx.stream_reader(fh, read_size=8192) as reader:
            ...
 
 The stream returned by ``stream_reader()`` is not writable.
 
 The stream returned by ``stream_reader()`` is *partially* seekable.
 Absolute and relative positions (``SEEK_SET`` and ``SEEK_CUR``) forward
 of the current position are allowed. Offsets behind the current read
 position and offsets relative to the end of stream are not allowed and
 will raise ``ValueError`` if attempted.
 
 ``tell()`` returns the number of decompressed bytes read so far.
 
 Not all I/O methods are implemented. Notably missing is support for
 ``readline()``, ``readlines()``, and linewise iteration support. This is
 because streams operate on binary data - not text data. If you want to
 convert decompressed output to text, you can chain an ``io.TextIOWrapper``
 to the stream::
 
    with open(path, 'rb') as fh:
        dctx = zstd.ZstdDecompressor()
        stream_reader = dctx.stream_reader(fh)
        text_stream = io.TextIOWrapper(stream_reader, encoding='utf-8')
 
        for line in text_stream:
            ...
 
 The ``read_across_frames`` argument to ``stream_reader()`` controls the
 behavior of read operations when the end of a zstd *frame* is encountered.
 When ``False`` (the default), a read will complete when the end of a
 zstd *frame* is encountered. When ``True``, a read can potentially
 return data spanning multiple zstd *frames*.
 
 Streaming Input API
 ^^^^^^^^^^^^^^^^^^^
 
 ``stream_writer(fh)`` allows you to *stream* data into a decompressor.
 
 Returned instances implement the ``io.RawIOBase`` interface. Only methods
 that involve writing will do useful things.
 
 The argument to ``stream_writer()`` is typically an object that also implements
 ``io.RawIOBase``. But any object with a ``write(data)`` method will work. Many
 common Python types conform to this interface, including open file handles
 and ``io.BytesIO``.
 
 Behavior is similar to ``ZstdCompressor.stream_writer()``: compressed data
 is sent to the decompressor by calling ``write(data)`` and decompressed
 output is written to the underlying stream by calling its ``write(data)``
 method.::
 
     dctx = zstd.ZstdDecompressor()
     decompressor = dctx.stream_writer(fh)
 
     decompressor.write(compressed_data)
     ...
 
 
 Calls to ``write()`` will return the number of bytes written to the output
 object. Not all inputs will result in bytes being written, so return values
 of ``0`` are possible.
 
 Like the ``stream_writer()`` compressor, instances can be used as context
 managers. However, context managers add no extra special behavior and offer
 little to no benefit to being used.
 
 Calling ``close()`` will mark the stream as closed and subsequent I/O operations
 will raise ``ValueError`` (per the documented behavior of ``io.RawIOBase``).
 ``close()`` will also call ``close()`` on the underlying stream if such a
 method exists.
 
 The size of chunks being ``write()`` to the destination can be specified::
 
     dctx = zstd.ZstdDecompressor()
     with dctx.stream_writer(fh, write_size=16384) as decompressor:
         pass
 
 You can see how much memory is being used by the decompressor::
 
     dctx = zstd.ZstdDecompressor()
     with dctx.stream_writer(fh) as decompressor:
         byte_size = decompressor.memory_size()
 
 ``stream_writer()`` accepts a ``write_return_read`` boolean argument to control
 the return value of ``write()``. When ``False`` (the default)``, ``write()``
 returns the number of bytes that were ``write()``en to the underlying stream.
 When ``True``, ``write()`` returns the number of bytes read from the input.
 ``True`` is the *proper* behavior for ``write()`` as specified by the
 ``io.RawIOBase`` interface and will become the default in a future release.
 
 Streaming Output API
 ^^^^^^^^^^^^^^^^^^^^
 
 ``read_to_iter(fh)`` provides a mechanism to stream decompressed data out of a
 compressed source as an iterator of data chunks.:: 
 
     dctx = zstd.ZstdDecompressor()
     for chunk in dctx.read_to_iter(fh):
         # Do something with original data.
 
 ``read_to_iter()`` accepts an object with a ``read(size)`` method that will
 return  compressed bytes or an object conforming to the buffer protocol that
 can expose its data as a contiguous range of bytes.
 
 ``read_to_iter()`` returns an iterator whose elements are chunks of the
 decompressed data.
 
 The size of requested ``read()`` from the source can be specified::
 
     dctx = zstd.ZstdDecompressor()
     for chunk in dctx.read_to_iter(fh, read_size=16384):
         pass
 
 It is also possible to skip leading bytes in the input data::
 
     dctx = zstd.ZstdDecompressor()
     for chunk in dctx.read_to_iter(fh, skip_bytes=1):
         pass
 
 .. tip::
 
    Skipping leading bytes is useful if the source data contains extra
    *header* data. Traditionally, you would need to create a slice or
    ``memoryview`` of the data you want to decompress. This would create
    overhead. It is more efficient to pass the offset into this API.
 
 Similarly to ``ZstdCompressor.read_to_iter()``, the consumer of the iterator
 controls when data is decompressed. If the iterator isn't consumed,
 decompression is put on hold.
 
 When ``read_to_iter()`` is passed an object conforming to the buffer protocol,
 the behavior may seem similar to what occurs when the simple decompression
 API is used. However, this API works when the decompressed size is unknown.
 Furthermore, if feeding large inputs, the decompressor will work in chunks
 instead of performing a single operation.
 
 Stream Copying API
 ^^^^^^^^^^^^^^^^^^
 
 ``copy_stream(ifh, ofh)`` can be used to copy data across 2 streams while
 performing decompression.::
 
     dctx = zstd.ZstdDecompressor()
     dctx.copy_stream(ifh, ofh)
 
 e.g. to decompress a file to another file::
 
     dctx = zstd.ZstdDecompressor()
     with open(input_path, 'rb') as ifh, open(output_path, 'wb') as ofh:
         dctx.copy_stream(ifh, ofh)
 
 The size of chunks being ``read()`` and ``write()`` from and to the streams
 can be specified::
 
     dctx = zstd.ZstdDecompressor()
     dctx.copy_stream(ifh, ofh, read_size=8192, write_size=16384)
 
 Decompressor API
 ^^^^^^^^^^^^^^^^
 
 ``decompressobj()`` returns an object that exposes a ``decompress(data)``
 method. Compressed data chunks are fed into ``decompress(data)`` and
 uncompressed output (or an empty bytes) is returned. Output from subsequent
 calls needs to be concatenated to reassemble the full decompressed byte
 sequence.
 
 The purpose of ``decompressobj()`` is to provide an API-compatible interface
 with ``zlib.decompressobj`` and ``bz2.BZ2Decompressor``. This allows callers
 to swap in different decompressor objects while using the same API.
 
 Each object is single use: once an input frame is decoded, ``decompress()``
 can no longer be called.
 
 Here is how this API should be used::
 
    dctx = zstd.ZstdDecompressor()
    dobj = dctx.decompressobj()
    data = dobj.decompress(compressed_chunk_0)
    data = dobj.decompress(compressed_chunk_1)
 
 By default, calls to ``decompress()`` write output data in chunks of size
 ``DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE``. These chunks are concatenated
 before being returned to the caller. It is possible to define the size of
 these temporary chunks by passing ``write_size`` to ``decompressobj()``::
 
    dctx = zstd.ZstdDecompressor()
    dobj = dctx.decompressobj(write_size=1048576)
 
 .. note::
 
    Because calls to ``decompress()`` may need to perform multiple
    memory (re)allocations, this streaming decompression API isn't as
    efficient as other APIs.
 
 For compatibility with the standard library APIs, instances expose a
 ``flush([length=None])`` method. This method no-ops and has no meaningful
 side-effects, making it safe to call any time.
 
 Batch Decompression API
 ^^^^^^^^^^^^^^^^^^^^^^^
 
 (Experimental. Not yet supported in CFFI bindings.)
 
 ``multi_decompress_to_buffer()`` performs decompression of multiple
 frames as a single operation and returns a ``BufferWithSegmentsCollection``
 containing decompressed data for all inputs.
 
 Compressed frames can be passed to the function as a ``BufferWithSegments``,
 a ``BufferWithSegmentsCollection``, or as a list containing objects that
 conform to the buffer protocol. For best performance, pass a
 ``BufferWithSegmentsCollection`` or a ``BufferWithSegments``, as
 minimal input validation will be done for that type. If calling from
 Python (as opposed to C), constructing one of these instances may add
 overhead cancelling out the performance overhead of validation for list
 inputs.::
 
     dctx = zstd.ZstdDecompressor()
     results = dctx.multi_decompress_to_buffer([b'...', b'...'])
 
 The decompressed size of each frame MUST be discoverable. It can either be
 embedded within the zstd frame (``write_content_size=True`` argument to
 ``ZstdCompressor``) or passed in via the ``decompressed_sizes`` argument.
 
 The ``decompressed_sizes`` argument is an object conforming to the buffer
 protocol which holds an array of 64-bit unsigned integers in the machine's
 native format defining the decompressed sizes of each frame. If this argument
 is passed, it avoids having to scan each frame for its decompressed size.
 This frame scanning can add noticeable overhead in some scenarios.::
 
     frames = [...]
     sizes = struct.pack('=QQQQ', len0, len1, len2, len3)
 
     dctx = zstd.ZstdDecompressor()
     results = dctx.multi_decompress_to_buffer(frames, decompressed_sizes=sizes)
 
 The ``threads`` argument controls the number of threads to use to perform
 decompression operations. The default (``0``) or the value ``1`` means to
 use a single thread. Negative values use the number of logical CPUs in the
 machine.
 
 .. note::
 
    It is possible to pass a ``mmap.mmap()`` instance into this function by
    wrapping it with a ``BufferWithSegments`` instance (which will define the
    offsets of frames within the memory mapped region).
 
 This function is logically equivalent to performing ``dctx.decompress()``
 on each input frame and returning the result.
 
 This function exists to perform decompression on multiple frames as fast
 as possible by having as little overhead as possible. Since decompression is
 performed as a single operation and since the decompressed output is stored in
 a single buffer, extra memory allocations, Python objects, and Python function
 calls are avoided. This is ideal for scenarios where callers know up front that
 they need to access data for multiple frames, such as when  *delta chains* are
 being used.
 
 Currently, the implementation always spawns multiple threads when requested,
 even if the amount of work to do is small. In the future, it will be smarter
 about avoiding threads and their associated overhead when the amount of
 work to do is small.
 
 Prefix Dictionary Chain Decompression
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 ``decompress_content_dict_chain(frames)`` performs decompression of a list of
 zstd frames produced using chained *prefix* dictionary compression. Such
 a list of frames is produced by compressing discrete inputs where each
 non-initial input is compressed with a *prefix* dictionary consisting of the
 content of the previous input.
 
 For example, say you have the following inputs::
 
    inputs = [b'input 1', b'input 2', b'input 3']
 
 The zstd frame chain consists of:
 
 1. ``b'input 1'`` compressed in standalone/discrete mode
 2. ``b'input 2'`` compressed using ``b'input 1'`` as a *prefix* dictionary
 3. ``b'input 3'`` compressed using ``b'input 2'`` as a *prefix* dictionary
 
 Each zstd frame **must** have the content size written.
 
 The following Python code can be used to produce a *prefix dictionary chain*::
 
     def make_chain(inputs):
         frames = []
 
         # First frame is compressed in standalone/discrete mode.
         zctx = zstd.ZstdCompressor()
         frames.append(zctx.compress(inputs[0]))
 
         # Subsequent frames use the previous fulltext as a prefix dictionary
         for i, raw in enumerate(inputs[1:]):
             dict_data = zstd.ZstdCompressionDict(
                 inputs[i], dict_type=zstd.DICT_TYPE_RAWCONTENT)
             zctx = zstd.ZstdCompressor(dict_data=dict_data)
             frames.append(zctx.compress(raw))
 
         return frames
 
 ``decompress_content_dict_chain()`` returns the uncompressed data of the last
 element in the input chain.
 
 
 .. note::
 
    It is possible to implement *prefix dictionary chain* decompression
    on top of other APIs. However, this function will likely be faster -
    especially for long input chains - as it avoids the overhead of instantiating
    and passing around intermediate objects between C and Python.
 
 Multi-Threaded Compression
 --------------------------
 
 ``ZstdCompressor`` accepts a ``threads`` argument that controls the number
 of threads to use for compression. The way this works is that input is split
 into segments and each segment is fed into a worker pool for compression. Once
 a segment is compressed, it is flushed/appended to the output.
 
 .. note::
 
    These threads are created at the C layer and are not Python threads. So they
    work outside the GIL. It is therefore possible to CPU saturate multiple cores
    from Python.
 
 The segment size for multi-threaded compression is chosen from the window size
 of the compressor. This is derived from the ``window_log`` attribute of a
 ``ZstdCompressionParameters`` instance. By default, segment sizes are in the 1+MB
 range.
 
 If multi-threaded compression is requested and the input is smaller than the
 configured segment size, only a single compression thread will be used. If the
 input is smaller than the segment size multiplied by the thread pool size or
 if data cannot be delivered to the compressor fast enough, not all requested
 compressor threads may be active simultaneously.
 
 Compared to non-multi-threaded compression, multi-threaded compression has
 higher per-operation overhead. This includes extra memory operations,
 thread creation, lock acquisition, etc.
 
 Due to the nature of multi-threaded compression using *N* compression
 *states*, the output from multi-threaded compression will likely be larger
 than non-multi-threaded compression. The difference is usually small. But
 there is a CPU/wall time versus size trade off that may warrant investigation.
 
 Output from multi-threaded compression does not require any special handling
 on the decompression side. To the decompressor, data generated with single
 threaded compressor looks the same as data generated by a multi-threaded
 compressor and does not require any special handling or additional resource
 requirements.
 
 Dictionary Creation and Management
 ----------------------------------
 
 Compression dictionaries are represented with the ``ZstdCompressionDict`` type.
 
 Instances can be constructed from bytes::
 
    dict_data = zstd.ZstdCompressionDict(data)
 
 It is possible to construct a dictionary from *any* data. If the data doesn't
 begin with a magic header, it will be treated as a *prefix* dictionary.
 *Prefix* dictionaries allow compression operations to reference raw data
 within the dictionary.
 
 It is possible to force the use of *prefix* dictionaries or to require a
 dictionary header:
 
    dict_data = zstd.ZstdCompressionDict(data,
                                         dict_type=zstd.DICT_TYPE_RAWCONTENT)
 
    dict_data = zstd.ZstdCompressionDict(data,
                                         dict_type=zstd.DICT_TYPE_FULLDICT)
 
 You can see how many bytes are in the dictionary by calling ``len()``::
 
    dict_data = zstd.train_dictionary(size, samples)
    dict_size = len(dict_data)  # will not be larger than ``size``
 
 Once you have a dictionary, you can pass it to the objects performing
 compression and decompression::
 
    dict_data = zstd.train_dictionary(131072, samples)
 
    cctx = zstd.ZstdCompressor(dict_data=dict_data)
    for source_data in input_data:
        compressed = cctx.compress(source_data)
        # Do something with compressed data.
 
    dctx = zstd.ZstdDecompressor(dict_data=dict_data)
    for compressed_data in input_data:
        buffer = io.BytesIO()
        with dctx.stream_writer(buffer) as decompressor:
            decompressor.write(compressed_data)
        # Do something with raw data in ``buffer``.
 
 Dictionaries have unique integer IDs. You can retrieve this ID via::
 
    dict_id = zstd.dictionary_id(dict_data)
 
 You can obtain the raw data in the dict (useful for persisting and constructing
 a ``ZstdCompressionDict`` later) via ``as_bytes()``::
 
    dict_data = zstd.train_dictionary(size, samples)
    raw_data = dict_data.as_bytes()
 
 By default, when a ``ZstdCompressionDict`` is *attached* to a
 ``ZstdCompressor``, each ``ZstdCompressor`` performs work to prepare the
 dictionary for use. This is fine if only 1 compression operation is being
 performed or if the ``ZstdCompressor`` is being reused for multiple operations.
 But if multiple ``ZstdCompressor`` instances are being used with the dictionary,
 this can add overhead.
 
 It is possible to *precompute* the dictionary so it can readily be consumed
 by multiple ``ZstdCompressor`` instances::
 
     d = zstd.ZstdCompressionDict(data)
 
     # Precompute for compression level 3.
     d.precompute_compress(level=3)
 
     # Precompute with specific compression parameters.
     params = zstd.ZstdCompressionParameters(...)
     d.precompute_compress(compression_params=params)
 
 .. note::
 
    When a dictionary is precomputed, the compression parameters used to
    precompute the dictionary overwrite some of the compression parameters
    specified to ``ZstdCompressor.__init__``.
 
 Training Dictionaries
 ^^^^^^^^^^^^^^^^^^^^^
 
 Unless using *prefix* dictionaries, dictionary data is produced by *training*
 on existing data::
 
    dict_data = zstd.train_dictionary(size, samples)
 
 This takes a target dictionary size and list of bytes instances and creates and
 returns a ``ZstdCompressionDict``.
 
 The dictionary training mechanism is known as *cover*. More details about it are
 available in the paper *Effective Construction of Relative Lempel-Ziv
 Dictionaries* (authors: Liao, Petri, Moffat, Wirth).
 
 The cover algorithm takes parameters ``k` and ``d``. These are the
 *segment size* and *dmer size*, respectively. The returned dictionary
 instance created by this function has ``k`` and ``d`` attributes
 containing the values for these parameters. If a ``ZstdCompressionDict``
 is constructed from raw bytes data (a content-only dictionary), the
 ``k`` and ``d`` attributes will be ``0``.
 
 The segment and dmer size parameters to the cover algorithm can either be
 specified manually or ``train_dictionary()`` can try multiple values
 and pick the best one, where *best* means the smallest compressed data size.
 This later mode is called *optimization* mode.
 
 If none of ``k``, ``d``, ``steps``, ``threads``, ``level``, ``notifications``,
 or ``dict_id`` (basically anything from the underlying ``ZDICT_cover_params_t``
 struct) are defined, *optimization* mode is used with default parameter
 values.
 
 If ``steps`` or ``threads`` are defined, then *optimization* mode is engaged
 with explicit control over those parameters. Specifying ``threads=0`` or
 ``threads=1`` can be used to engage *optimization* mode if other parameters
 are not defined.
 
 Otherwise, non-*optimization* mode is used with the parameters specified.
 
 This function takes the following arguments:
 
 dict_size
    Target size in bytes of the dictionary to generate.
 samples
    A list of bytes holding samples the dictionary will be trained from.
 k
    Parameter to cover algorithm defining the segment size. A reasonable range
    is [16, 2048+].
 d
    Parameter to cover algorithm defining the dmer size. A reasonable range is
    [6, 16]. ``d`` must be less than or equal to ``k``.
 dict_id
    Integer dictionary ID for the produced dictionary. Default is 0, which uses
    a random value.
 steps
    Number of steps through ``k`` values to perform when trying parameter
    variations.
 threads
    Number of threads to use when trying parameter variations. Default is 0,
    which means to use a single thread. A negative value can be specified to
    use as many threads as there are detected logical CPUs.
 level
    Integer target compression level when trying parameter variations.
 notifications
    Controls writing of informational messages to ``stderr``. ``0`` (the
    default) means to write nothing. ``1`` writes errors. ``2`` writes
    progression info. ``3`` writes more details. And ``4`` writes all info.
 
 Explicit Compression Parameters
 -------------------------------
 
 Zstandard offers a high-level *compression level* that maps to lower-level
 compression parameters. For many consumers, this numeric level is the only
 compression setting you'll need to touch.
 
 But for advanced use cases, it might be desirable to tweak these lower-level
 settings.
 
 The ``ZstdCompressionParameters`` type represents these low-level compression
 settings.
 
 Instances of this type can be constructed from a myriad of keyword arguments
 (defined below) for complete low-level control over each adjustable
 compression setting.
 
 From a higher level, one can construct a ``ZstdCompressionParameters`` instance
 given a desired compression level and target input and dictionary size
 using ``ZstdCompressionParameters.from_level()``. e.g.::
 
     # Derive compression settings for compression level 7.
     params = zstd.ZstdCompressionParameters.from_level(7)
 
     # With an input size of 1MB
     params = zstd.ZstdCompressionParameters.from_level(7, source_size=1048576)
 
 Using ``from_level()``, it is also possible to override individual compression
 parameters or to define additional settings that aren't automatically derived.
 e.g.::
 
     params = zstd.ZstdCompressionParameters.from_level(4, window_log=10)
     params = zstd.ZstdCompressionParameters.from_level(5, threads=4)
 
 Or you can define low-level compression settings directly::
 
     params = zstd.ZstdCompressionParameters(window_log=12, enable_ldm=True)
 
 Once a ``ZstdCompressionParameters`` instance is obtained, it can be used to
 configure a compressor::
 
     cctx = zstd.ZstdCompressor(compression_params=params)
 
 The named arguments and attributes of ``ZstdCompressionParameters`` are as
 follows:
 
 * format
 * compression_level
 * window_log
 * hash_log
 * chain_log
 * search_log
 * min_match
 * target_length
 * strategy
 * compression_strategy (deprecated: same as ``strategy``)
 * write_content_size
 * write_checksum
 * write_dict_id
 * job_size
 * overlap_log
 * overlap_size_log (deprecated: same as ``overlap_log``)
 * force_max_window
 * enable_ldm
 * ldm_hash_log
 * ldm_min_match
 * ldm_bucket_size_log
 * ldm_hash_rate_log
 * ldm_hash_every_log (deprecated: same as ``ldm_hash_rate_log``)
 * threads
 
 Some of these are very low-level settings. It may help to consult the official
 zstandard documentation for their behavior. Look for the ``ZSTD_p_*`` constants
 in ``zstd.h`` (https://github.com/facebook/zstd/blob/dev/lib/zstd.h).
 
 Frame Inspection
 ----------------
 
 Data emitted from zstd compression is encapsulated in a *frame*. This frame
 begins with a 4 byte *magic number* header followed by 2 to 14 bytes describing
 the frame in more detail. For more info, see
 https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md.
 
 ``zstd.get_frame_parameters(data)`` parses a zstd *frame* header from a bytes
 instance and return a ``FrameParameters`` object describing the frame.
 
 Depending on which fields are present in the frame and their values, the
 length of the frame parameters varies. If insufficient bytes are passed
 in to fully parse the frame parameters, ``ZstdError`` is raised. To ensure
 frame parameters can be parsed, pass in at least 18 bytes.
 
 ``FrameParameters`` instances have the following attributes:
 
 content_size
    Integer size of original, uncompressed content. This will be ``0`` if the
    original content size isn't written to the frame (controlled with the
    ``write_content_size`` argument to ``ZstdCompressor``) or if the input
    content size was ``0``.
 
 window_size
    Integer size of maximum back-reference distance in compressed data.
 
 dict_id
    Integer of dictionary ID used for compression. ``0`` if no dictionary
    ID was used or if the dictionary ID was ``0``.
 
 has_checksum
    Bool indicating whether a 4 byte content checksum is stored at the end
    of the frame.
 
 ``zstd.frame_header_size(data)`` returns the size of the zstandard frame
 header.
 
 ``zstd.frame_content_size(data)`` returns the content size as parsed from
 the frame header. ``-1`` means the content size is unknown. ``0`` means
 an empty frame. The content size is usually correct. However, it may not
 be accurate.
 
 Misc Functionality
 ------------------
 
 estimate_decompression_context_size()
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Estimate the memory size requirements for a decompressor instance.
 
 Constants
 ---------
 
 The following module constants/attributes are exposed:
 
 ZSTD_VERSION
     This module attribute exposes a 3-tuple of the Zstandard version. e.g.
     ``(1, 0, 0)``
 MAX_COMPRESSION_LEVEL
     Integer max compression level accepted by compression functions
 COMPRESSION_RECOMMENDED_INPUT_SIZE
     Recommended chunk size to feed to compressor functions
 COMPRESSION_RECOMMENDED_OUTPUT_SIZE
     Recommended chunk size for compression output
 DECOMPRESSION_RECOMMENDED_INPUT_SIZE
     Recommended chunk size to feed into decompresor functions
 DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE
     Recommended chunk size for decompression output
 
 FRAME_HEADER
     bytes containing header of the Zstandard frame
 MAGIC_NUMBER
     Frame header as an integer
 
 FLUSH_BLOCK
     Flushing behavior that denotes to flush a zstd block. A decompressor will
     be able to decode all data fed into the compressor so far.
 FLUSH_FRAME
     Flushing behavior that denotes to end a zstd frame. Any new data fed
     to the compressor will start a new frame.
 
 CONTENTSIZE_UNKNOWN
     Value for content size when the content size is unknown.
 CONTENTSIZE_ERROR
     Value for content size when content size couldn't be determined.
 
 WINDOWLOG_MIN
     Minimum value for compression parameter
 WINDOWLOG_MAX
     Maximum value for compression parameter
 CHAINLOG_MIN
     Minimum value for compression parameter
 CHAINLOG_MAX
     Maximum value for compression parameter
 HASHLOG_MIN
     Minimum value for compression parameter
 HASHLOG_MAX
     Maximum value for compression parameter
 SEARCHLOG_MIN
     Minimum value for compression parameter
 SEARCHLOG_MAX
     Maximum value for compression parameter
 MINMATCH_MIN
     Minimum value for compression parameter
 MINMATCH_MAX
     Maximum value for compression parameter
 SEARCHLENGTH_MIN
     Minimum value for compression parameter
 
     Deprecated: use ``MINMATCH_MIN``
 SEARCHLENGTH_MAX
     Maximum value for compression parameter
 
     Deprecated: use ``MINMATCH_MAX``
 TARGETLENGTH_MIN
     Minimum value for compression parameter
 STRATEGY_FAST
     Compression strategy
 STRATEGY_DFAST
     Compression strategy
 STRATEGY_GREEDY
     Compression strategy
 STRATEGY_LAZY
     Compression strategy
 STRATEGY_LAZY2
     Compression strategy
 STRATEGY_BTLAZY2
     Compression strategy
 STRATEGY_BTOPT
     Compression strategy
 STRATEGY_BTULTRA
     Compression strategy
 STRATEGY_BTULTRA2
     Compression strategy
 
 FORMAT_ZSTD1
     Zstandard frame format
 FORMAT_ZSTD1_MAGICLESS
     Zstandard frame format without magic header
 
 Performance Considerations
 --------------------------
 
 The ``ZstdCompressor`` and ``ZstdDecompressor`` types maintain state to a
 persistent compression or decompression *context*. Reusing a ``ZstdCompressor``
 or ``ZstdDecompressor`` instance for multiple operations is faster than
 instantiating a new ``ZstdCompressor`` or ``ZstdDecompressor`` for each
 operation. The differences are magnified as the size of data decreases. For
 example, the difference between *context* reuse and non-reuse for 100,000
 100 byte inputs will be significant (possiby over 10x faster to reuse contexts)
 whereas 10 100,000,000 byte inputs will be more similar in speed (because the
 time spent doing compression dwarfs time spent creating new *contexts*).
 
 Buffer Types
 ------------
 
 The API exposes a handful of custom types for interfacing with memory buffers.
 The primary goal of these types is to facilitate efficient multi-object
 operations.
 
 The essential idea is to have a single memory allocation provide backing
 storage for multiple logical objects. This has 2 main advantages: fewer
 allocations and optimal memory access patterns. This avoids having to allocate
 a Python object for each logical object and furthermore ensures that access of
 data for objects can be sequential (read: fast) in memory.
 
 BufferWithSegments
 ^^^^^^^^^^^^^^^^^^
 
 The ``BufferWithSegments`` type represents a memory buffer containing N
 discrete items of known lengths (segments). It is essentially a fixed size
 memory address and an array of 2-tuples of ``(offset, length)`` 64-bit
 unsigned native endian integers defining the byte offset and length of each
 segment within the buffer.
 
 Instances behave like containers.
 
 ``len()`` returns the number of segments within the instance.
 
 ``o[index]`` or ``__getitem__`` obtains a ``BufferSegment`` representing an
 individual segment within the backing buffer. That returned object references
 (not copies) memory. This means that iterating all objects doesn't copy
 data within the buffer.
 
 The ``.size`` attribute contains the total size in bytes of the backing
 buffer.
 
 Instances conform to the buffer protocol. So a reference to the backing bytes
 can be obtained via ``memoryview(o)``. A *copy* of the backing bytes can also
 be obtained via ``.tobytes()``.
 
 The ``.segments`` attribute exposes the array of ``(offset, length)`` for
 segments within the buffer. It is a ``BufferSegments`` type.
 
 BufferSegment
 ^^^^^^^^^^^^^
 
 The ``BufferSegment`` type represents a segment within a ``BufferWithSegments``.
 It is essentially a reference to N bytes within a ``BufferWithSegments``.
 
 ``len()`` returns the length of the segment in bytes.
 
 ``.offset`` contains the byte offset of this segment within its parent
 ``BufferWithSegments`` instance.
 
 The object conforms to the buffer protocol. ``.tobytes()`` can be called to
 obtain a ``bytes`` instance with a copy of the backing bytes.
 
 BufferSegments
 ^^^^^^^^^^^^^^
 
 This type represents an array of ``(offset, length)`` integers defining segments
 within a ``BufferWithSegments``.
 
 The array members are 64-bit unsigned integers using host/native bit order.
 
 Instances conform to the buffer protocol.
 
 BufferWithSegmentsCollection
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The ``BufferWithSegmentsCollection`` type represents a virtual spanning view
 of multiple ``BufferWithSegments`` instances.
 
 Instances are constructed from 1 or more ``BufferWithSegments`` instances. The
 resulting object behaves like an ordered sequence whose members are the
 segments within each ``BufferWithSegments``.
 
 ``len()`` returns the number of segments within all ``BufferWithSegments``
 instances.
 
 ``o[index]`` and ``__getitem__(index)`` return the ``BufferSegment`` at
 that offset as if all ``BufferWithSegments`` instances were a single
 entity.
 
 If the object is composed of 2 ``BufferWithSegments`` instances with the
 first having 2 segments and the second have 3 segments, then ``b[0]``
 and ``b[1]`` access segments in the first object and ``b[2]``, ``b[3]``,
 and ``b[4]`` access segments from the second.
 
 Choosing an API
 ===============
 
 There are multiple APIs for performing compression and decompression. This is
 because different applications have different needs and the library wants to
 facilitate optimal use in as many use cases as possible.
 
 From a high-level, APIs are divided into *one-shot* and *streaming*: either you
 are operating on all data at once or you operate on it piecemeal.
 
 The *one-shot* APIs are useful for small data, where the input or output
 size is known. (The size can come from a buffer length, file size, or
 stored in the zstd frame header.) A limitation of the *one-shot* APIs is that
 input and output must fit in memory simultaneously. For say a 4 GB input,
 this is often not feasible.
 
 The *one-shot* APIs also perform all work as a single operation. So, if you
 feed it large input, it could take a long time for the function to return.
 
 The streaming APIs do not have the limitations of the simple API. But the
 price you pay for this flexibility is that they are more complex than a
 single function call.
 
 The streaming APIs put the caller in control of compression and decompression
 behavior by allowing them to directly control either the input or output side
 of the operation.
 
 With the *streaming input*, *compressor*, and *decompressor* APIs, the caller
 has full control over the input to the compression or decompression stream.
 They can directly choose when new data is operated on.
 
 With the *streaming ouput* APIs, the caller has full control over the output
 of the compression or decompression stream. It can choose when to receive
 new data.
 
 When using the *streaming* APIs that operate on file-like or stream objects,
 it is important to consider what happens in that object when I/O is requested.
 There is potential for long pauses as data is read or written from the
 underlying stream (say from interacting with a filesystem or network). This
 could add considerable overhead.
 
 Thread Safety
 =============
 
 ``ZstdCompressor`` and ``ZstdDecompressor`` instances have no guarantees
 about thread safety. Do not operate on the same ``ZstdCompressor`` and
 ``ZstdDecompressor`` instance simultaneously from different threads. It is
 fine to have different threads call into a single instance, just not at the
 same time.
 
 Some operations require multiple function calls to complete. e.g. streaming
 operations. A single ``ZstdCompressor`` or ``ZstdDecompressor`` cannot be used
 for simultaneously active operations. e.g. you must not start a streaming
 operation when another streaming operation is already active.
 
 The C extension releases the GIL during non-trivial calls into the zstd C
 API. Non-trivial calls are notably compression and decompression. Trivial
 calls are things like parsing frame parameters. Where the GIL is released
 is considered an implementation detail and can change in any release.
 
 APIs that accept bytes-like objects don't enforce that the underlying object
 is read-only. However, it is assumed that the passed object is read-only for
 the duration of the function call. It is possible to pass a mutable object
 (like a ``bytearray``) to e.g. ``ZstdCompressor.compress()``, have the GIL
 released, and mutate the object from another thread. Such a race condition
 is a bug in the consumer of python-zstandard. Most Python data types are
 immutable, so unless you are doing something fancy, you don't need to
 worry about this.
 
 Note on Zstandard's *Experimental* API
 ======================================
 
 Many of the Zstandard APIs used by this module are marked as *experimental*
 within the Zstandard project.
 
 It is unclear how Zstandard's C API will evolve over time, especially with
 regards to this *experimental* functionality. We will try to maintain
 backwards compatibility at the Python API level. However, we cannot
 guarantee this for things not under our control.
 
 Since a copy of the Zstandard source code is distributed with this
 module and since we compile against it, the behavior of a specific
 version of this module should be constant for all of time. So if you
 pin the version of this module used in your projects (which is a Python
 best practice), you should be shielded from unwanted future changes.
 
 Donate
 ======
 
 A lot of time has been invested into this project by the author.
 
 If you find this project useful and would like to thank the author for
 their work, consider donating some money. Any amount is appreciated.
 
 .. image:: https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif
     :target: https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=gregory%2eszorc%40gmail%2ecom&lc=US&item_name=python%2dzstandard&currency_code=USD&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted
     :alt: Donate via PayPal
 
-.. |ci-status| image:: https://travis-ci.org/indygreg/python-zstandard.svg?branch=master
-    :target: https://travis-ci.org/indygreg/python-zstandard
-
-.. |win-ci-status| image:: https://ci.appveyor.com/api/projects/status/github/indygreg/python-zstandard?svg=true
-    :target: https://ci.appveyor.com/project/indygreg/python-zstandard
-    :alt: Windows build status
+.. |ci-status| image:: https://dev.azure.com/gregoryszorc/python-zstandard/_apis/build/status/indygreg.python-zstandard?branchName=master
+    :target: https://dev.azure.com/gregoryszorc/python-zstandard/_apis/build/status/indygreg.python-zstandard?branchName=master

contrib/python-zstandard/c-ext/compressionparams.c

@@ -1,572 +1,572 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value) {
-	size_t zresult = ZSTD_CCtxParam_setParameter(params, param, value);
+	size_t zresult = ZSTD_CCtxParams_setParameter(params, param, value);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "unable to set compression context parameter: %s",
 			ZSTD_getErrorName(zresult));
 		return 1;
 	}
 
 	return 0;
 }
 
 #define TRY_SET_PARAMETER(params, param, value) if (set_parameter(params, param, value)) return -1;
 
 #define TRY_COPY_PARAMETER(source, dest, param) { \
 	int result; \
-	size_t zresult = ZSTD_CCtxParam_getParameter(source, param, &result); \
+	size_t zresult = ZSTD_CCtxParams_getParameter(source, param, &result); \
 	if (ZSTD_isError(zresult)) { \
 		return 1; \
 	} \
-	zresult = ZSTD_CCtxParam_setParameter(dest, param, result); \
+	zresult = ZSTD_CCtxParams_setParameter(dest, param, result); \
 	if (ZSTD_isError(zresult)) { \
 		return 1; \
 	} \
 }
 
 int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj) {
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_nbWorkers);
 
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_format);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_compressionLevel);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_windowLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_hashLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_chainLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_searchLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_minMatch);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_targetLength);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_strategy);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_contentSizeFlag);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_checksumFlag);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_dictIDFlag);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_jobSize);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_overlapLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_forceMaxWindow);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_enableLongDistanceMatching);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmHashLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmMinMatch);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmBucketSizeLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmHashRateLog);
 
 	return 0;
 }
 
 int reset_params(ZstdCompressionParametersObject* params) {
 	if (params->params) {
 		ZSTD_CCtxParams_reset(params->params);
 	}
 	else {
 		params->params = ZSTD_createCCtxParams();
 		if (!params->params) {
 			PyErr_NoMemory();
 			return 1;
 		}
 	}
 
 	return set_parameters(params->params, params);
 }
 
 #define TRY_GET_PARAMETER(params, param, value) { \
-    size_t zresult = ZSTD_CCtxParam_getParameter(params, param, value); \
+    size_t zresult = ZSTD_CCtxParams_getParameter(params, param, value); \
     if (ZSTD_isError(zresult)) { \
         PyErr_Format(ZstdError, "unable to retrieve parameter: %s", ZSTD_getErrorName(zresult)); \
         return 1; \
     } \
 }
 
 int to_cparams(ZstdCompressionParametersObject* params, ZSTD_compressionParameters* cparams) {
 	int value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_windowLog, &value);
 	cparams->windowLog = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_chainLog, &value);
 	cparams->chainLog = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_hashLog, &value);
 	cparams->hashLog = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_searchLog, &value);
 	cparams->searchLog = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_minMatch, &value);
 	cparams->minMatch = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_targetLength, &value);
 	cparams->targetLength = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_strategy, &value);
 	cparams->strategy = value;
 
 	return 0;
 }
 
 static int ZstdCompressionParameters_init(ZstdCompressionParametersObject* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"format",
 		"compression_level",
 		"window_log",
 		"hash_log",
 		"chain_log",
 		"search_log",
 		"min_match",
 		"target_length",
 		"compression_strategy",
 		"strategy",
 		"write_content_size",
 		"write_checksum",
 		"write_dict_id",
 		"job_size",
 		"overlap_log",
 		"overlap_size_log",
 		"force_max_window",
 		"enable_ldm",
 		"ldm_hash_log",
 		"ldm_min_match",
 		"ldm_bucket_size_log",
 		"ldm_hash_rate_log",
 		"ldm_hash_every_log",
 		"threads",
 		NULL
 	};
 
 	int format = 0;
 	int compressionLevel = 0;
 	int windowLog = 0;
 	int hashLog = 0;
 	int chainLog = 0;
 	int searchLog = 0;
 	int minMatch = 0;
 	int targetLength = 0;
 	int compressionStrategy = -1;
 	int strategy = -1;
 	int contentSizeFlag = 1;
 	int checksumFlag = 0;
 	int dictIDFlag = 0;
 	int jobSize = 0;
 	int overlapLog = -1;
 	int overlapSizeLog = -1;
 	int forceMaxWindow = 0;
 	int enableLDM = 0;
 	int ldmHashLog = 0;
 	int ldmMinMatch = 0;
 	int ldmBucketSizeLog = 0;
 	int ldmHashRateLog = -1;
 	int ldmHashEveryLog = -1;
 	int threads = 0;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs,
 		"|iiiiiiiiiiiiiiiiiiiiiiii:CompressionParameters",
 		kwlist, &format, &compressionLevel, &windowLog, &hashLog, &chainLog,
 		&searchLog, &minMatch, &targetLength, &compressionStrategy, &strategy,
 		&contentSizeFlag, &checksumFlag, &dictIDFlag, &jobSize, &overlapLog,
 		&overlapSizeLog, &forceMaxWindow, &enableLDM, &ldmHashLog, &ldmMinMatch,
 		&ldmBucketSizeLog, &ldmHashRateLog, &ldmHashEveryLog, &threads)) {
 		return -1;
 	}
 
 	if (reset_params(self)) {
 		return -1;
 	}
 
 	if (threads < 0) {
 		threads = cpu_count();
 	}
 
 	/* We need to set ZSTD_c_nbWorkers before ZSTD_c_jobSize and ZSTD_c_overlapLog
 	 * because setting ZSTD_c_nbWorkers resets the other parameters. */
 	TRY_SET_PARAMETER(self->params, ZSTD_c_nbWorkers, threads);
 
 	TRY_SET_PARAMETER(self->params, ZSTD_c_format, format);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_compressionLevel, compressionLevel);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_windowLog, windowLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_hashLog, hashLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_chainLog, chainLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_searchLog, searchLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_minMatch, minMatch);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_targetLength, targetLength);
 
 	if (compressionStrategy != -1 && strategy != -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot specify both compression_strategy and strategy");
 		return -1;
     }
 
 	if (compressionStrategy != -1) {
 		strategy = compressionStrategy;
 	}
 	else if (strategy == -1) {
 		strategy = 0;
 	}
 
 	TRY_SET_PARAMETER(self->params, ZSTD_c_strategy, strategy);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_contentSizeFlag, contentSizeFlag);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_checksumFlag, checksumFlag);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_dictIDFlag, dictIDFlag);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_jobSize, jobSize);
 
 	if (overlapLog != -1 && overlapSizeLog != -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot specify both overlap_log and overlap_size_log");
 		return -1;
 	}
 
 	if (overlapSizeLog != -1) {
 		overlapLog = overlapSizeLog;
 	}
 	else if (overlapLog == -1) {
 		overlapLog = 0;
 	}
 
 	TRY_SET_PARAMETER(self->params, ZSTD_c_overlapLog, overlapLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_forceMaxWindow, forceMaxWindow);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_enableLongDistanceMatching, enableLDM);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmHashLog, ldmHashLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmMinMatch, ldmMinMatch);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmBucketSizeLog, ldmBucketSizeLog);
 
 	if (ldmHashRateLog != -1 && ldmHashEveryLog != -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot specify both ldm_hash_rate_log and ldm_hash_everyLog");
 		return -1;
 	}
 
 	if (ldmHashEveryLog != -1) {
 		ldmHashRateLog = ldmHashEveryLog;
 	}
 	else if (ldmHashRateLog == -1) {
 		ldmHashRateLog = 0;
 	}
 
 	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmHashRateLog, ldmHashRateLog);
 
 	return 0;
 }
 
 PyDoc_STRVAR(ZstdCompressionParameters_from_level__doc__,
 "Create a CompressionParameters from a compression level and target sizes\n"
 );
 
 ZstdCompressionParametersObject* CompressionParameters_from_level(PyObject* undef, PyObject* args, PyObject* kwargs) {
 	int managedKwargs = 0;
 	int level;
 	PyObject* sourceSize = NULL;
 	PyObject* dictSize = NULL;
 	unsigned PY_LONG_LONG iSourceSize = 0;
 	Py_ssize_t iDictSize = 0;
 	PyObject* val;
 	ZSTD_compressionParameters params;
 	ZstdCompressionParametersObject* result = NULL;
 	int res;
 
 	if (!PyArg_ParseTuple(args, "i:from_level",
 		&level)) {
 		return NULL;
 	}
 
 	if (!kwargs) {
 		kwargs = PyDict_New();
 		if (!kwargs) {
 			return NULL;
 		}
 		managedKwargs = 1;
 	}
 
 	sourceSize = PyDict_GetItemString(kwargs, "source_size");
 	if (sourceSize) {
 #if PY_MAJOR_VERSION >= 3
 		iSourceSize = PyLong_AsUnsignedLongLong(sourceSize);
 		if (iSourceSize == (unsigned PY_LONG_LONG)(-1)) {
 			goto cleanup;
 		}
 #else
 		iSourceSize = PyInt_AsUnsignedLongLongMask(sourceSize);
 #endif
 
 		PyDict_DelItemString(kwargs, "source_size");
 	}
 
 	dictSize = PyDict_GetItemString(kwargs, "dict_size");
 	if (dictSize) {
 #if PY_MAJOR_VERSION >= 3
 		iDictSize = PyLong_AsSsize_t(dictSize);
 #else
 		iDictSize = PyInt_AsSsize_t(dictSize);
 #endif
 		if (iDictSize == -1) {
 			goto cleanup;
 		}
 
 		PyDict_DelItemString(kwargs, "dict_size");
 	}
 
 
 	params = ZSTD_getCParams(level, iSourceSize, iDictSize);
 
 	/* Values derived from the input level and sizes are passed along to the
 	   constructor. But only if a value doesn't already exist. */
 	val = PyDict_GetItemString(kwargs, "window_log");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.windowLog);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "window_log", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "chain_log");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.chainLog);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "chain_log", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "hash_log");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.hashLog);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "hash_log", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "search_log");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.searchLog);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "search_log", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "min_match");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.minMatch);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "min_match", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "target_length");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.targetLength);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "target_length", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "compression_strategy");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.strategy);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "compression_strategy", val);
 		Py_DECREF(val);
 	}
 
 	result = PyObject_New(ZstdCompressionParametersObject, &ZstdCompressionParametersType);
 	if (!result) {
 		goto cleanup;
 	}
 
 	result->params = NULL;
 
 	val = PyTuple_New(0);
 	if (!val) {
 		Py_CLEAR(result);
 		goto cleanup;
 	}
 
 	res = ZstdCompressionParameters_init(result, val, kwargs);
 	Py_DECREF(val);
 
 	if (res) {
 		Py_CLEAR(result);
 		goto cleanup;
 	}
 
 cleanup:
 	if (managedKwargs) {
 		Py_DECREF(kwargs);
 	}
 
 	return result;
 }
 
 PyDoc_STRVAR(ZstdCompressionParameters_estimated_compression_context_size__doc__,
 "Estimate the size in bytes of a compression context for compression parameters\n"
 );
 
 PyObject* ZstdCompressionParameters_estimated_compression_context_size(ZstdCompressionParametersObject* self) {
 	return PyLong_FromSize_t(ZSTD_estimateCCtxSize_usingCCtxParams(self->params));
 }
 
 PyDoc_STRVAR(ZstdCompressionParameters__doc__,
 "ZstdCompressionParameters: low-level control over zstd compression");
 
 static void ZstdCompressionParameters_dealloc(ZstdCompressionParametersObject* self) {
 	if (self->params) {
 		ZSTD_freeCCtxParams(self->params);
 		self->params = NULL;
 	}
 
 	PyObject_Del(self);
 }
 
 #define PARAM_GETTER(name, param) PyObject* ZstdCompressionParameters_get_##name(PyObject* self, void* unused) { \
     int result; \
     size_t zresult; \
     ZstdCompressionParametersObject* p = (ZstdCompressionParametersObject*)(self); \
-    zresult = ZSTD_CCtxParam_getParameter(p->params, param, &result); \
+    zresult = ZSTD_CCtxParams_getParameter(p->params, param, &result); \
     if (ZSTD_isError(zresult)) { \
         PyErr_Format(ZstdError, "unable to get compression parameter: %s", \
             ZSTD_getErrorName(zresult)); \
         return NULL; \
     } \
     return PyLong_FromLong(result); \
 }
 
 PARAM_GETTER(format, ZSTD_c_format)
 PARAM_GETTER(compression_level, ZSTD_c_compressionLevel)
 PARAM_GETTER(window_log, ZSTD_c_windowLog)
 PARAM_GETTER(hash_log, ZSTD_c_hashLog)
 PARAM_GETTER(chain_log, ZSTD_c_chainLog)
 PARAM_GETTER(search_log, ZSTD_c_searchLog)
 PARAM_GETTER(min_match, ZSTD_c_minMatch)
 PARAM_GETTER(target_length, ZSTD_c_targetLength)
 PARAM_GETTER(compression_strategy, ZSTD_c_strategy)
 PARAM_GETTER(write_content_size, ZSTD_c_contentSizeFlag)
 PARAM_GETTER(write_checksum, ZSTD_c_checksumFlag)
 PARAM_GETTER(write_dict_id, ZSTD_c_dictIDFlag)
 PARAM_GETTER(job_size, ZSTD_c_jobSize)
 PARAM_GETTER(overlap_log, ZSTD_c_overlapLog)
 PARAM_GETTER(force_max_window, ZSTD_c_forceMaxWindow)
 PARAM_GETTER(enable_ldm, ZSTD_c_enableLongDistanceMatching)
 PARAM_GETTER(ldm_hash_log, ZSTD_c_ldmHashLog)
 PARAM_GETTER(ldm_min_match, ZSTD_c_ldmMinMatch)
 PARAM_GETTER(ldm_bucket_size_log, ZSTD_c_ldmBucketSizeLog)
 PARAM_GETTER(ldm_hash_rate_log, ZSTD_c_ldmHashRateLog)
 PARAM_GETTER(threads, ZSTD_c_nbWorkers)
 
 static PyMethodDef ZstdCompressionParameters_methods[] = {
 	{
 		"from_level",
 		(PyCFunction)CompressionParameters_from_level,
 		METH_VARARGS | METH_KEYWORDS | METH_STATIC,
 		ZstdCompressionParameters_from_level__doc__
 	},
 	{
 		"estimated_compression_context_size",
 		(PyCFunction)ZstdCompressionParameters_estimated_compression_context_size,
 		METH_NOARGS,
 		ZstdCompressionParameters_estimated_compression_context_size__doc__
 	},
 	{ NULL, NULL }
 };
 
 #define GET_SET_ENTRY(name) { #name, ZstdCompressionParameters_get_##name, NULL, NULL, NULL }
 
 static PyGetSetDef ZstdCompressionParameters_getset[] = {
 	GET_SET_ENTRY(format),
 	GET_SET_ENTRY(compression_level),
 	GET_SET_ENTRY(window_log),
 	GET_SET_ENTRY(hash_log),
 	GET_SET_ENTRY(chain_log),
 	GET_SET_ENTRY(search_log),
 	GET_SET_ENTRY(min_match),
 	GET_SET_ENTRY(target_length),
 	GET_SET_ENTRY(compression_strategy),
 	GET_SET_ENTRY(write_content_size),
 	GET_SET_ENTRY(write_checksum),
 	GET_SET_ENTRY(write_dict_id),
 	GET_SET_ENTRY(threads),
 	GET_SET_ENTRY(job_size),
 	GET_SET_ENTRY(overlap_log),
 	/* TODO remove this deprecated attribute */
 	{ "overlap_size_log", ZstdCompressionParameters_get_overlap_log, NULL, NULL, NULL },
 	GET_SET_ENTRY(force_max_window),
 	GET_SET_ENTRY(enable_ldm),
 	GET_SET_ENTRY(ldm_hash_log),
 	GET_SET_ENTRY(ldm_min_match),
 	GET_SET_ENTRY(ldm_bucket_size_log),
 	GET_SET_ENTRY(ldm_hash_rate_log),
 	/* TODO remove this deprecated attribute */
 	{ "ldm_hash_every_log", ZstdCompressionParameters_get_ldm_hash_rate_log, NULL, NULL, NULL },
 	{ NULL }
 };
 
 PyTypeObject ZstdCompressionParametersType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"ZstdCompressionParameters", /* tp_name */
 	sizeof(ZstdCompressionParametersObject), /* tp_basicsize */
 	0,                         /* tp_itemsize */
 	(destructor)ZstdCompressionParameters_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
 	0,                         /* tp_compare */
 	0,                         /* tp_repr */
 	0,                         /* tp_as_number */
 	0,                         /* tp_as_sequence */
 	0,                         /* tp_as_mapping */
 	0,                         /* tp_hash  */
 	0,                         /* tp_call */
 	0,                         /* tp_str */
 	0,                         /* tp_getattro */
 	0,                         /* tp_setattro */
 	0,                         /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompressionParameters__doc__, /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	ZstdCompressionParameters_methods, /* tp_methods */
 	0,                          /* tp_members */
 	ZstdCompressionParameters_getset,  /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	(initproc)ZstdCompressionParameters_init, /* tp_init */
 	0,                         /* tp_alloc */
 	PyType_GenericNew,         /* tp_new */
 };
 
 void compressionparams_module_init(PyObject* mod) {
 	Py_TYPE(&ZstdCompressionParametersType) = &PyType_Type;
 	if (PyType_Ready(&ZstdCompressionParametersType) < 0) {
 		return;
 	}
 
 	Py_INCREF(&ZstdCompressionParametersType);
 	PyModule_AddObject(mod, "ZstdCompressionParameters",
 		(PyObject*)&ZstdCompressionParametersType);
 
 	/* TODO remove deprecated alias. */
 	Py_INCREF(&ZstdCompressionParametersType);
 	PyModule_AddObject(mod, "CompressionParameters",
 		(PyObject*)&ZstdCompressionParametersType);
 }

contrib/python-zstandard/c-ext/decompressionreader.c

@@ -1,779 +1,781 b''
 /**
 * Copyright (c) 2017-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 static void set_unsupported_operation(void) {
 	PyObject* iomod;
 	PyObject* exc;
 
 	iomod = PyImport_ImportModule("io");
 	if (NULL == iomod) {
 		return;
 	}
 
 	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
 	if (NULL == exc) {
 		Py_DECREF(iomod);
 		return;
 	}
 
 	PyErr_SetNone(exc);
 	Py_DECREF(exc);
 	Py_DECREF(iomod);
 }
 
 static void reader_dealloc(ZstdDecompressionReader* self) {
 	Py_XDECREF(self->decompressor);
 	Py_XDECREF(self->reader);
 
 	if (self->buffer.buf) {
 		PyBuffer_Release(&self->buffer);
 	}
 
 	PyObject_Del(self);
 }
 
 static ZstdDecompressionReader* reader_enter(ZstdDecompressionReader* self) {
 	if (self->entered) {
 		PyErr_SetString(PyExc_ValueError, "cannot __enter__ multiple times");
 		return NULL;
 	}
 
 	self->entered = 1;
 
 	Py_INCREF(self);
 	return self;
 }
 
 static PyObject* reader_exit(ZstdDecompressionReader* self, PyObject* args) {
 	PyObject* exc_type;
 	PyObject* exc_value;
 	PyObject* exc_tb;
 
 	if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
 		return NULL;
 	}
 
 	self->entered = 0;
 	self->closed = 1;
 
 	/* Release resources. */
 	Py_CLEAR(self->reader);
 	if (self->buffer.buf) {
 		PyBuffer_Release(&self->buffer);
 		memset(&self->buffer, 0, sizeof(self->buffer));
 	}
 
 	Py_CLEAR(self->decompressor);
 
 	Py_RETURN_FALSE;
 }
 
 static PyObject* reader_readable(PyObject* self) {
 	Py_RETURN_TRUE;
 }
 
 static PyObject* reader_writable(PyObject* self) {
 	Py_RETURN_FALSE;
 }
 
 static PyObject* reader_seekable(PyObject* self) {
 	Py_RETURN_TRUE;
 }
 
 static PyObject* reader_close(ZstdDecompressionReader* self) {
 	self->closed = 1;
 	Py_RETURN_NONE;
 }
 
 static PyObject* reader_flush(PyObject* self) {
 	Py_RETURN_NONE;
 }
 
 static PyObject* reader_isatty(PyObject* self) {
 	Py_RETURN_FALSE;
 }
 
 /**
  * Read available input.
  *
  * Returns 0 if no data was added to input.
  * Returns 1 if new input data is available.
  * Returns -1 on error and sets a Python exception as a side-effect.
  */
 int read_decompressor_input(ZstdDecompressionReader* self) {
 	if (self->finishedInput) {
 		return 0;
 	}
 
 	if (self->input.pos != self->input.size) {
 		return 0;
 	}
 
 	if (self->reader) {
         Py_buffer buffer;
 
         assert(self->readResult == NULL);
         self->readResult = PyObject_CallMethod(self->reader, "read",
             "k", self->readSize);
         if (NULL == self->readResult) {
             return -1;
         }
 
         memset(&buffer, 0, sizeof(buffer));
 
         if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
             return -1;
         }
 
         /* EOF */
         if (0 == buffer.len) {
             self->finishedInput = 1;
             Py_CLEAR(self->readResult);
         }
         else {
             self->input.src = buffer.buf;
             self->input.size = buffer.len;
             self->input.pos = 0;
         }
 
         PyBuffer_Release(&buffer);
 	}
 	else {
 		assert(self->buffer.buf);
         /*
          * We should only get here once since expectation is we always
          * exhaust input buffer before reading again.
          */
         assert(self->input.src == NULL);
 
 		self->input.src = self->buffer.buf;
         self->input.size = self->buffer.len;
         self->input.pos = 0;
 	}
 
 	return 1;
 }
 
 /**
  * Decompresses available input into an output buffer.
  *
  * Returns 0 if we need more input.
  * Returns 1 if output buffer should be emitted.
  * Returns -1 on error and sets a Python exception.
  */
 int decompress_input(ZstdDecompressionReader* self, ZSTD_outBuffer* output) {
 	size_t zresult;
 
 	if (self->input.pos >= self->input.size) {
 		return 0;
 	}
 
 	Py_BEGIN_ALLOW_THREADS
 	zresult = ZSTD_decompressStream(self->decompressor->dctx, output, &self->input);
 	Py_END_ALLOW_THREADS
 
 	/* Input exhausted. Clear our state tracking. */
 	if (self->input.pos == self->input.size) {
 		memset(&self->input, 0, sizeof(self->input));
 		Py_CLEAR(self->readResult);
 
 		if (self->buffer.buf) {
 			self->finishedInput = 1;
 		}
 	}
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "zstd decompress error: %s", ZSTD_getErrorName(zresult));
 		return -1;
 	}
 
 	/* We fulfilled the full read request. Signal to emit. */
 	if (output->pos && output->pos == output->size) {
 		return 1;
 	}
 	/* We're at the end of a frame and we aren't allowed to return data
 	   spanning frames. */
 	else if (output->pos && zresult == 0 && !self->readAcrossFrames) {
 		return 1;
 	}
 
 	/* There is more room in the output. Signal to collect more data. */
 	return 0;
 }
 
 static PyObject* reader_read(ZstdDecompressionReader* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
 		NULL
 	};
 
 	Py_ssize_t size = -1;
 	PyObject* result = NULL;
 	char* resultBuffer;
 	Py_ssize_t resultSize;
 	ZSTD_outBuffer output;
 	int decompressResult, readResult;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &size)) {
 		return NULL;
 	}
 
 	if (size < -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
 		return NULL;
 	}
 
 	if (size == -1) {
 		return PyObject_CallMethod((PyObject*)self, "readall", NULL);
 	}
 
 	if (self->finishedOutput || size == 0) {
 		return PyBytes_FromStringAndSize("", 0);
 	}
 
 	result = PyBytes_FromStringAndSize(NULL, size);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
 
 	output.dst = resultBuffer;
 	output.size = resultSize;
 	output.pos = 0;
 
 readinput:
 
 	decompressResult = decompress_input(self, &output);
 
 	if (-1 == decompressResult) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 	else if (0 == decompressResult) { }
 	else if (1 == decompressResult) {
 		self->bytesDecompressed += output.pos;
 
 		if (output.pos != output.size) {
 			if (safe_pybytes_resize(&result, output.pos)) {
 				Py_XDECREF(result);
 				return NULL;
 			}
 		}
 		return result;
 	}
 	else {
 		assert(0);
 	}
 
 	readResult = read_decompressor_input(self);
 
 	if (-1 == readResult) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 	else if (0 == readResult) {}
 	else if (1 == readResult) {}
 	else {
 		assert(0);
 	}
 
 	if (self->input.size) {
 		goto readinput;
 	}
 
 	/* EOF */
 	self->bytesDecompressed += output.pos;
 
 	if (safe_pybytes_resize(&result, output.pos)) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 
 	return result;
 }
 
 static PyObject* reader_read1(ZstdDecompressionReader* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
 		NULL
 	};
 
 	Py_ssize_t size = -1;
 	PyObject* result = NULL;
 	char* resultBuffer;
 	Py_ssize_t resultSize;
 	ZSTD_outBuffer output;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &size)) {
 		return NULL;
 	}
 
 	if (size < -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
 		return NULL;
 	}
 
 	if (self->finishedOutput || size == 0) {
 		return PyBytes_FromStringAndSize("", 0);
 	}
 
 	if (size == -1) {
 		size = ZSTD_DStreamOutSize();
 	}
 
 	result = PyBytes_FromStringAndSize(NULL, size);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
 
 	output.dst = resultBuffer;
 	output.size = resultSize;
 	output.pos = 0;
 
 	/* read1() is supposed to use at most 1 read() from the underlying stream.
 	 * However, we can't satisfy this requirement with decompression due to the
 	 * nature of how decompression works. Our strategy is to read + decompress
 	 * until we get any output, at which point we return. This satisfies the
 	 * intent of the read1() API to limit read operations.
 	 */
 	while (!self->finishedInput) {
 		int readResult, decompressResult;
 
 		readResult = read_decompressor_input(self);
 		if (-1 == readResult) {
 			Py_XDECREF(result);
 			return NULL;
 		}
 		else if (0 == readResult || 1 == readResult) { }
 		else {
 			assert(0);
 		}
 
 		decompressResult = decompress_input(self, &output);
 
 		if (-1 == decompressResult) {
 			Py_XDECREF(result);
 			return NULL;
 		}
 		else if (0 == decompressResult || 1 == decompressResult) { }
 		else {
 			assert(0);
 		}
 
 		if (output.pos) {
 		    break;
 		}
 	}
 
 	self->bytesDecompressed += output.pos;
 	if (safe_pybytes_resize(&result, output.pos)) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 
 	return result;
 }
 
 static PyObject* reader_readinto(ZstdDecompressionReader* self, PyObject* args) {
 	Py_buffer dest;
 	ZSTD_outBuffer output;
 	int decompressResult, readResult;
 	PyObject* result = NULL;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (self->finishedOutput) {
 		return PyLong_FromLong(0);
 	}
 
 	if (!PyArg_ParseTuple(args, "w*:readinto", &dest)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"destination buffer should be contiguous and have at most one dimension");
 	    goto finally;
 	}
 
 	output.dst = dest.buf;
 	output.size = dest.len;
 	output.pos = 0;
 
 readinput:
 
 	decompressResult = decompress_input(self, &output);
 
 	if (-1 == decompressResult) {
 		goto finally;
 	}
 	else if (0 == decompressResult) { }
 	else if (1 == decompressResult) {
 		self->bytesDecompressed += output.pos;
 		result = PyLong_FromSize_t(output.pos);
 		goto finally;
 	}
 	else {
 		assert(0);
 	}
 
 	readResult = read_decompressor_input(self);
 
 	if (-1 == readResult) {
 		goto finally;
 	}
 	else if (0 == readResult) {}
 	else if (1 == readResult) {}
 	else {
 		assert(0);
 	}
 
 	if (self->input.size) {
 		goto readinput;
 	}
 
 	/* EOF */
 	self->bytesDecompressed += output.pos;
 	result = PyLong_FromSize_t(output.pos);
 
 finally:
 	PyBuffer_Release(&dest);
 
 	return result;
 }
 
 static PyObject* reader_readinto1(ZstdDecompressionReader* self, PyObject* args) {
 	Py_buffer dest;
 	ZSTD_outBuffer output;
 	PyObject* result = NULL;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (self->finishedOutput) {
 		return PyLong_FromLong(0);
 	}
 
 	if (!PyArg_ParseTuple(args, "w*:readinto1", &dest)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"destination buffer should be contiguous and have at most one dimension");
 	    goto finally;
 	}
 
 	output.dst = dest.buf;
 	output.size = dest.len;
 	output.pos = 0;
 
 	while (!self->finishedInput && !self->finishedOutput) {
 		int decompressResult, readResult;
 
 		readResult = read_decompressor_input(self);
 
 		if (-1 == readResult) {
 			goto finally;
 		}
 		else if (0 == readResult || 1 == readResult) {}
 		else {
 			assert(0);
 		}
 
 		decompressResult = decompress_input(self, &output);
 
 		if (-1 == decompressResult) {
 			goto finally;
 		}
 		else if (0 == decompressResult || 1 == decompressResult) {}
 		else {
 			assert(0);
 		}
 
 		if (output.pos) {
 			break;
 		}
 	}
 
 	self->bytesDecompressed += output.pos;
 	result = PyLong_FromSize_t(output.pos);
 
 finally:
 	PyBuffer_Release(&dest);
 
 	return result;
 }
 
 static PyObject* reader_readall(PyObject* self) {
 	PyObject* chunks = NULL;
 	PyObject* empty = NULL;
 	PyObject* result = NULL;
 
 	/* Our strategy is to collect chunks into a list then join all the
 	 * chunks at the end. We could potentially use e.g. an io.BytesIO. But
 	 * this feels simple enough to implement and avoids potentially expensive
 	 * reallocations of large buffers.
 	 */
 	chunks = PyList_New(0);
 	if (NULL == chunks) {
 		return NULL;
 	}
 
 	while (1) {
 		PyObject* chunk = PyObject_CallMethod(self, "read", "i", 1048576);
 		if (NULL == chunk) {
 			Py_DECREF(chunks);
 			return NULL;
 		}
 
 		if (!PyBytes_Size(chunk)) {
 			Py_DECREF(chunk);
 			break;
 		}
 
 		if (PyList_Append(chunks, chunk)) {
 			Py_DECREF(chunk);
 			Py_DECREF(chunks);
 			return NULL;
 		}
 
 		Py_DECREF(chunk);
 	}
 
 	empty = PyBytes_FromStringAndSize("", 0);
 	if (NULL == empty) {
 		Py_DECREF(chunks);
 		return NULL;
 	}
 
 	result = PyObject_CallMethod(empty, "join", "O", chunks);
 
 	Py_DECREF(empty);
 	Py_DECREF(chunks);
 
 	return result;
 }
 
 static PyObject* reader_readline(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_readlines(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_seek(ZstdDecompressionReader* self, PyObject* args) {
 	Py_ssize_t pos;
 	int whence = 0;
 	unsigned long long readAmount = 0;
 	size_t defaultOutSize = ZSTD_DStreamOutSize();
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &whence)) {
 		return NULL;
 	}
 
 	if (whence == SEEK_SET) {
 		if (pos < 0) {
 			PyErr_SetString(PyExc_ValueError,
 				"cannot seek to negative position with SEEK_SET");
 			return NULL;
 		}
 
 		if ((unsigned long long)pos < self->bytesDecompressed) {
 			PyErr_SetString(PyExc_ValueError,
 				"cannot seek zstd decompression stream backwards");
 			return NULL;
 		}
 
 		readAmount = pos - self->bytesDecompressed;
 	}
 	else if (whence == SEEK_CUR) {
 		if (pos < 0) {
 			PyErr_SetString(PyExc_ValueError,
 				"cannot seek zstd decompression stream backwards");
 			return NULL;
 		}
 
 		readAmount = pos;
 	}
 	else if (whence == SEEK_END) {
 		/* We /could/ support this with pos==0. But let's not do that until someone
 		   needs it. */
 		PyErr_SetString(PyExc_ValueError,
 			"zstd decompression streams cannot be seeked with SEEK_END");
 		return NULL;
 	}
 
 	/* It is a bit inefficient to do this via the Python API. But since there
 	   is a bit of state tracking involved to read from this type, it is the
 	   easiest to implement. */
 	while (readAmount) {
 		Py_ssize_t readSize;
 		PyObject* readResult = PyObject_CallMethod((PyObject*)self, "read", "K",
 			readAmount < defaultOutSize ? readAmount : defaultOutSize);
 
 		if (!readResult) {
 			return NULL;
 		}
 
 		readSize = PyBytes_GET_SIZE(readResult);
 
+		Py_CLEAR(readResult);
+
 		/* Empty read means EOF. */
 		if (!readSize) {
 			break;
 		}
 
 		readAmount -= readSize;
 	}
 
 	return PyLong_FromUnsignedLongLong(self->bytesDecompressed);
 }
 
 static PyObject* reader_tell(ZstdDecompressionReader* self) {
 	/* TODO should this raise OSError since stream isn't seekable? */
 	return PyLong_FromUnsignedLongLong(self->bytesDecompressed);
 }
 
 static PyObject* reader_write(PyObject* self, PyObject* args) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_writelines(PyObject* self, PyObject* args) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_iter(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_iternext(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyMethodDef reader_methods[] = {
 	{ "__enter__", (PyCFunction)reader_enter, METH_NOARGS,
 	PyDoc_STR("Enter a compression context") },
 	{ "__exit__", (PyCFunction)reader_exit, METH_VARARGS,
 	PyDoc_STR("Exit a compression context") },
 	{ "close", (PyCFunction)reader_close, METH_NOARGS,
 	PyDoc_STR("Close the stream so it cannot perform any more operations") },
 	{ "flush", (PyCFunction)reader_flush, METH_NOARGS, PyDoc_STR("no-ops") },
 	{ "isatty", (PyCFunction)reader_isatty, METH_NOARGS, PyDoc_STR("Returns False") },
 	{ "readable", (PyCFunction)reader_readable, METH_NOARGS,
 	PyDoc_STR("Returns True") },
 	{ "read", (PyCFunction)reader_read, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("read compressed data") },
 	{ "read1", (PyCFunction)reader_read1, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("read compressed data") },
 	{ "readinto", (PyCFunction)reader_readinto, METH_VARARGS, NULL },
 	{ "readinto1", (PyCFunction)reader_readinto1, METH_VARARGS, NULL },
 	{ "readall", (PyCFunction)reader_readall, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "readline", (PyCFunction)reader_readline, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "readlines", (PyCFunction)reader_readlines, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "seek", (PyCFunction)reader_seek, METH_VARARGS, PyDoc_STR("Seek the stream") },
 	{ "seekable", (PyCFunction)reader_seekable, METH_NOARGS,
 	PyDoc_STR("Returns True") },
 	{ "tell", (PyCFunction)reader_tell, METH_NOARGS,
 	PyDoc_STR("Returns current number of bytes compressed") },
 	{ "writable", (PyCFunction)reader_writable, METH_NOARGS,
 	PyDoc_STR("Returns False") },
 	{ "write", (PyCFunction)reader_write, METH_VARARGS, PyDoc_STR("unsupported operation") },
 	{ "writelines", (PyCFunction)reader_writelines, METH_VARARGS, PyDoc_STR("unsupported operation") },
 	{ NULL, NULL }
 };
 
 static PyMemberDef reader_members[] = {
 	{ "closed", T_BOOL, offsetof(ZstdDecompressionReader, closed),
 	  READONLY, "whether stream is closed" },
 	{ NULL }
 };
 
 PyTypeObject ZstdDecompressionReaderType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdDecompressionReader", /* tp_name */
 	sizeof(ZstdDecompressionReader), /* tp_basicsize */
 	0, /* tp_itemsize */
 	(destructor)reader_dealloc, /* tp_dealloc */
 	0, /* tp_print */
 	0, /* tp_getattr */
 	0, /* tp_setattr */
 	0, /* tp_compare */
 	0, /* tp_repr */
 	0, /* tp_as_number */
 	0, /* tp_as_sequence */
 	0, /* tp_as_mapping */
 	0, /* tp_hash */
 	0, /* tp_call */
 	0, /* tp_str */
 	0, /* tp_getattro */
 	0, /* tp_setattro */
 	0, /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT, /* tp_flags */
 	0, /* tp_doc */
 	0, /* tp_traverse */
 	0, /* tp_clear */
 	0, /* tp_richcompare */
 	0, /* tp_weaklistoffset */
 	reader_iter, /* tp_iter */
 	reader_iternext, /* tp_iternext */
 	reader_methods, /* tp_methods */
 	reader_members, /* tp_members */
 	0, /* tp_getset */
 	0, /* tp_base */
 	0, /* tp_dict */
 	0, /* tp_descr_get */
 	0, /* tp_descr_set */
 	0, /* tp_dictoffset */
 	0, /* tp_init */
 	0, /* tp_alloc */
 	PyType_GenericNew, /* tp_new */
 };
 
 
 void decompressionreader_module_init(PyObject* mod) {
 	/* TODO make reader a sub-class of io.RawIOBase */
 
 	Py_TYPE(&ZstdDecompressionReaderType) = &PyType_Type;
 	if (PyType_Ready(&ZstdDecompressionReaderType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/python-zstandard.h

@@ -1,359 +1,359 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include "structmember.h"
 
 #define ZSTD_STATIC_LINKING_ONLY
 #define ZDICT_STATIC_LINKING_ONLY
 #include <zstd.h>
 #include <zdict.h>
 
 /* Remember to change the string in zstandard/__init__ as well */
-#define PYTHON_ZSTANDARD_VERSION "0.11.0"
+#define PYTHON_ZSTANDARD_VERSION "0.12.0"
 
 typedef enum {
 	compressorobj_flush_finish,
 	compressorobj_flush_block,
 } CompressorObj_Flush;
 
 /*
    Represents a ZstdCompressionParameters type.
 
    This type holds all the low-level compression parameters that can be set.
 */
 typedef struct {
 	PyObject_HEAD
 	ZSTD_CCtx_params* params;
 } ZstdCompressionParametersObject;
 
 extern PyTypeObject ZstdCompressionParametersType;
 
 /*
    Represents a FrameParameters type.
 
    This type is basically a wrapper around ZSTD_frameParams.
 */
 typedef struct {
 	PyObject_HEAD
 	unsigned long long frameContentSize;
 	unsigned long long windowSize;
 	unsigned dictID;
 	char checksumFlag;
 } FrameParametersObject;
 
 extern PyTypeObject FrameParametersType;
 
 /*
    Represents a ZstdCompressionDict type.
 
    Instances hold data used for a zstd compression dictionary.
 */
 typedef struct {
 	PyObject_HEAD
 
 	/* Pointer to dictionary data. Owned by self. */
 	void* dictData;
 	/* Size of dictionary data. */
 	size_t dictSize;
 	ZSTD_dictContentType_e dictType;
 	/* k parameter for cover dictionaries. Only populated by train_cover_dict(). */
 	unsigned k;
 	/* d parameter for cover dictionaries. Only populated by train_cover_dict(). */
 	unsigned d;
 	/* Digested dictionary, suitable for reuse. */
 	ZSTD_CDict* cdict;
 	ZSTD_DDict* ddict;
 } ZstdCompressionDict;
 
 extern PyTypeObject ZstdCompressionDictType;
 
 /*
    Represents a ZstdCompressor type.
 */
 typedef struct {
 	PyObject_HEAD
 
 	/* Number of threads to use for operations. */
 	unsigned int threads;
 	/* Pointer to compression dictionary to use. NULL if not using dictionary
 	   compression. */
 	ZstdCompressionDict* dict;
 	/* Compression context to use. Populated during object construction. */
 	ZSTD_CCtx* cctx;
 	/* Compression parameters in use. */
 	ZSTD_CCtx_params* params;
 } ZstdCompressor;
 
 extern PyTypeObject ZstdCompressorType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	ZSTD_outBuffer output;
 	int finished;
 } ZstdCompressionObj;
 
 extern PyTypeObject ZstdCompressionObjType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	PyObject* writer;
 	ZSTD_outBuffer output;
 	size_t outSize;
 	int entered;
 	int closed;
 	int writeReturnRead;
 	unsigned long long bytesCompressed;
 } ZstdCompressionWriter;
 
 extern PyTypeObject ZstdCompressionWriterType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	PyObject* reader;
 	Py_buffer buffer;
 	Py_ssize_t bufferOffset;
 	size_t inSize;
 	size_t outSize;
 
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	int finishedOutput;
 	int finishedInput;
 	PyObject* readResult;
 } ZstdCompressorIterator;
 
 extern PyTypeObject ZstdCompressorIteratorType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	PyObject* reader;
 	Py_buffer buffer;
 	size_t readSize;
 
 	int entered;
 	int closed;
 	unsigned long long bytesCompressed;
 
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	int finishedInput;
 	int finishedOutput;
 	PyObject* readResult;
 } ZstdCompressionReader;
 
 extern PyTypeObject ZstdCompressionReaderType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	Py_buffer inBuffer;
 	int finished;
 	size_t chunkSize;
 } ZstdCompressionChunker;
 
 extern PyTypeObject ZstdCompressionChunkerType;
 
 typedef enum {
 	compressionchunker_mode_normal,
 	compressionchunker_mode_flush,
 	compressionchunker_mode_finish,
 } CompressionChunkerMode;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressionChunker* chunker;
 	CompressionChunkerMode mode;
 } ZstdCompressionChunkerIterator;
 
 extern PyTypeObject ZstdCompressionChunkerIteratorType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZSTD_DCtx* dctx;
 	ZstdCompressionDict* dict;
 	size_t maxWindowSize;
 	ZSTD_format_e format;
 } ZstdDecompressor;
 
 extern PyTypeObject ZstdDecompressorType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdDecompressor* decompressor;
 	size_t outSize;
 	int finished;
 } ZstdDecompressionObj;
 
 extern PyTypeObject ZstdDecompressionObjType;
 
 typedef struct {
 	PyObject_HEAD
 
 	/* Parent decompressor to which this object is associated. */
 	ZstdDecompressor* decompressor;
 	/* Object to read() from (if reading from a stream). */
 	PyObject* reader;
 	/* Size for read() operations on reader. */
 	size_t readSize;
 	/* Whether a read() can return data spanning multiple zstd frames. */
 	int readAcrossFrames;
 	/* Buffer to read from (if reading from a buffer). */
 	Py_buffer buffer;
 
 	/* Whether the context manager is active. */
 	int entered;
 	/* Whether we've closed the stream. */
 	int closed;
 
 	/* Number of bytes decompressed and returned to user. */
 	unsigned long long bytesDecompressed;
 
 	/* Tracks data going into decompressor. */
 	ZSTD_inBuffer input;
 
 	/* Holds output from read() operation on reader. */
 	PyObject* readResult;
 
 	/* Whether all input has been sent to the decompressor. */
 	int finishedInput;
 	/* Whether all output has been flushed from the decompressor. */
 	int finishedOutput;
 } ZstdDecompressionReader;
 
 extern PyTypeObject ZstdDecompressionReaderType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdDecompressor* decompressor;
 	PyObject* writer;
 	size_t outSize;
 	int entered;
 	int closed;
 	int writeReturnRead;
 } ZstdDecompressionWriter;
 
 extern PyTypeObject ZstdDecompressionWriterType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdDecompressor* decompressor;
 	PyObject* reader;
 	Py_buffer buffer;
 	Py_ssize_t bufferOffset;
 	size_t inSize;
 	size_t outSize;
 	size_t skipBytes;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	Py_ssize_t readCount;
 	int finishedInput;
 	int finishedOutput;
 } ZstdDecompressorIterator;
 
 extern PyTypeObject ZstdDecompressorIteratorType;
 
 typedef struct {
 	int errored;
 	PyObject* chunk;
 } DecompressorIteratorResult;
 
 typedef struct {
 	/* The public API is that these are 64-bit unsigned integers. So these can't
 	 * be size_t, even though values larger than SIZE_MAX or PY_SSIZE_T_MAX may 
 	 * be nonsensical for this platform. */
 	unsigned long long offset;
 	unsigned long long length;
 } BufferSegment;
 
 typedef struct {
 	PyObject_HEAD
 
 	PyObject* parent;
 	BufferSegment* segments;
 	Py_ssize_t segmentCount;
 } ZstdBufferSegments;
 
 extern PyTypeObject ZstdBufferSegmentsType;
 
 typedef struct {
 	PyObject_HEAD
 
 	PyObject* parent;
 	void* data;
 	Py_ssize_t dataSize;
 	unsigned long long offset;
 } ZstdBufferSegment;
 
 extern PyTypeObject ZstdBufferSegmentType;
 
 typedef struct {
 	PyObject_HEAD
 
 	Py_buffer parent;
 	void* data;
 	unsigned long long dataSize;
 	BufferSegment* segments;
 	Py_ssize_t segmentCount;
 	int useFree;
 } ZstdBufferWithSegments;
 
 extern PyTypeObject ZstdBufferWithSegmentsType;
 
 /**
  * An ordered collection of BufferWithSegments exposed as a squashed collection.
  *
  * This type provides a virtual view spanning multiple BufferWithSegments
  * instances. It allows multiple instances to be "chained" together and
  * exposed as a single collection. e.g. if there are 2 buffers holding
  * 10 segments each, then o[14] will access the 5th segment in the 2nd buffer.
  */
 typedef struct {
 	PyObject_HEAD
 
 	/* An array of buffers that should be exposed through this instance. */
 	ZstdBufferWithSegments** buffers;
 	/* Number of elements in buffers array. */
 	Py_ssize_t bufferCount;
 	/* Array of first offset in each buffer instance. 0th entry corresponds
 	   to number of elements in the 0th buffer. 1st entry corresponds to the
 	   sum of elements in 0th and 1st buffers. */
 	Py_ssize_t* firstElements;
 } ZstdBufferWithSegmentsCollection;
 
 extern PyTypeObject ZstdBufferWithSegmentsCollectionType;
 
 int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj);
 int to_cparams(ZstdCompressionParametersObject* params, ZSTD_compressionParameters* cparams);
 FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args, PyObject* kwargs);
 int ensure_ddict(ZstdCompressionDict* dict);
 int ensure_dctx(ZstdDecompressor* decompressor, int loadDict);
 ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs);
 ZstdBufferWithSegments* BufferWithSegments_FromMemory(void* data, unsigned long long dataSize, BufferSegment* segments, Py_ssize_t segmentsSize);
 Py_ssize_t BufferWithSegmentsCollection_length(ZstdBufferWithSegmentsCollection*);
 int cpu_count(void);
 size_t roundpow2(size_t);
 int safe_pybytes_resize(PyObject** obj, Py_ssize_t size);

contrib/python-zstandard/make_cffi.py

@@ -1,201 +1,207 b''
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 from __future__ import absolute_import
 
 import cffi
 import distutils.ccompiler
 import os
 import re
 import subprocess
 import tempfile
 
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
 SOURCES = ['zstd/%s' % p for p in (
     'common/debug.c',
     'common/entropy_common.c',
     'common/error_private.c',
     'common/fse_decompress.c',
     'common/pool.c',
     'common/threading.c',
     'common/xxhash.c',
     'common/zstd_common.c',
     'compress/fse_compress.c',
     'compress/hist.c',
     'compress/huf_compress.c',
     'compress/zstd_compress.c',
+    'compress/zstd_compress_literals.c',
+    'compress/zstd_compress_sequences.c',
     'compress/zstd_double_fast.c',
     'compress/zstd_fast.c',
     'compress/zstd_lazy.c',
     'compress/zstd_ldm.c',
     'compress/zstd_opt.c',
     'compress/zstdmt_compress.c',
     'decompress/huf_decompress.c',
     'decompress/zstd_ddict.c',
     'decompress/zstd_decompress.c',
     'decompress/zstd_decompress_block.c',
     'dictBuilder/cover.c',
     'dictBuilder/fastcover.c',
     'dictBuilder/divsufsort.c',
     'dictBuilder/zdict.c',
 )]
 
 # Headers whose preprocessed output will be fed into cdef().
 HEADERS = [os.path.join(HERE, 'zstd', *p) for p in (
     ('zstd.h',),
     ('dictBuilder', 'zdict.h'),
 )]
 
 INCLUDE_DIRS = [os.path.join(HERE, d) for d in (
     'zstd',
     'zstd/common',
     'zstd/compress',
     'zstd/decompress',
     'zstd/dictBuilder',
 )]
 
 # cffi can't parse some of the primitives in zstd.h. So we invoke the
 # preprocessor and feed its output into cffi.
 compiler = distutils.ccompiler.new_compiler()
 
 # Needed for MSVC.
 if hasattr(compiler, 'initialize'):
     compiler.initialize()
 
 # Distutils doesn't set compiler.preprocessor, so invoke the preprocessor
 # manually.
 if compiler.compiler_type == 'unix':
     args = list(compiler.executables['compiler'])
     args.extend([
         '-E',
         '-DZSTD_STATIC_LINKING_ONLY',
         '-DZDICT_STATIC_LINKING_ONLY',
     ])
 elif compiler.compiler_type == 'msvc':
     args = [compiler.cc]
     args.extend([
         '/EP',
         '/DZSTD_STATIC_LINKING_ONLY',
         '/DZDICT_STATIC_LINKING_ONLY',
     ])
 else:
     raise Exception('unsupported compiler type: %s' % compiler.compiler_type)
 
 def preprocess(path):
     with open(path, 'rb') as fh:
         lines = []
         it = iter(fh)
 
         for l in it:
             # zstd.h includes <stddef.h>, which is also included by cffi's
             # boilerplate. This can lead to duplicate declarations. So we strip
             # this include from the preprocessor invocation.
             #
             # The same things happens for including zstd.h, so give it the same
             # treatment.
             #
             # We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline
             # #define in zstdmt_compress.h and results in a compiler warning. So drop
             # the inline #define.
             if l.startswith((b'#include <stddef.h>',
                              b'#include "zstd.h"',
                              b'#define ZSTD_STATIC_LINKING_ONLY')):
                 continue
 
             # ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't
             # important so just filter it out.
             if l.startswith(b'ZSTDLIB_API'):
                 l = l[len(b'ZSTDLIB_API '):]
 
             lines.append(l)
 
     fd, input_file = tempfile.mkstemp(suffix='.h')
     os.write(fd, b''.join(lines))
     os.close(fd)
 
     try:
-        process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE)
+        env = dict(os.environ)
+        if getattr(compiler, '_paths', None):
+            env['PATH'] = compiler._paths
+        process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE,
+                                   env=env)
         output = process.communicate()[0]
         ret = process.poll()
         if ret:
             raise Exception('preprocessor exited with error')
 
         return output
     finally:
         os.unlink(input_file)
 
 
 def normalize_output(output):
     lines = []
     for line in output.splitlines():
         # CFFI's parser doesn't like __attribute__ on UNIX compilers.
         if line.startswith(b'__attribute__ ((visibility ("default"))) '):
             line = line[len(b'__attribute__ ((visibility ("default"))) '):]
 
         if line.startswith(b'__attribute__((deprecated('):
             continue
         elif b'__declspec(deprecated(' in line:
             continue
 
         lines.append(line)
 
     return b'\n'.join(lines)
 
 
 ffi = cffi.FFI()
 # zstd.h uses a possible undefined MIN(). Define it until
 # https://github.com/facebook/zstd/issues/976 is fixed.
 # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
 # when cffi uses the function. Since we statically link against zstd, even
 # if we use the deprecated functions it shouldn't be a huge problem.
 ffi.set_source('_zstd_cffi', '''
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define ZSTD_STATIC_LINKING_ONLY
 #include <zstd.h>
 #define ZDICT_STATIC_LINKING_ONLY
 #define ZDICT_DISABLE_DEPRECATE_WARNINGS
 #include <zdict.h>
 ''', sources=SOURCES,
      include_dirs=INCLUDE_DIRS,
      extra_compile_args=['-DZSTD_MULTITHREAD'])
 
 DEFINE = re.compile(b'^\\#define ([a-zA-Z0-9_]+) ')
 
 sources = []
 
 # Feed normalized preprocessor output for headers into the cdef parser.
 for header in HEADERS:
     preprocessed = preprocess(header)
     sources.append(normalize_output(preprocessed))
 
     # #define's are effectively erased as part of going through preprocessor.
     # So perform a manual pass to re-add those to the cdef source.
     with open(header, 'rb') as fh:
         for line in fh:
             line = line.strip()
             m = DEFINE.match(line)
             if not m:
                 continue
 
             if m.group(1) == b'ZSTD_STATIC_LINKING_ONLY':
                 continue
 
             # The parser doesn't like some constants with complex values.
             if m.group(1) in (b'ZSTD_LIB_VERSION', b'ZSTD_VERSION_STRING'):
                 continue
 
             # The ... is magic syntax by the cdef parser to resolve the
             # value at compile time.
             sources.append(m.group(0) + b' ...')
 
 cdeflines = b'\n'.join(sources).splitlines()
 cdeflines = [l for l in cdeflines if l.strip()]
 ffi.cdef(b'\n'.join(cdeflines).decode('latin1'))
 
 if __name__ == '__main__':
     ffi.compile()

contrib/python-zstandard/setup.py

@@ -1,113 +1,112 b''
 #!/usr/bin/env python
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 from __future__ import print_function
 
 from distutils.version import LooseVersion
 import os
 import sys
 from setuptools import setup
 
 # Need change in 1.10 for ffi.from_buffer() to handle all buffer types
 # (like memoryview).
 # Need feature in 1.11 for ffi.gc() to declare size of objects so we avoid
 # garbage collection pitfalls.
 MINIMUM_CFFI_VERSION = '1.11'
 
 try:
     import cffi
 
     # PyPy (and possibly other distros) have CFFI distributed as part of
     # them. The install_requires for CFFI below won't work. We need to sniff
     # out the CFFI version here and reject CFFI if it is too old.
     cffi_version = LooseVersion(cffi.__version__)
     if cffi_version < LooseVersion(MINIMUM_CFFI_VERSION):
         print('CFFI 1.11 or newer required (%s found); '
               'not building CFFI backend' % cffi_version,
               file=sys.stderr)
         cffi = None
 
 except ImportError:
     cffi = None
 
 import setup_zstd
 
 SUPPORT_LEGACY = False
 SYSTEM_ZSTD = False
 WARNINGS_AS_ERRORS = False
 
 if os.environ.get('ZSTD_WARNINGS_AS_ERRORS', ''):
     WARNINGS_AS_ERRORS = True
 
 if '--legacy' in sys.argv:
     SUPPORT_LEGACY = True
     sys.argv.remove('--legacy')
 
 if '--system-zstd' in sys.argv:
     SYSTEM_ZSTD = True
     sys.argv.remove('--system-zstd')
 
 if '--warnings-as-errors' in sys.argv:
     WARNINGS_AS_ERRORS = True
     sys.argv.remove('--warning-as-errors')
 
 # Code for obtaining the Extension instance is in its own module to
 # facilitate reuse in other projects.
 extensions = [
     setup_zstd.get_c_extension(name='zstd',
                                support_legacy=SUPPORT_LEGACY,
                                system_zstd=SYSTEM_ZSTD,
                                warnings_as_errors=WARNINGS_AS_ERRORS),
 ]
 
 install_requires = []
 
 if cffi:
     import make_cffi
     extensions.append(make_cffi.ffi.distutils_extension())
     install_requires.append('cffi>=%s' % MINIMUM_CFFI_VERSION)
 
 version = None
 
 with open('c-ext/python-zstandard.h', 'r') as fh:
     for line in fh:
         if not line.startswith('#define PYTHON_ZSTANDARD_VERSION'):
             continue
 
         version = line.split()[2][1:-1]
         break
 
 if not version:
     raise Exception('could not resolve package version; '
                     'this should never happen')
 
 setup(
     name='zstandard',
     version=version,
     description='Zstandard bindings for Python',
     long_description=open('README.rst', 'r').read(),
     url='https://github.com/indygreg/python-zstandard',
     author='Gregory Szorc',
     author_email='[e-mail]',
     license='BSD',
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Programming Language :: C',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
     ],
     keywords='zstandard zstd compression',
     packages=['zstandard'],
     ext_modules=extensions,
     test_suite='tests',
     install_requires=install_requires,
 )

contrib/python-zstandard/setup_zstd.py

@@ -1,190 +1,192 b''
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 import distutils.ccompiler
 import os
 
 from distutils.extension import Extension
 
 
 zstd_sources = ['zstd/%s' % p for p in (
     'common/debug.c',
     'common/entropy_common.c',
     'common/error_private.c',
     'common/fse_decompress.c',
     'common/pool.c',
     'common/threading.c',
     'common/xxhash.c',
     'common/zstd_common.c',
     'compress/fse_compress.c',
     'compress/hist.c',
     'compress/huf_compress.c',
+    'compress/zstd_compress_literals.c',
+    'compress/zstd_compress_sequences.c',
     'compress/zstd_compress.c',
     'compress/zstd_double_fast.c',
     'compress/zstd_fast.c',
     'compress/zstd_lazy.c',
     'compress/zstd_ldm.c',
     'compress/zstd_opt.c',
     'compress/zstdmt_compress.c',
     'decompress/huf_decompress.c',
     'decompress/zstd_ddict.c',
     'decompress/zstd_decompress.c',
     'decompress/zstd_decompress_block.c',
     'dictBuilder/cover.c',
     'dictBuilder/divsufsort.c',
     'dictBuilder/fastcover.c',
     'dictBuilder/zdict.c',
 )]
 
 zstd_sources_legacy = ['zstd/%s' % p for p in (
     'deprecated/zbuff_common.c',
     'deprecated/zbuff_compress.c',
     'deprecated/zbuff_decompress.c',
     'legacy/zstd_v01.c',
     'legacy/zstd_v02.c',
     'legacy/zstd_v03.c',
     'legacy/zstd_v04.c',
     'legacy/zstd_v05.c',
     'legacy/zstd_v06.c',
     'legacy/zstd_v07.c'
 )]
 
 zstd_includes = [
     'zstd',
     'zstd/common',
     'zstd/compress',
     'zstd/decompress',
     'zstd/dictBuilder',
 ]
 
 zstd_includes_legacy = [
     'zstd/deprecated',
     'zstd/legacy',
 ]
 
 ext_includes = [
     'c-ext',
     'zstd/common',
 ]
 
 ext_sources = [
     'zstd/common/pool.c',
     'zstd/common/threading.c',
     'zstd.c',
     'c-ext/bufferutil.c',
     'c-ext/compressiondict.c',
     'c-ext/compressobj.c',
     'c-ext/compressor.c',
     'c-ext/compressoriterator.c',
     'c-ext/compressionchunker.c',
     'c-ext/compressionparams.c',
     'c-ext/compressionreader.c',
     'c-ext/compressionwriter.c',
     'c-ext/constants.c',
     'c-ext/decompressobj.c',
     'c-ext/decompressor.c',
     'c-ext/decompressoriterator.c',
     'c-ext/decompressionreader.c',
     'c-ext/decompressionwriter.c',
     'c-ext/frameparams.c',
 ]
 
 zstd_depends = [
     'c-ext/python-zstandard.h',
 ]
 
 
 def get_c_extension(support_legacy=False, system_zstd=False, name='zstd',
                     warnings_as_errors=False, root=None):
     """Obtain a distutils.extension.Extension for the C extension.
 
     ``support_legacy`` controls whether to compile in legacy zstd format support.
 
     ``system_zstd`` controls whether to compile against the system zstd library.
     For this to work, the system zstd library and headers must match what
     python-zstandard is coded against exactly.
 
     ``name`` is the module name of the C extension to produce.
 
     ``warnings_as_errors`` controls whether compiler warnings are turned into
     compiler errors.
 
     ``root`` defines a root path that source should be computed as relative
     to. This should be the directory with the main ``setup.py`` that is
     being invoked. If not defined, paths will be relative to this file.
     """
     actual_root = os.path.abspath(os.path.dirname(__file__))
     root = root or actual_root
 
     sources = set([os.path.join(actual_root, p) for p in ext_sources])
     if not system_zstd:
         sources.update([os.path.join(actual_root, p) for p in zstd_sources])
         if support_legacy:
             sources.update([os.path.join(actual_root, p)
                             for p in zstd_sources_legacy])
     sources = list(sources)
 
     include_dirs = set([os.path.join(actual_root, d) for d in ext_includes])
     if not system_zstd:
         include_dirs.update([os.path.join(actual_root, d)
                              for d in zstd_includes])
         if support_legacy:
             include_dirs.update([os.path.join(actual_root, d)
                                  for d in zstd_includes_legacy])
     include_dirs = list(include_dirs)
 
     depends = [os.path.join(actual_root, p) for p in zstd_depends]
 
     compiler = distutils.ccompiler.new_compiler()
 
     # Needed for MSVC.
     if hasattr(compiler, 'initialize'):
         compiler.initialize()
 
     if compiler.compiler_type == 'unix':
         compiler_type = 'unix'
     elif compiler.compiler_type == 'msvc':
         compiler_type = 'msvc'
     elif compiler.compiler_type == 'mingw32':
         compiler_type = 'mingw32'
     else:
         raise Exception('unhandled compiler type: %s' %
                         compiler.compiler_type)
 
     extra_args = ['-DZSTD_MULTITHREAD']
 
     if not system_zstd:
         extra_args.append('-DZSTDLIB_VISIBILITY=')
         extra_args.append('-DZDICTLIB_VISIBILITY=')
         extra_args.append('-DZSTDERRORLIB_VISIBILITY=')
 
         if compiler_type == 'unix':
             extra_args.append('-fvisibility=hidden')
 
     if not system_zstd and support_legacy:
         extra_args.append('-DZSTD_LEGACY_SUPPORT=1')
 
     if warnings_as_errors:
         if compiler_type in ('unix', 'mingw32'):
             extra_args.append('-Werror')
         elif compiler_type == 'msvc':
             extra_args.append('/WX')
         else:
             assert False
 
     libraries = ['zstd'] if system_zstd else []
 
     # Python 3.7 doesn't like absolute paths. So normalize to relative.
     sources = [os.path.relpath(p, root) for p in sources]
     include_dirs = [os.path.relpath(p, root) for p in include_dirs]
     depends = [os.path.relpath(p, root) for p in depends]
 
     # TODO compile with optimizations.
     return Extension(name, sources,
                      include_dirs=include_dirs,
                      depends=depends,
                      extra_compile_args=extra_args,
                      libraries=libraries)

contrib/python-zstandard/tests/test_compressor.py

@@ -1,1735 +1,1735 b''
 import hashlib
 import io
 import os
 import struct
 import sys
 import tarfile
 import tempfile
 import unittest
 
 import zstandard as zstd
 
 from .common import (
     make_cffi,
     NonClosingBytesIO,
     OpCountingBytesIO,
 )
 
 
 if sys.version_info[0] >= 3:
     next = lambda it: it.__next__()
 else:
     next = lambda it: it.next()
 
 
 def multithreaded_chunk_size(level, source_size=0):
     params = zstd.ZstdCompressionParameters.from_level(level,
                                                        source_size=source_size)
 
     return 1 << (params.window_log + 2)
 
 
 @make_cffi
 class TestCompressor(unittest.TestCase):
     def test_level_bounds(self):
         with self.assertRaises(ValueError):
             zstd.ZstdCompressor(level=23)
 
     def test_memory_size(self):
         cctx = zstd.ZstdCompressor(level=1)
         self.assertGreater(cctx.memory_size(), 100)
 
 
 @make_cffi
 class TestCompressor_compress(unittest.TestCase):
     def test_compress_empty(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         result = cctx.compress(b'')
         self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 524288)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum, 0)
 
         cctx = zstd.ZstdCompressor()
         result = cctx.compress(b'')
         self.assertEqual(result, b'\x28\xb5\x2f\xfd\x20\x00\x01\x00\x00')
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, 0)
 
     def test_input_types(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         expected = b'\x28\xb5\x2f\xfd\x00\x00\x19\x00\x00\x66\x6f\x6f'
 
         mutable_array = bytearray(3)
         mutable_array[:] = b'foo'
 
         sources = [
             memoryview(b'foo'),
             bytearray(b'foo'),
             mutable_array,
         ]
 
         for source in sources:
             self.assertEqual(cctx.compress(source), expected)
 
     def test_compress_large(self):
         chunks = []
         for i in range(255):
             chunks.append(struct.Struct('>B').pack(i) * 16384)
 
         cctx = zstd.ZstdCompressor(level=3, write_content_size=False)
         result = cctx.compress(b''.join(chunks))
         self.assertEqual(len(result), 999)
         self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd')
 
         # This matches the test for read_to_iter() below.
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         result = cctx.compress(b'f' * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b'o')
         self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x40\x54\x00\x00'
                                  b'\x10\x66\x66\x01\x00\xfb\xff\x39\xc0'
                                  b'\x02\x09\x00\x00\x6f')
 
     def test_negative_level(self):
         cctx = zstd.ZstdCompressor(level=-4)
         result = cctx.compress(b'foo' * 256)
 
     def test_no_magic(self):
         params = zstd.ZstdCompressionParameters.from_level(
             1, format=zstd.FORMAT_ZSTD1)
         cctx = zstd.ZstdCompressor(compression_params=params)
         magic = cctx.compress(b'foobar')
 
         params = zstd.ZstdCompressionParameters.from_level(
             1, format=zstd.FORMAT_ZSTD1_MAGICLESS)
         cctx = zstd.ZstdCompressor(compression_params=params)
         no_magic = cctx.compress(b'foobar')
 
         self.assertEqual(magic[0:4], b'\x28\xb5\x2f\xfd')
         self.assertEqual(magic[4:], no_magic)
 
     def test_write_checksum(self):
         cctx = zstd.ZstdCompressor(level=1)
         no_checksum = cctx.compress(b'foobar')
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         with_checksum = cctx.compress(b'foobar')
 
         self.assertEqual(len(with_checksum), len(no_checksum) + 4)
 
         no_params = zstd.get_frame_parameters(no_checksum)
         with_params = zstd.get_frame_parameters(with_checksum)
 
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
     def test_write_content_size(self):
         cctx = zstd.ZstdCompressor(level=1)
         with_size = cctx.compress(b'foobar' * 256)
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         no_size = cctx.compress(b'foobar' * 256)
 
         self.assertEqual(len(with_size), len(no_size) + 1)
 
         no_params = zstd.get_frame_parameters(no_size)
         with_params = zstd.get_frame_parameters(with_size)
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
 
     def test_no_dict_id(self):
         samples = []
         for i in range(128):
             samples.append(b'foo' * 64)
             samples.append(b'bar' * 64)
             samples.append(b'foobar' * 64)
 
         d = zstd.train_dictionary(1024, samples)
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         with_dict_id = cctx.compress(b'foobarfoobar')
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False)
         no_dict_id = cctx.compress(b'foobarfoobar')
 
         self.assertEqual(len(with_dict_id), len(no_dict_id) + 4)
 
         no_params = zstd.get_frame_parameters(no_dict_id)
         with_params = zstd.get_frame_parameters(with_dict_id)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 1880053135)
 
     def test_compress_dict_multiple(self):
         samples = []
         for i in range(128):
             samples.append(b'foo' * 64)
             samples.append(b'bar' * 64)
             samples.append(b'foobar' * 64)
 
         d = zstd.train_dictionary(8192, samples)
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
 
         for i in range(32):
             cctx.compress(b'foo bar foobar foo bar foobar')
 
     def test_dict_precompute(self):
         samples = []
         for i in range(128):
             samples.append(b'foo' * 64)
             samples.append(b'bar' * 64)
             samples.append(b'foobar' * 64)
 
         d = zstd.train_dictionary(8192, samples)
         d.precompute_compress(level=1)
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
 
         for i in range(32):
             cctx.compress(b'foo bar foobar foo bar foobar')
 
     def test_multithreaded(self):
         chunk_size = multithreaded_chunk_size(1)
         source = b''.join([b'x' * chunk_size, b'y' * chunk_size])
 
         cctx = zstd.ZstdCompressor(level=1, threads=2)
         compressed = cctx.compress(source)
 
         params = zstd.get_frame_parameters(compressed)
         self.assertEqual(params.content_size, chunk_size * 2)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         dctx = zstd.ZstdDecompressor()
         self.assertEqual(dctx.decompress(compressed), source)
 
     def test_multithreaded_dict(self):
         samples = []
         for i in range(128):
             samples.append(b'foo' * 64)
             samples.append(b'bar' * 64)
             samples.append(b'foobar' * 64)
 
         d = zstd.train_dictionary(1024, samples)
 
         cctx = zstd.ZstdCompressor(dict_data=d, threads=2)
 
         result = cctx.compress(b'foo')
         params = zstd.get_frame_parameters(result);
         self.assertEqual(params.content_size, 3);
         self.assertEqual(params.dict_id, d.dict_id())
 
         self.assertEqual(result,
                          b'\x28\xb5\x2f\xfd\x23\x8f\x55\x0f\x70\x03\x19\x00\x00'
                          b'\x66\x6f\x6f')
 
     def test_multithreaded_compression_params(self):
         params = zstd.ZstdCompressionParameters.from_level(0, threads=2)
         cctx = zstd.ZstdCompressor(compression_params=params)
 
         result = cctx.compress(b'foo')
         params = zstd.get_frame_parameters(result);
         self.assertEqual(params.content_size, 3);
 
         self.assertEqual(result,
                          b'\x28\xb5\x2f\xfd\x20\x03\x19\x00\x00\x66\x6f\x6f')
 
 
 @make_cffi
 class TestCompressor_compressobj(unittest.TestCase):
     def test_compressobj_empty(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         cobj = cctx.compressobj()
         self.assertEqual(cobj.compress(b''), b'')
         self.assertEqual(cobj.flush(),
                          b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
 
     def test_input_types(self):
         expected = b'\x28\xb5\x2f\xfd\x00\x48\x19\x00\x00\x66\x6f\x6f'
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
 
         mutable_array = bytearray(3)
         mutable_array[:] = b'foo'
 
         sources = [
             memoryview(b'foo'),
             bytearray(b'foo'),
             mutable_array,
         ]
 
         for source in sources:
             cobj = cctx.compressobj()
             self.assertEqual(cobj.compress(source), b'')
             self.assertEqual(cobj.flush(), expected)
 
     def test_compressobj_large(self):
         chunks = []
         for i in range(255):
             chunks.append(struct.Struct('>B').pack(i) * 16384)
 
         cctx = zstd.ZstdCompressor(level=3)
         cobj = cctx.compressobj()
 
         result = cobj.compress(b''.join(chunks)) + cobj.flush()
         self.assertEqual(len(result), 999)
         self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd')
 
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
     def test_write_checksum(self):
         cctx = zstd.ZstdCompressor(level=1)
         cobj = cctx.compressobj()
         no_checksum = cobj.compress(b'foobar') + cobj.flush()
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         cobj = cctx.compressobj()
         with_checksum = cobj.compress(b'foobar') + cobj.flush()
 
         no_params = zstd.get_frame_parameters(no_checksum)
         with_params = zstd.get_frame_parameters(with_checksum)
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
         self.assertEqual(len(with_checksum), len(no_checksum) + 4)
 
     def test_write_content_size(self):
         cctx = zstd.ZstdCompressor(level=1)
         cobj = cctx.compressobj(size=len(b'foobar' * 256))
         with_size = cobj.compress(b'foobar' * 256) + cobj.flush()
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         cobj = cctx.compressobj(size=len(b'foobar' * 256))
         no_size = cobj.compress(b'foobar' * 256) + cobj.flush()
 
         no_params = zstd.get_frame_parameters(no_size)
         with_params = zstd.get_frame_parameters(with_size)
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
         self.assertEqual(len(with_size), len(no_size) + 1)
 
     def test_compress_after_finished(self):
         cctx = zstd.ZstdCompressor()
         cobj = cctx.compressobj()
 
         cobj.compress(b'foo')
         cobj.flush()
 
         with self.assertRaisesRegexp(zstd.ZstdError, r'cannot call compress\(\) after compressor'):
             cobj.compress(b'foo')
 
         with self.assertRaisesRegexp(zstd.ZstdError, 'compressor object already finished'):
             cobj.flush()
 
     def test_flush_block_repeated(self):
         cctx = zstd.ZstdCompressor(level=1)
         cobj = cctx.compressobj()
 
         self.assertEqual(cobj.compress(b'foo'), b'')
         self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK),
                          b'\x28\xb5\x2f\xfd\x00\x48\x18\x00\x00foo')
         self.assertEqual(cobj.compress(b'bar'), b'')
         # 3 byte header plus content.
         self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK),
                          b'\x18\x00\x00bar')
         self.assertEqual(cobj.flush(), b'\x01\x00\x00')
 
     def test_flush_empty_block(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
         cobj = cctx.compressobj()
 
         cobj.compress(b'foobar')
         cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK)
         # No-op if no block is active (this is internal to zstd).
         self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b'')
 
         trailing = cobj.flush()
         # 3 bytes block header + 4 bytes frame checksum
         self.assertEqual(len(trailing), 7)
         header = trailing[0:3]
         self.assertEqual(header, b'\x01\x00\x00')
 
     def test_multithreaded(self):
         source = io.BytesIO()
         source.write(b'a' * 1048576)
         source.write(b'b' * 1048576)
         source.write(b'c' * 1048576)
         source.seek(0)
 
         cctx = zstd.ZstdCompressor(level=1, threads=2)
         cobj = cctx.compressobj()
 
         chunks = []
         while True:
             d = source.read(8192)
             if not d:
                 break
 
             chunks.append(cobj.compress(d))
 
         chunks.append(cobj.flush())
 
         compressed = b''.join(chunks)
 
         self.assertEqual(len(compressed), 295)
 
     def test_frame_progression(self):
         cctx = zstd.ZstdCompressor()
 
         self.assertEqual(cctx.frame_progression(), (0, 0, 0))
 
         cobj = cctx.compressobj()
 
         cobj.compress(b'foobar')
         self.assertEqual(cctx.frame_progression(), (6, 0, 0))
 
         cobj.flush()
         self.assertEqual(cctx.frame_progression(), (6, 6, 15))
 
     def test_bad_size(self):
         cctx = zstd.ZstdCompressor()
 
         cobj = cctx.compressobj(size=2)
         with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
             cobj.compress(b'foo')
 
         # Try another operation on this instance.
         with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
             cobj.compress(b'aa')
 
         # Try another operation on the compressor.
         cctx.compressobj(size=4)
         cctx.compress(b'foobar')
 
 
 @make_cffi
 class TestCompressor_copy_stream(unittest.TestCase):
     def test_no_read(self):
         source = object()
         dest = io.BytesIO()
 
         cctx = zstd.ZstdCompressor()
         with self.assertRaises(ValueError):
             cctx.copy_stream(source, dest)
 
     def test_no_write(self):
         source = io.BytesIO()
         dest = object()
 
         cctx = zstd.ZstdCompressor()
         with self.assertRaises(ValueError):
             cctx.copy_stream(source, dest)
 
     def test_empty(self):
         source = io.BytesIO()
         dest = io.BytesIO()
 
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         r, w = cctx.copy_stream(source, dest)
         self.assertEqual(int(r), 0)
         self.assertEqual(w, 9)
 
         self.assertEqual(dest.getvalue(),
                          b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
 
     def test_large_data(self):
         source = io.BytesIO()
         for i in range(255):
             source.write(struct.Struct('>B').pack(i) * 16384)
         source.seek(0)
 
         dest = io.BytesIO()
         cctx = zstd.ZstdCompressor()
         r, w = cctx.copy_stream(source, dest)
 
         self.assertEqual(r, 255 * 16384)
         self.assertEqual(w, 999)
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
     def test_write_checksum(self):
         source = io.BytesIO(b'foobar')
         no_checksum = io.BytesIO()
 
         cctx = zstd.ZstdCompressor(level=1)
         cctx.copy_stream(source, no_checksum)
 
         source.seek(0)
         with_checksum = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         cctx.copy_stream(source, with_checksum)
 
         self.assertEqual(len(with_checksum.getvalue()),
                          len(no_checksum.getvalue()) + 4)
 
         no_params = zstd.get_frame_parameters(no_checksum.getvalue())
         with_params = zstd.get_frame_parameters(with_checksum.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
     def test_write_content_size(self):
         source = io.BytesIO(b'foobar' * 256)
         no_size = io.BytesIO()
 
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         cctx.copy_stream(source, no_size)
 
         source.seek(0)
         with_size = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         cctx.copy_stream(source, with_size)
 
         # Source content size is unknown, so no content size written.
         self.assertEqual(len(with_size.getvalue()),
                          len(no_size.getvalue()))
 
         source.seek(0)
         with_size = io.BytesIO()
         cctx.copy_stream(source, with_size, size=len(source.getvalue()))
 
         # We specified source size, so content size header is present.
         self.assertEqual(len(with_size.getvalue()),
                          len(no_size.getvalue()) + 1)
 
         no_params = zstd.get_frame_parameters(no_size.getvalue())
         with_params = zstd.get_frame_parameters(with_size.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
     def test_read_write_size(self):
         source = OpCountingBytesIO(b'foobarfoobar')
         dest = OpCountingBytesIO()
         cctx = zstd.ZstdCompressor()
         r, w = cctx.copy_stream(source, dest, read_size=1, write_size=1)
 
         self.assertEqual(r, len(source.getvalue()))
         self.assertEqual(w, 21)
         self.assertEqual(source._read_count, len(source.getvalue()) + 1)
         self.assertEqual(dest._write_count, len(dest.getvalue()))
 
     def test_multithreaded(self):
         source = io.BytesIO()
         source.write(b'a' * 1048576)
         source.write(b'b' * 1048576)
         source.write(b'c' * 1048576)
         source.seek(0)
 
         dest = io.BytesIO()
         cctx = zstd.ZstdCompressor(threads=2, write_content_size=False)
         r, w = cctx.copy_stream(source, dest)
         self.assertEqual(r, 3145728)
         self.assertEqual(w, 295)
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Writing content size and checksum works.
         cctx = zstd.ZstdCompressor(threads=2, write_checksum=True)
         dest = io.BytesIO()
         source.seek(0)
         cctx.copy_stream(source, dest, size=len(source.getvalue()))
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, 3145728)
         self.assertEqual(params.dict_id, 0)
         self.assertTrue(params.has_checksum)
 
     def test_bad_size(self):
         source = io.BytesIO()
         source.write(b'a' * 32768)
         source.write(b'b' * 32768)
         source.seek(0)
 
         dest = io.BytesIO()
 
         cctx = zstd.ZstdCompressor()
 
         with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
             cctx.copy_stream(source, dest, size=42)
 
         # Try another operation on this compressor.
         source.seek(0)
         dest = io.BytesIO()
         cctx.copy_stream(source, dest)
 
 
 @make_cffi
 class TestCompressor_stream_reader(unittest.TestCase):
     def test_context_manager(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b'foo') as reader:
             with self.assertRaisesRegexp(ValueError, 'cannot __enter__ multiple times'):
                 with reader as reader2:
                     pass
 
     def test_no_context_manager(self):
         cctx = zstd.ZstdCompressor()
 
         reader = cctx.stream_reader(b'foo')
         reader.read(4)
         self.assertFalse(reader.closed)
 
         reader.close()
         self.assertTrue(reader.closed)
         with self.assertRaisesRegexp(ValueError, 'stream is closed'):
             reader.read(1)
 
     def test_not_implemented(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b'foo' * 60) as reader:
             with self.assertRaises(io.UnsupportedOperation):
                 reader.readline()
 
             with self.assertRaises(io.UnsupportedOperation):
                 reader.readlines()
 
             with self.assertRaises(io.UnsupportedOperation):
                 iter(reader)
 
             with self.assertRaises(io.UnsupportedOperation):
                 next(reader)
 
             with self.assertRaises(OSError):
                 reader.writelines([])
 
             with self.assertRaises(OSError):
                 reader.write(b'foo')
 
     def test_constant_methods(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b'boo') as reader:
             self.assertTrue(reader.readable())
             self.assertFalse(reader.writable())
             self.assertFalse(reader.seekable())
             self.assertFalse(reader.isatty())
             self.assertFalse(reader.closed)
             self.assertIsNone(reader.flush())
             self.assertFalse(reader.closed)
 
         self.assertTrue(reader.closed)
 
     def test_read_closed(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b'foo' * 60) as reader:
             reader.close()
             self.assertTrue(reader.closed)
             with self.assertRaisesRegexp(ValueError, 'stream is closed'):
                 reader.read(10)
 
     def test_read_sizes(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b'foo')
 
         with cctx.stream_reader(b'foo') as reader:
             with self.assertRaisesRegexp(ValueError, 'cannot read negative amounts less than -1'):
                 reader.read(-2)
 
             self.assertEqual(reader.read(0), b'')
             self.assertEqual(reader.read(), foo)
 
     def test_read_buffer(self):
         cctx = zstd.ZstdCompressor()
 
         source = b''.join([b'foo' * 60, b'bar' * 60, b'baz' * 60])
         frame = cctx.compress(source)
 
         with cctx.stream_reader(source) as reader:
             self.assertEqual(reader.tell(), 0)
 
             # We should get entire frame in one read.
             result = reader.read(8192)
             self.assertEqual(result, frame)
             self.assertEqual(reader.tell(), len(result))
             self.assertEqual(reader.read(), b'')
             self.assertEqual(reader.tell(), len(result))
 
     def test_read_buffer_small_chunks(self):
         cctx = zstd.ZstdCompressor()
 
         source = b'foo' * 60
         chunks = []
 
         with cctx.stream_reader(source) as reader:
             self.assertEqual(reader.tell(), 0)
 
             while True:
                 chunk = reader.read(1)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
                 self.assertEqual(reader.tell(), sum(map(len, chunks)))
 
         self.assertEqual(b''.join(chunks), cctx.compress(source))
 
     def test_read_stream(self):
         cctx = zstd.ZstdCompressor()
 
         source = b''.join([b'foo' * 60, b'bar' * 60, b'baz' * 60])
         frame = cctx.compress(source)
 
         with cctx.stream_reader(io.BytesIO(source), size=len(source)) as reader:
             self.assertEqual(reader.tell(), 0)
 
             chunk = reader.read(8192)
             self.assertEqual(chunk, frame)
             self.assertEqual(reader.tell(), len(chunk))
             self.assertEqual(reader.read(), b'')
             self.assertEqual(reader.tell(), len(chunk))
 
     def test_read_stream_small_chunks(self):
         cctx = zstd.ZstdCompressor()
 
         source = b'foo' * 60
         chunks = []
 
         with cctx.stream_reader(io.BytesIO(source), size=len(source)) as reader:
             self.assertEqual(reader.tell(), 0)
 
             while True:
                 chunk = reader.read(1)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
                 self.assertEqual(reader.tell(), sum(map(len, chunks)))
 
         self.assertEqual(b''.join(chunks), cctx.compress(source))
 
     def test_read_after_exit(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b'foo' * 60) as reader:
             while reader.read(8192):
                 pass
 
         with self.assertRaisesRegexp(ValueError, 'stream is closed'):
             reader.read(10)
 
     def test_bad_size(self):
         cctx = zstd.ZstdCompressor()
 
         source = io.BytesIO(b'foobar')
 
         with cctx.stream_reader(source, size=2) as reader:
             with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
                 reader.read(10)
 
         # Try another compression operation.
         with cctx.stream_reader(source, size=42):
             pass
 
     def test_readall(self):
         cctx = zstd.ZstdCompressor()
         frame = cctx.compress(b'foo' * 1024)
 
         reader = cctx.stream_reader(b'foo' * 1024)
         self.assertEqual(reader.readall(), frame)
 
     def test_readinto(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b'foo')
 
         reader = cctx.stream_reader(b'foo')
         with self.assertRaises(Exception):
             reader.readinto(b'foobar')
 
         # readinto() with sufficiently large destination.
         b = bytearray(1024)
         reader = cctx.stream_reader(b'foo')
         self.assertEqual(reader.readinto(b), len(foo))
         self.assertEqual(b[0:len(foo)], foo)
         self.assertEqual(reader.readinto(b), 0)
         self.assertEqual(b[0:len(foo)], foo)
 
         # readinto() with small reads.
         b = bytearray(1024)
         reader = cctx.stream_reader(b'foo', read_size=1)
         self.assertEqual(reader.readinto(b), len(foo))
         self.assertEqual(b[0:len(foo)], foo)
 
         # Too small destination buffer.
         b = bytearray(2)
         reader = cctx.stream_reader(b'foo')
         self.assertEqual(reader.readinto(b), 2)
         self.assertEqual(b[:], foo[0:2])
         self.assertEqual(reader.readinto(b), 2)
         self.assertEqual(b[:], foo[2:4])
         self.assertEqual(reader.readinto(b), 2)
         self.assertEqual(b[:], foo[4:6])
 
     def test_readinto1(self):
         cctx = zstd.ZstdCompressor()
         foo = b''.join(cctx.read_to_iter(io.BytesIO(b'foo')))
 
         reader = cctx.stream_reader(b'foo')
         with self.assertRaises(Exception):
             reader.readinto1(b'foobar')
 
         b = bytearray(1024)
         source = OpCountingBytesIO(b'foo')
         reader = cctx.stream_reader(source)
         self.assertEqual(reader.readinto1(b), len(foo))
         self.assertEqual(b[0:len(foo)], foo)
         self.assertEqual(source._read_count, 2)
 
         # readinto1() with small reads.
         b = bytearray(1024)
         source = OpCountingBytesIO(b'foo')
         reader = cctx.stream_reader(source, read_size=1)
         self.assertEqual(reader.readinto1(b), len(foo))
         self.assertEqual(b[0:len(foo)], foo)
         self.assertEqual(source._read_count, 4)
 
     def test_read1(self):
         cctx = zstd.ZstdCompressor()
         foo = b''.join(cctx.read_to_iter(io.BytesIO(b'foo')))
 
         b = OpCountingBytesIO(b'foo')
         reader = cctx.stream_reader(b)
 
         self.assertEqual(reader.read1(), foo)
         self.assertEqual(b._read_count, 2)
 
         b = OpCountingBytesIO(b'foo')
         reader = cctx.stream_reader(b)
 
         self.assertEqual(reader.read1(0), b'')
         self.assertEqual(reader.read1(2), foo[0:2])
         self.assertEqual(b._read_count, 2)
         self.assertEqual(reader.read1(2), foo[2:4])
         self.assertEqual(reader.read1(1024), foo[4:])
 
 
 @make_cffi
 class TestCompressor_stream_writer(unittest.TestCase):
     def test_io_api(self):
         buffer = io.BytesIO()
         cctx = zstd.ZstdCompressor()
         writer = cctx.stream_writer(buffer)
 
         self.assertFalse(writer.isatty())
         self.assertFalse(writer.readable())
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline(size=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines(hint=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.seek(0)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.seek(10, os.SEEK_SET)
 
         self.assertFalse(writer.seekable())
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate(size=42)
 
         self.assertTrue(writer.writable())
 
         with self.assertRaises(NotImplementedError):
             writer.writelines([])
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read(size=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readall()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readinto(None)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.fileno()
 
         self.assertFalse(writer.closed)
 
     def test_fileno_file(self):
         with tempfile.TemporaryFile('wb') as tf:
             cctx = zstd.ZstdCompressor()
             writer = cctx.stream_writer(tf)
 
             self.assertEqual(writer.fileno(), tf.fileno())
 
     def test_close(self):
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         writer = cctx.stream_writer(buffer)
 
         writer.write(b'foo' * 1024)
         self.assertFalse(writer.closed)
         self.assertFalse(buffer.closed)
         writer.close()
         self.assertTrue(writer.closed)
         self.assertTrue(buffer.closed)
 
         with self.assertRaisesRegexp(ValueError, 'stream is closed'):
             writer.write(b'foo')
 
         with self.assertRaisesRegexp(ValueError, 'stream is closed'):
             writer.flush()
 
         with self.assertRaisesRegexp(ValueError, 'stream is closed'):
             with writer:
                 pass
 
         self.assertEqual(buffer.getvalue(),
                          b'\x28\xb5\x2f\xfd\x00\x48\x55\x00\x00\x18\x66\x6f'
                          b'\x6f\x01\x00\xfa\xd3\x77\x43')
 
         # Context manager exit should close stream.
         buffer = io.BytesIO()
         writer = cctx.stream_writer(buffer)
 
         with writer:
             writer.write(b'foo')
 
         self.assertTrue(writer.closed)
 
     def test_empty(self):
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         with cctx.stream_writer(buffer) as compressor:
             compressor.write(b'')
 
         result = buffer.getvalue()
         self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
 
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 524288)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Test without context manager.
         buffer = io.BytesIO()
         compressor = cctx.stream_writer(buffer)
         self.assertEqual(compressor.write(b''), 0)
         self.assertEqual(buffer.getvalue(), b'')
         self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 9)
         result = buffer.getvalue()
         self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
 
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 524288)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Test write_return_read=True
         compressor = cctx.stream_writer(buffer, write_return_read=True)
         self.assertEqual(compressor.write(b''), 0)
 
     def test_input_types(self):
         expected = b'\x28\xb5\x2f\xfd\x00\x48\x19\x00\x00\x66\x6f\x6f'
         cctx = zstd.ZstdCompressor(level=1)
 
         mutable_array = bytearray(3)
         mutable_array[:] = b'foo'
 
         sources = [
             memoryview(b'foo'),
             bytearray(b'foo'),
             mutable_array,
         ]
 
         for source in sources:
             buffer = NonClosingBytesIO()
             with cctx.stream_writer(buffer) as compressor:
                 compressor.write(source)
 
             self.assertEqual(buffer.getvalue(), expected)
 
             compressor = cctx.stream_writer(buffer, write_return_read=True)
             self.assertEqual(compressor.write(source), len(source))
 
     def test_multiple_compress(self):
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=5)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(compressor.write(b'bar'), 0)
             self.assertEqual(compressor.write(b'x' * 8192), 0)
 
         result = buffer.getvalue()
         self.assertEqual(result,
                          b'\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x38\x66\x6f'
                          b'\x6f\x62\x61\x72\x78\x01\x00\xfc\xdf\x03\x23')
 
         # Test without context manager.
         buffer = io.BytesIO()
         compressor = cctx.stream_writer(buffer)
         self.assertEqual(compressor.write(b'foo'), 0)
         self.assertEqual(compressor.write(b'bar'), 0)
         self.assertEqual(compressor.write(b'x' * 8192), 0)
         self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 23)
         result = buffer.getvalue()
         self.assertEqual(result,
                          b'\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x38\x66\x6f'
                          b'\x6f\x62\x61\x72\x78\x01\x00\xfc\xdf\x03\x23')
 
         # Test with write_return_read=True.
         compressor = cctx.stream_writer(buffer, write_return_read=True)
         self.assertEqual(compressor.write(b'foo'), 3)
         self.assertEqual(compressor.write(b'barbiz'), 6)
         self.assertEqual(compressor.write(b'x' * 8192), 8192)
 
     def test_dictionary(self):
         samples = []
         for i in range(128):
             samples.append(b'foo' * 64)
             samples.append(b'bar' * 64)
             samples.append(b'foobar' * 64)
 
         d = zstd.train_dictionary(8192, samples)
 
         h = hashlib.sha1(d.as_bytes()).hexdigest()
-        self.assertEqual(h, '88ca0d38332aff379d4ced166a51c280a7679aad')
+        self.assertEqual(h, '7a2e59a876db958f74257141045af8f912e00d4e')
 
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=9, dict_data=d)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(compressor.write(b'bar'), 0)
             self.assertEqual(compressor.write(b'foo' * 16384), 0)
 
         compressed = buffer.getvalue()
 
         params = zstd.get_frame_parameters(compressed)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, d.dict_id())
         self.assertFalse(params.has_checksum)
 
         h = hashlib.sha1(compressed).hexdigest()
-        self.assertEqual(h, '8703b4316f274d26697ea5dd480f29c08e85d940')
+        self.assertEqual(h, '0a7c05635061f58039727cdbe76388c6f4cfef06')
 
         source = b'foo' + b'bar' + (b'foo' * 16384)
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
 
         self.assertEqual(dctx.decompress(compressed, max_output_size=len(source)),
                          source)
 
     def test_compression_params(self):
         params = zstd.ZstdCompressionParameters(
             window_log=20,
             chain_log=6,
             hash_log=12,
             min_match=5,
             search_log=4,
             target_length=10,
             strategy=zstd.STRATEGY_FAST)
 
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(compression_params=params)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(compressor.write(b'bar'), 0)
             self.assertEqual(compressor.write(b'foobar' * 16384), 0)
 
         compressed = buffer.getvalue()
 
         params = zstd.get_frame_parameters(compressed)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1048576)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         h = hashlib.sha1(compressed).hexdigest()
-        self.assertEqual(h, '2a8111d72eb5004cdcecbdac37da9f26720d30ef')
+        self.assertEqual(h, 'dd4bb7d37c1a0235b38a2f6b462814376843ef0b')
 
     def test_write_checksum(self):
         no_checksum = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         with cctx.stream_writer(no_checksum) as compressor:
             self.assertEqual(compressor.write(b'foobar'), 0)
 
         with_checksum = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         with cctx.stream_writer(with_checksum) as compressor:
             self.assertEqual(compressor.write(b'foobar'), 0)
 
         no_params = zstd.get_frame_parameters(no_checksum.getvalue())
         with_params = zstd.get_frame_parameters(with_checksum.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
         self.assertEqual(len(with_checksum.getvalue()),
                          len(no_checksum.getvalue()) + 4)
 
     def test_write_content_size(self):
         no_size = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         with cctx.stream_writer(no_size) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
 
         with_size = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         with cctx.stream_writer(with_size) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
 
         # Source size is not known in streaming mode, so header not
         # written.
         self.assertEqual(len(with_size.getvalue()),
                          len(no_size.getvalue()))
 
         # Declaring size will write the header.
         with_size = NonClosingBytesIO()
         with cctx.stream_writer(with_size, size=len(b'foobar' * 256)) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
 
         no_params = zstd.get_frame_parameters(no_size.getvalue())
         with_params = zstd.get_frame_parameters(with_size.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
         self.assertEqual(len(with_size.getvalue()),
                          len(no_size.getvalue()) + 1)
 
     def test_no_dict_id(self):
         samples = []
         for i in range(128):
             samples.append(b'foo' * 64)
             samples.append(b'bar' * 64)
             samples.append(b'foobar' * 64)
 
         d = zstd.train_dictionary(1024, samples)
 
         with_dict_id = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         with cctx.stream_writer(with_dict_id) as compressor:
             self.assertEqual(compressor.write(b'foobarfoobar'), 0)
 
         self.assertEqual(with_dict_id.getvalue()[4:5], b'\x03')
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False)
         no_dict_id = NonClosingBytesIO()
         with cctx.stream_writer(no_dict_id) as compressor:
             self.assertEqual(compressor.write(b'foobarfoobar'), 0)
 
         self.assertEqual(no_dict_id.getvalue()[4:5], b'\x00')
 
         no_params = zstd.get_frame_parameters(no_dict_id.getvalue())
         with_params = zstd.get_frame_parameters(with_dict_id.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, d.dict_id())
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
         self.assertEqual(len(with_dict_id.getvalue()),
                          len(no_dict_id.getvalue()) + 4)
 
     def test_memory_size(self):
         cctx = zstd.ZstdCompressor(level=3)
         buffer = io.BytesIO()
         with cctx.stream_writer(buffer) as compressor:
             compressor.write(b'foo')
             size = compressor.memory_size()
 
         self.assertGreater(size, 100000)
 
     def test_write_size(self):
         cctx = zstd.ZstdCompressor(level=3)
         dest = OpCountingBytesIO()
         with cctx.stream_writer(dest, write_size=1) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(compressor.write(b'bar'), 0)
             self.assertEqual(compressor.write(b'foobar'), 0)
 
         self.assertEqual(len(dest.getvalue()), dest._write_count)
 
     def test_flush_repeated(self):
         cctx = zstd.ZstdCompressor(level=3)
         dest = OpCountingBytesIO()
         with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(dest._write_count, 0)
             self.assertEqual(compressor.flush(), 12)
             self.assertEqual(dest._write_count, 1)
             self.assertEqual(compressor.write(b'bar'), 0)
             self.assertEqual(dest._write_count, 1)
             self.assertEqual(compressor.flush(), 6)
             self.assertEqual(dest._write_count, 2)
             self.assertEqual(compressor.write(b'baz'), 0)
 
         self.assertEqual(dest._write_count, 3)
 
     def test_flush_empty_block(self):
         cctx = zstd.ZstdCompressor(level=3, write_checksum=True)
         dest = OpCountingBytesIO()
         with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 8192), 0)
             count = dest._write_count
             offset = dest.tell()
             self.assertEqual(compressor.flush(), 23)
             self.assertGreater(dest._write_count, count)
             self.assertGreater(dest.tell(), offset)
             offset = dest.tell()
             # Ending the write here should cause an empty block to be written
             # to denote end of frame.
 
         trailing = dest.getvalue()[offset:]
         # 3 bytes block header + 4 bytes frame checksum
         self.assertEqual(len(trailing), 7)
 
         header = trailing[0:3]
         self.assertEqual(header, b'\x01\x00\x00')
 
     def test_flush_frame(self):
         cctx = zstd.ZstdCompressor(level=3)
         dest = OpCountingBytesIO()
 
         with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 8192), 0)
             self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 23)
             compressor.write(b'biz' * 16384)
 
         self.assertEqual(dest.getvalue(),
                          # Frame 1.
                          b'\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x30\x66\x6f\x6f'
                          b'\x62\x61\x72\x01\x00\xf7\xbf\xe8\xa5\x08'
                          # Frame 2.
                          b'\x28\xb5\x2f\xfd\x00\x58\x5d\x00\x00\x18\x62\x69\x7a'
                          b'\x01\x00\xfa\x3f\x75\x37\x04')
 
     def test_bad_flush_mode(self):
         cctx = zstd.ZstdCompressor()
         dest = io.BytesIO()
         with cctx.stream_writer(dest) as compressor:
             with self.assertRaisesRegexp(ValueError, 'unknown flush_mode: 42'):
                 compressor.flush(flush_mode=42)
 
     def test_multithreaded(self):
         dest = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(threads=2)
         with cctx.stream_writer(dest) as compressor:
             compressor.write(b'a' * 1048576)
             compressor.write(b'b' * 1048576)
             compressor.write(b'c' * 1048576)
 
         self.assertEqual(len(dest.getvalue()), 295)
 
     def test_tell(self):
         dest = io.BytesIO()
         cctx = zstd.ZstdCompressor()
         with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.tell(), 0)
 
             for i in range(256):
                 compressor.write(b'foo' * (i + 1))
                 self.assertEqual(compressor.tell(), dest.tell())
 
     def test_bad_size(self):
         cctx = zstd.ZstdCompressor()
 
         dest = io.BytesIO()
 
         with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
             with cctx.stream_writer(dest, size=2) as compressor:
                 compressor.write(b'foo')
 
         # Test another operation.
         with cctx.stream_writer(dest, size=42):
             pass
 
     def test_tarfile_compat(self):
         dest = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor()
         with cctx.stream_writer(dest) as compressor:
             with tarfile.open('tf', mode='w|', fileobj=compressor) as tf:
                 tf.add(__file__, 'test_compressor.py')
 
         dest = io.BytesIO(dest.getvalue())
 
         dctx = zstd.ZstdDecompressor()
         with dctx.stream_reader(dest) as reader:
             with tarfile.open(mode='r|', fileobj=reader) as tf:
                 for member in tf:
                     self.assertEqual(member.name, 'test_compressor.py')
 
 
 @make_cffi
 class TestCompressor_read_to_iter(unittest.TestCase):
     def test_type_validation(self):
         cctx = zstd.ZstdCompressor()
 
         # Object with read() works.
         for chunk in cctx.read_to_iter(io.BytesIO()):
             pass
 
         # Buffer protocol works.
         for chunk in cctx.read_to_iter(b'foobar'):
             pass
 
         with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'):
             for chunk in cctx.read_to_iter(True):
                 pass
 
     def test_read_empty(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
 
         source = io.BytesIO()
         it = cctx.read_to_iter(source)
         chunks = list(it)
         self.assertEqual(len(chunks), 1)
         compressed = b''.join(chunks)
         self.assertEqual(compressed, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
 
         # And again with the buffer protocol.
         it = cctx.read_to_iter(b'')
         chunks = list(it)
         self.assertEqual(len(chunks), 1)
         compressed2 = b''.join(chunks)
         self.assertEqual(compressed2, compressed)
 
     def test_read_large(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
 
         source = io.BytesIO()
         source.write(b'f' * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE)
         source.write(b'o')
         source.seek(0)
 
         # Creating an iterator should not perform any compression until
         # first read.
         it = cctx.read_to_iter(source, size=len(source.getvalue()))
         self.assertEqual(source.tell(), 0)
 
         # We should have exactly 2 output chunks.
         chunks = []
         chunk = next(it)
         self.assertIsNotNone(chunk)
         self.assertEqual(source.tell(), zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE)
         chunks.append(chunk)
         chunk = next(it)
         self.assertIsNotNone(chunk)
         chunks.append(chunk)
 
         self.assertEqual(source.tell(), len(source.getvalue()))
 
         with self.assertRaises(StopIteration):
             next(it)
 
         # And again for good measure.
         with self.assertRaises(StopIteration):
             next(it)
 
         # We should get the same output as the one-shot compression mechanism.
         self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue()))
 
         params = zstd.get_frame_parameters(b''.join(chunks))
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Now check the buffer protocol.
         it = cctx.read_to_iter(source.getvalue())
         chunks = list(it)
         self.assertEqual(len(chunks), 2)
 
         params = zstd.get_frame_parameters(b''.join(chunks))
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         #self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue()))
 
     def test_read_write_size(self):
         source = OpCountingBytesIO(b'foobarfoobar')
         cctx = zstd.ZstdCompressor(level=3)
         for chunk in cctx.read_to_iter(source, read_size=1, write_size=1):
             self.assertEqual(len(chunk), 1)
 
         self.assertEqual(source._read_count, len(source.getvalue()) + 1)
 
     def test_multithreaded(self):
         source = io.BytesIO()
         source.write(b'a' * 1048576)
         source.write(b'b' * 1048576)
         source.write(b'c' * 1048576)
         source.seek(0)
 
         cctx = zstd.ZstdCompressor(threads=2)
 
         compressed = b''.join(cctx.read_to_iter(source))
         self.assertEqual(len(compressed), 295)
 
     def test_bad_size(self):
         cctx = zstd.ZstdCompressor()
 
         source = io.BytesIO(b'a' * 42)
 
         with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
             b''.join(cctx.read_to_iter(source, size=2))
 
         # Test another operation on errored compressor.
         b''.join(cctx.read_to_iter(source))
 
 
 @make_cffi
 class TestCompressor_chunker(unittest.TestCase):
     def test_empty(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         chunker = cctx.chunker()
 
         it = chunker.compress(b'')
 
         with self.assertRaises(StopIteration):
             next(it)
 
         it = chunker.finish()
 
         self.assertEqual(next(it), b'\x28\xb5\x2f\xfd\x00\x58\x01\x00\x00')
 
         with self.assertRaises(StopIteration):
             next(it)
 
     def test_simple_input(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         it = chunker.compress(b'foobar')
 
         with self.assertRaises(StopIteration):
             next(it)
 
         it = chunker.compress(b'baz' * 30)
 
         with self.assertRaises(StopIteration):
             next(it)
 
         it = chunker.finish()
 
         self.assertEqual(next(it),
                          b'\x28\xb5\x2f\xfd\x00\x58\x7d\x00\x00\x48\x66\x6f'
                          b'\x6f\x62\x61\x72\x62\x61\x7a\x01\x00\xe4\xe4\x8e')
 
         with self.assertRaises(StopIteration):
             next(it)
 
     def test_input_size(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker(size=1024)
 
         it = chunker.compress(b'x' * 1000)
 
         with self.assertRaises(StopIteration):
             next(it)
 
         it = chunker.compress(b'y' * 24)
 
         with self.assertRaises(StopIteration):
             next(it)
 
         chunks = list(chunker.finish())
 
         self.assertEqual(chunks, [
             b'\x28\xb5\x2f\xfd\x60\x00\x03\x65\x00\x00\x18\x78\x78\x79\x02\x00'
             b'\xa0\x16\xe3\x2b\x80\x05'
         ])
 
         dctx = zstd.ZstdDecompressor()
 
         self.assertEqual(dctx.decompress(b''.join(chunks)),
                          (b'x' * 1000) + (b'y' * 24))
 
     def test_small_chunk_size(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker(chunk_size=1)
 
         chunks = list(chunker.compress(b'foo' * 1024))
         self.assertEqual(chunks, [])
 
         chunks = list(chunker.finish())
         self.assertTrue(all(len(chunk) == 1 for chunk in chunks))
 
         self.assertEqual(
             b''.join(chunks),
             b'\x28\xb5\x2f\xfd\x00\x58\x55\x00\x00\x18\x66\x6f\x6f\x01\x00'
             b'\xfa\xd3\x77\x43')
 
         dctx = zstd.ZstdDecompressor()
         self.assertEqual(dctx.decompress(b''.join(chunks),
                                          max_output_size=10000),
                          b'foo' * 1024)
 
     def test_input_types(self):
         cctx = zstd.ZstdCompressor()
 
         mutable_array = bytearray(3)
         mutable_array[:] = b'foo'
 
         sources = [
             memoryview(b'foo'),
             bytearray(b'foo'),
             mutable_array,
         ]
 
         for source in sources:
             chunker = cctx.chunker()
 
             self.assertEqual(list(chunker.compress(source)), [])
             self.assertEqual(list(chunker.finish()), [
                 b'\x28\xb5\x2f\xfd\x00\x58\x19\x00\x00\x66\x6f\x6f'
             ])
 
     def test_flush(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         self.assertEqual(list(chunker.compress(b'foo' * 1024)), [])
         self.assertEqual(list(chunker.compress(b'bar' * 1024)), [])
 
         chunks1 = list(chunker.flush())
 
         self.assertEqual(chunks1, [
             b'\x28\xb5\x2f\xfd\x00\x58\x8c\x00\x00\x30\x66\x6f\x6f\x62\x61\x72'
             b'\x02\x00\xfa\x03\xfe\xd0\x9f\xbe\x1b\x02'
         ])
 
         self.assertEqual(list(chunker.flush()), [])
         self.assertEqual(list(chunker.flush()), [])
 
         self.assertEqual(list(chunker.compress(b'baz' * 1024)), [])
 
         chunks2 = list(chunker.flush())
         self.assertEqual(len(chunks2), 1)
 
         chunks3 = list(chunker.finish())
         self.assertEqual(len(chunks2), 1)
 
         dctx = zstd.ZstdDecompressor()
 
         self.assertEqual(dctx.decompress(b''.join(chunks1 + chunks2 + chunks3),
                                          max_output_size=10000),
                          (b'foo' * 1024) + (b'bar' * 1024) + (b'baz' * 1024))
 
     def test_compress_after_finish(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         list(chunker.compress(b'foo'))
         list(chunker.finish())
 
         with self.assertRaisesRegexp(
                 zstd.ZstdError,
                 r'cannot call compress\(\) after compression finished'):
             list(chunker.compress(b'foo'))
 
     def test_flush_after_finish(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         list(chunker.compress(b'foo'))
         list(chunker.finish())
 
         with self.assertRaisesRegexp(
                 zstd.ZstdError,
                 r'cannot call flush\(\) after compression finished'):
             list(chunker.flush())
 
     def test_finish_after_finish(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         list(chunker.compress(b'foo'))
         list(chunker.finish())
 
         with self.assertRaisesRegexp(
                 zstd.ZstdError,
                 r'cannot call finish\(\) after compression finished'):
             list(chunker.finish())
 
 
 class TestCompressor_multi_compress_to_buffer(unittest.TestCase):
     def test_invalid_inputs(self):
         cctx = zstd.ZstdCompressor()
 
         if not hasattr(cctx, 'multi_compress_to_buffer'):
             self.skipTest('multi_compress_to_buffer not available')
 
         with self.assertRaises(TypeError):
             cctx.multi_compress_to_buffer(True)
 
         with self.assertRaises(TypeError):
             cctx.multi_compress_to_buffer((1, 2))
 
         with self.assertRaisesRegexp(TypeError, 'item 0 not a bytes like object'):
             cctx.multi_compress_to_buffer([u'foo'])
 
     def test_empty_input(self):
         cctx = zstd.ZstdCompressor()
 
         if not hasattr(cctx, 'multi_compress_to_buffer'):
             self.skipTest('multi_compress_to_buffer not available')
 
         with self.assertRaisesRegexp(ValueError, 'no source elements found'):
             cctx.multi_compress_to_buffer([])
 
         with self.assertRaisesRegexp(ValueError, 'source elements are empty'):
             cctx.multi_compress_to_buffer([b'', b'', b''])
 
     def test_list_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
         if not hasattr(cctx, 'multi_compress_to_buffer'):
             self.skipTest('multi_compress_to_buffer not available')
 
         original = [b'foo' * 12, b'bar' * 6]
         frames = [cctx.compress(c) for c in original]
         b = cctx.multi_compress_to_buffer(original)
 
         self.assertIsInstance(b, zstd.BufferWithSegmentsCollection)
 
         self.assertEqual(len(b), 2)
         self.assertEqual(b.size(), 44)
 
         self.assertEqual(b[0].tobytes(), frames[0])
         self.assertEqual(b[1].tobytes(), frames[1])
 
     def test_buffer_with_segments_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
         if not hasattr(cctx, 'multi_compress_to_buffer'):
             self.skipTest('multi_compress_to_buffer not available')
 
         original = [b'foo' * 4, b'bar' * 6]
         frames = [cctx.compress(c) for c in original]
 
         offsets = struct.pack('=QQQQ', 0, len(original[0]),
                                        len(original[0]), len(original[1]))
         segments = zstd.BufferWithSegments(b''.join(original), offsets)
 
         result = cctx.multi_compress_to_buffer(segments)
 
         self.assertEqual(len(result), 2)
         self.assertEqual(result.size(), 47)
 
         self.assertEqual(result[0].tobytes(), frames[0])
         self.assertEqual(result[1].tobytes(), frames[1])
 
     def test_buffer_with_segments_collection_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
         if not hasattr(cctx, 'multi_compress_to_buffer'):
             self.skipTest('multi_compress_to_buffer not available')
 
         original = [
             b'foo1',
             b'foo2' * 2,
             b'foo3' * 3,
             b'foo4' * 4,
             b'foo5' * 5,
         ]
 
         frames = [cctx.compress(c) for c in original]
 
         b = b''.join([original[0], original[1]])
         b1 = zstd.BufferWithSegments(b, struct.pack('=QQQQ',
                                                     0, len(original[0]),
                                                     len(original[0]), len(original[1])))
         b = b''.join([original[2], original[3], original[4]])
         b2 = zstd.BufferWithSegments(b, struct.pack('=QQQQQQ',
                                                     0, len(original[2]),
                                                     len(original[2]), len(original[3]),
                                                     len(original[2]) + len(original[3]), len(original[4])))
 
         c = zstd.BufferWithSegmentsCollection(b1, b2)
 
         result = cctx.multi_compress_to_buffer(c)
 
         self.assertEqual(len(result), len(frames))
 
         for i, frame in enumerate(frames):
             self.assertEqual(result[i].tobytes(), frame)
 
     def test_multiple_threads(self):
         # threads argument will cause multi-threaded ZSTD APIs to be used, which will
         # make output different.
         refcctx = zstd.ZstdCompressor(write_checksum=True)
         reference = [refcctx.compress(b'x' * 64), refcctx.compress(b'y' * 64)]
 
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
         if not hasattr(cctx, 'multi_compress_to_buffer'):
             self.skipTest('multi_compress_to_buffer not available')
 
         frames = []
         frames.extend(b'x' * 64 for i in range(256))
         frames.extend(b'y' * 64 for i in range(256))
 
         result = cctx.multi_compress_to_buffer(frames, threads=-1)
 
         self.assertEqual(len(result), 512)
         for i in range(512):
             if i < 256:
                 self.assertEqual(result[i].tobytes(), reference[0])
             else:
                 self.assertEqual(result[i].tobytes(), reference[1])

contrib/python-zstandard/tests/test_data_structures.py

@@ -1,228 +1,228 b''
 import sys
 import unittest
 
 import zstandard as zstd
 
 from . common import (
     make_cffi,
 )
 
 
 @make_cffi
 class TestCompressionParameters(unittest.TestCase):
     def test_bounds(self):
         zstd.ZstdCompressionParameters(window_log=zstd.WINDOWLOG_MIN,
                                        chain_log=zstd.CHAINLOG_MIN,
                                        hash_log=zstd.HASHLOG_MIN,
                                        search_log=zstd.SEARCHLOG_MIN,
                                        min_match=zstd.MINMATCH_MIN + 1,
                                        target_length=zstd.TARGETLENGTH_MIN,
                                        strategy=zstd.STRATEGY_FAST)
 
         zstd.ZstdCompressionParameters(window_log=zstd.WINDOWLOG_MAX,
                                        chain_log=zstd.CHAINLOG_MAX,
                                        hash_log=zstd.HASHLOG_MAX,
                                        search_log=zstd.SEARCHLOG_MAX,
                                        min_match=zstd.MINMATCH_MAX - 1,
                                        target_length=zstd.TARGETLENGTH_MAX,
                                        strategy=zstd.STRATEGY_BTULTRA2)
 
     def test_from_level(self):
         p = zstd.ZstdCompressionParameters.from_level(1)
         self.assertIsInstance(p, zstd.CompressionParameters)
 
         self.assertEqual(p.window_log, 19)
 
         p = zstd.ZstdCompressionParameters.from_level(-4)
         self.assertEqual(p.window_log, 19)
 
     def test_members(self):
         p = zstd.ZstdCompressionParameters(window_log=10,
                                            chain_log=6,
                                            hash_log=7,
                                            search_log=4,
                                            min_match=5,
                                            target_length=8,
                                            strategy=1)
         self.assertEqual(p.window_log, 10)
         self.assertEqual(p.chain_log, 6)
         self.assertEqual(p.hash_log, 7)
         self.assertEqual(p.search_log, 4)
         self.assertEqual(p.min_match, 5)
         self.assertEqual(p.target_length, 8)
         self.assertEqual(p.compression_strategy, 1)
 
         p = zstd.ZstdCompressionParameters(compression_level=2)
         self.assertEqual(p.compression_level, 2)
 
         p = zstd.ZstdCompressionParameters(threads=4)
         self.assertEqual(p.threads, 4)
 
         p = zstd.ZstdCompressionParameters(threads=2, job_size=1048576,
                                            overlap_log=6)
         self.assertEqual(p.threads, 2)
         self.assertEqual(p.job_size, 1048576)
         self.assertEqual(p.overlap_log, 6)
         self.assertEqual(p.overlap_size_log, 6)
 
         p = zstd.ZstdCompressionParameters(compression_level=-1)
         self.assertEqual(p.compression_level, -1)
 
         p = zstd.ZstdCompressionParameters(compression_level=-2)
         self.assertEqual(p.compression_level, -2)
 
         p = zstd.ZstdCompressionParameters(force_max_window=True)
         self.assertEqual(p.force_max_window, 1)
 
         p = zstd.ZstdCompressionParameters(enable_ldm=True)
         self.assertEqual(p.enable_ldm, 1)
 
         p = zstd.ZstdCompressionParameters(ldm_hash_log=7)
         self.assertEqual(p.ldm_hash_log, 7)
 
         p = zstd.ZstdCompressionParameters(ldm_min_match=6)
         self.assertEqual(p.ldm_min_match, 6)
 
         p = zstd.ZstdCompressionParameters(ldm_bucket_size_log=7)
         self.assertEqual(p.ldm_bucket_size_log, 7)
 
         p = zstd.ZstdCompressionParameters(ldm_hash_rate_log=8)
         self.assertEqual(p.ldm_hash_every_log, 8)
         self.assertEqual(p.ldm_hash_rate_log, 8)
 
     def test_estimated_compression_context_size(self):
         p = zstd.ZstdCompressionParameters(window_log=20,
                                            chain_log=16,
                                            hash_log=17,
                                            search_log=1,
                                            min_match=5,
                                            target_length=16,
                                            strategy=zstd.STRATEGY_DFAST)
 
         # 32-bit has slightly different values from 64-bit.
-        self.assertAlmostEqual(p.estimated_compression_context_size(), 1294072,
+        self.assertAlmostEqual(p.estimated_compression_context_size(), 1294144,
                                delta=250)
 
     def test_strategy(self):
         with self.assertRaisesRegexp(ValueError, 'cannot specify both compression_strategy'):
             zstd.ZstdCompressionParameters(strategy=0, compression_strategy=0)
 
         p = zstd.ZstdCompressionParameters(strategy=2)
         self.assertEqual(p.compression_strategy, 2)
 
         p = zstd.ZstdCompressionParameters(strategy=3)
         self.assertEqual(p.compression_strategy, 3)
 
     def test_ldm_hash_rate_log(self):
         with self.assertRaisesRegexp(ValueError, 'cannot specify both ldm_hash_rate_log'):
             zstd.ZstdCompressionParameters(ldm_hash_rate_log=8, ldm_hash_every_log=4)
 
         p = zstd.ZstdCompressionParameters(ldm_hash_rate_log=8)
         self.assertEqual(p.ldm_hash_every_log, 8)
 
         p = zstd.ZstdCompressionParameters(ldm_hash_every_log=16)
         self.assertEqual(p.ldm_hash_every_log, 16)
 
     def test_overlap_log(self):
         with self.assertRaisesRegexp(ValueError, 'cannot specify both overlap_log'):
             zstd.ZstdCompressionParameters(overlap_log=1, overlap_size_log=9)
 
         p = zstd.ZstdCompressionParameters(overlap_log=2)
         self.assertEqual(p.overlap_log, 2)
         self.assertEqual(p.overlap_size_log, 2)
 
         p = zstd.ZstdCompressionParameters(overlap_size_log=4)
         self.assertEqual(p.overlap_log, 4)
         self.assertEqual(p.overlap_size_log, 4)
 
 
 @make_cffi
 class TestFrameParameters(unittest.TestCase):
     def test_invalid_type(self):
         with self.assertRaises(TypeError):
             zstd.get_frame_parameters(None)
 
         # Python 3 doesn't appear to convert unicode to Py_buffer.
         if sys.version_info[0] >= 3:
             with self.assertRaises(TypeError):
                 zstd.get_frame_parameters(u'foobarbaz')
         else:
             # CPython will convert unicode to Py_buffer. But CFFI won't.
             if zstd.backend == 'cffi':
                 with self.assertRaises(TypeError):
                     zstd.get_frame_parameters(u'foobarbaz')
             else:
                 with self.assertRaises(zstd.ZstdError):
                     zstd.get_frame_parameters(u'foobarbaz')
 
     def test_invalid_input_sizes(self):
         with self.assertRaisesRegexp(zstd.ZstdError, 'not enough data for frame'):
             zstd.get_frame_parameters(b'')
 
         with self.assertRaisesRegexp(zstd.ZstdError, 'not enough data for frame'):
             zstd.get_frame_parameters(zstd.FRAME_HEADER)
 
     def test_invalid_frame(self):
         with self.assertRaisesRegexp(zstd.ZstdError, 'Unknown frame descriptor'):
             zstd.get_frame_parameters(b'foobarbaz')
 
     def test_attributes(self):
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x00\x00')
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Lowest 2 bits indicate a dictionary and length. Here, the dict id is 1 byte.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x01\x00\xff')
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 255)
         self.assertFalse(params.has_checksum)
 
         # Lowest 3rd bit indicates if checksum is present.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x04\x00')
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
         self.assertTrue(params.has_checksum)
 
         # Upper 2 bits indicate content size.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x40\x00\xff\x00')
         self.assertEqual(params.content_size, 511)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Window descriptor is 2nd byte after frame header.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x00\x40')
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Set multiple things.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x45\x40\x0f\x10\x00')
         self.assertEqual(params.content_size, 272)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 15)
         self.assertTrue(params.has_checksum)
 
     def test_input_types(self):
         v = zstd.FRAME_HEADER + b'\x00\x00'
 
         mutable_array = bytearray(len(v))
         mutable_array[:] = v
 
         sources = [
             memoryview(v),
             bytearray(v),
             mutable_array,
         ]
 
         for source in sources:
             params = zstd.get_frame_parameters(source)
             self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
             self.assertEqual(params.window_size, 1024)
             self.assertEqual(params.dict_id, 0)
             self.assertFalse(params.has_checksum)

contrib/python-zstandard/tests/test_module_attributes.py

@@ -1,69 +1,69 b''
 from __future__ import unicode_literals
 
 import unittest
 
 import zstandard as zstd
 
 from . common import (
     make_cffi,
 )
 
 
 @make_cffi
 class TestModuleAttributes(unittest.TestCase):
     def test_version(self):
-        self.assertEqual(zstd.ZSTD_VERSION, (1, 3, 8))
+        self.assertEqual(zstd.ZSTD_VERSION, (1, 4, 3))
 
-        self.assertEqual(zstd.__version__, '0.11.0')
+        self.assertEqual(zstd.__version__, '0.12.0')
 
     def test_constants(self):
         self.assertEqual(zstd.MAX_COMPRESSION_LEVEL, 22)
         self.assertEqual(zstd.FRAME_HEADER, b'\x28\xb5\x2f\xfd')
 
     def test_hasattr(self):
         attrs = (
             'CONTENTSIZE_UNKNOWN',
             'CONTENTSIZE_ERROR',
             'COMPRESSION_RECOMMENDED_INPUT_SIZE',
             'COMPRESSION_RECOMMENDED_OUTPUT_SIZE',
             'DECOMPRESSION_RECOMMENDED_INPUT_SIZE',
             'DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE',
             'MAGIC_NUMBER',
             'FLUSH_BLOCK',
             'FLUSH_FRAME',
             'BLOCKSIZELOG_MAX',
             'BLOCKSIZE_MAX',
             'WINDOWLOG_MIN',
             'WINDOWLOG_MAX',
             'CHAINLOG_MIN',
             'CHAINLOG_MAX',
             'HASHLOG_MIN',
             'HASHLOG_MAX',
             'HASHLOG3_MAX',
             'MINMATCH_MIN',
             'MINMATCH_MAX',
             'SEARCHLOG_MIN',
             'SEARCHLOG_MAX',
             'SEARCHLENGTH_MIN',
             'SEARCHLENGTH_MAX',
             'TARGETLENGTH_MIN',
             'TARGETLENGTH_MAX',
             'LDM_MINMATCH_MIN',
             'LDM_MINMATCH_MAX',
             'LDM_BUCKETSIZELOG_MAX',
             'STRATEGY_FAST',
             'STRATEGY_DFAST',
             'STRATEGY_GREEDY',
             'STRATEGY_LAZY',
             'STRATEGY_LAZY2',
             'STRATEGY_BTLAZY2',
             'STRATEGY_BTOPT',
             'STRATEGY_BTULTRA',
             'STRATEGY_BTULTRA2',
             'DICT_TYPE_AUTO',
             'DICT_TYPE_RAWCONTENT',
             'DICT_TYPE_FULLDICT',
         )
 
         for a in attrs:
             self.assertTrue(hasattr(zstd, a), a)

contrib/python-zstandard/tests/test_train_dictionary.py

@@ -1,88 +1,89 b''
 import struct
 import sys
 import unittest
 
 import zstandard as zstd
 
 from . common import (
     generate_samples,
     make_cffi,
+    random_input_data,
 )
 
 if sys.version_info[0] >= 3:
     int_type = int
 else:
     int_type = long
 
 
 @make_cffi
 class TestTrainDictionary(unittest.TestCase):
     def test_no_args(self):
         with self.assertRaises(TypeError):
             zstd.train_dictionary()
 
     def test_bad_args(self):
         with self.assertRaises(TypeError):
             zstd.train_dictionary(8192, u'foo')
 
         with self.assertRaises(ValueError):
             zstd.train_dictionary(8192, [u'foo'])
 
     def test_no_params(self):
-        d = zstd.train_dictionary(8192, generate_samples())
+        d = zstd.train_dictionary(8192, random_input_data())
         self.assertIsInstance(d.dict_id(), int_type)
 
         # The dictionary ID may be different across platforms.
         expected = b'\x37\xa4\x30\xec' + struct.pack('<I', d.dict_id())
 
         data = d.as_bytes()
         self.assertEqual(data[0:8], expected)
 
     def test_basic(self):
         d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
         self.assertIsInstance(d.dict_id(), int_type)
 
         data = d.as_bytes()
         self.assertEqual(data[0:4], b'\x37\xa4\x30\xec')
 
         self.assertEqual(d.k, 64)
         self.assertEqual(d.d, 16)
 
     def test_set_dict_id(self):
         d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16,
                                   dict_id=42)
         self.assertEqual(d.dict_id(), 42)
 
     def test_optimize(self):
         d = zstd.train_dictionary(8192, generate_samples(), threads=-1, steps=1,
                                   d=16)
 
         # This varies by platform.
         self.assertIn(d.k, (50, 2000))
         self.assertEqual(d.d, 16)
 
 @make_cffi
 class TestCompressionDict(unittest.TestCase):
     def test_bad_mode(self):
         with self.assertRaisesRegexp(ValueError, 'invalid dictionary load mode'):
             zstd.ZstdCompressionDict(b'foo', dict_type=42)
 
     def test_bad_precompute_compress(self):
         d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
 
         with self.assertRaisesRegexp(ValueError, 'must specify one of level or '):
             d.precompute_compress()
 
         with self.assertRaisesRegexp(ValueError, 'must only specify one of level or '):
             d.precompute_compress(level=3,
                                   compression_params=zstd.CompressionParameters())
 
     def test_precompute_compress_rawcontent(self):
         d = zstd.ZstdCompressionDict(b'dictcontent' * 64,
                                      dict_type=zstd.DICT_TYPE_RAWCONTENT)
         d.precompute_compress(level=1)
 
         d = zstd.ZstdCompressionDict(b'dictcontent' * 64,
                                      dict_type=zstd.DICT_TYPE_FULLDICT)
         with self.assertRaisesRegexp(zstd.ZstdError, 'unable to precompute dictionary'):
             d.precompute_compress(level=1)

contrib/python-zstandard/zstandard/__init__.py

@@ -1,65 +1,65 b''
 # Copyright (c) 2017-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 """Python interface to the Zstandard (zstd) compression library."""
 
 from __future__ import absolute_import, unicode_literals
 
 # This module serves 2 roles:
 #
 # 1) Export the C or CFFI "backend" through a central module.
 # 2) Implement additional functionality built on top of C or CFFI backend.
 
 import os
 import platform
 
 # Some Python implementations don't support C extensions. That's why we have
 # a CFFI implementation in the first place. The code here import one of our
 # "backends" then re-exports the symbols from this module. For convenience,
 # we support falling back to the CFFI backend if the C extension can't be
 # imported. But for performance reasons, we only do this on unknown Python
 # implementation. Notably, for CPython we require the C extension by default.
 # Because someone will inevitably want special behavior, the behavior is
 # configurable via an environment variable. A potentially better way to handle
 # this is to import a special ``__importpolicy__`` module or something
 # defining a variable and `setup.py` could write the file with whatever
 # policy was specified at build time. Until someone needs it, we go with
 # the hacky but simple environment variable approach.
 _module_policy = os.environ.get('PYTHON_ZSTANDARD_IMPORT_POLICY', 'default')
 
 if _module_policy == 'default':
     if platform.python_implementation() in ('CPython',):
         from zstd import *
         backend = 'cext'
     elif platform.python_implementation() in ('PyPy',):
         from .cffi import *
         backend = 'cffi'
     else:
         try:
             from zstd import *
             backend = 'cext'
         except ImportError:
             from .cffi import *
             backend = 'cffi'
 elif _module_policy == 'cffi_fallback':
     try:
         from zstd import *
         backend = 'cext'
     except ImportError:
         from .cffi import *
         backend = 'cffi'
 elif _module_policy == 'cext':
     from zstd import *
     backend = 'cext'
 elif _module_policy == 'cffi':
     from .cffi import *
     backend = 'cffi'
 else:
     raise ImportError('unknown module import policy: %s; use default, cffi_fallback, '
                       'cext, or cffi' % _module_policy)
 
 # Keep this in sync with python-zstandard.h.
-__version__ = '0.11.0'
+__version__ = '0.12.0'

contrib/python-zstandard/zstandard/cffi.py

@@ -1,2515 +1,2515 b''
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 """Python interface to the Zstandard (zstd) compression library."""
 
 from __future__ import absolute_import, unicode_literals
 
 # This should match what the C extension exports.
 __all__ = [
     #'BufferSegment',
     #'BufferSegments',
     #'BufferWithSegments',
     #'BufferWithSegmentsCollection',
     'CompressionParameters',
     'ZstdCompressionDict',
     'ZstdCompressionParameters',
     'ZstdCompressor',
     'ZstdError',
     'ZstdDecompressor',
     'FrameParameters',
     'estimate_decompression_context_size',
     'frame_content_size',
     'frame_header_size',
     'get_frame_parameters',
     'train_dictionary',
 
     # Constants.
     'FLUSH_BLOCK',
     'FLUSH_FRAME',
     'COMPRESSOBJ_FLUSH_FINISH',
     'COMPRESSOBJ_FLUSH_BLOCK',
     'ZSTD_VERSION',
     'FRAME_HEADER',
     'CONTENTSIZE_UNKNOWN',
     'CONTENTSIZE_ERROR',
     'MAX_COMPRESSION_LEVEL',
     'COMPRESSION_RECOMMENDED_INPUT_SIZE',
     'COMPRESSION_RECOMMENDED_OUTPUT_SIZE',
     'DECOMPRESSION_RECOMMENDED_INPUT_SIZE',
     'DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE',
     'MAGIC_NUMBER',
     'BLOCKSIZELOG_MAX',
     'BLOCKSIZE_MAX',
     'WINDOWLOG_MIN',
     'WINDOWLOG_MAX',
     'CHAINLOG_MIN',
     'CHAINLOG_MAX',
     'HASHLOG_MIN',
     'HASHLOG_MAX',
     'HASHLOG3_MAX',
     'MINMATCH_MIN',
     'MINMATCH_MAX',
     'SEARCHLOG_MIN',
     'SEARCHLOG_MAX',
     'SEARCHLENGTH_MIN',
     'SEARCHLENGTH_MAX',
     'TARGETLENGTH_MIN',
     'TARGETLENGTH_MAX',
     'LDM_MINMATCH_MIN',
     'LDM_MINMATCH_MAX',
     'LDM_BUCKETSIZELOG_MAX',
     'STRATEGY_FAST',
     'STRATEGY_DFAST',
     'STRATEGY_GREEDY',
     'STRATEGY_LAZY',
     'STRATEGY_LAZY2',
     'STRATEGY_BTLAZY2',
     'STRATEGY_BTOPT',
     'STRATEGY_BTULTRA',
     'STRATEGY_BTULTRA2',
     'DICT_TYPE_AUTO',
     'DICT_TYPE_RAWCONTENT',
     'DICT_TYPE_FULLDICT',
     'FORMAT_ZSTD1',
     'FORMAT_ZSTD1_MAGICLESS',
 ]
 
 import io
 import os
 import sys
 
 from _zstd_cffi import (
     ffi,
     lib,
 )
 
 if sys.version_info[0] == 2:
     bytes_type = str
     int_type = long
 else:
     bytes_type = bytes
     int_type = int
 
 
 COMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_CStreamInSize()
 COMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_CStreamOutSize()
 DECOMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_DStreamInSize()
 DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_DStreamOutSize()
 
 new_nonzero = ffi.new_allocator(should_clear_after_alloc=False)
 
 
 MAX_COMPRESSION_LEVEL = lib.ZSTD_maxCLevel()
 MAGIC_NUMBER = lib.ZSTD_MAGICNUMBER
 FRAME_HEADER = b'\x28\xb5\x2f\xfd'
 CONTENTSIZE_UNKNOWN = lib.ZSTD_CONTENTSIZE_UNKNOWN
 CONTENTSIZE_ERROR = lib.ZSTD_CONTENTSIZE_ERROR
 ZSTD_VERSION = (lib.ZSTD_VERSION_MAJOR, lib.ZSTD_VERSION_MINOR, lib.ZSTD_VERSION_RELEASE)
 
 BLOCKSIZELOG_MAX = lib.ZSTD_BLOCKSIZELOG_MAX
 BLOCKSIZE_MAX = lib.ZSTD_BLOCKSIZE_MAX
 WINDOWLOG_MIN = lib.ZSTD_WINDOWLOG_MIN
 WINDOWLOG_MAX = lib.ZSTD_WINDOWLOG_MAX
 CHAINLOG_MIN = lib.ZSTD_CHAINLOG_MIN
 CHAINLOG_MAX = lib.ZSTD_CHAINLOG_MAX
 HASHLOG_MIN = lib.ZSTD_HASHLOG_MIN
 HASHLOG_MAX = lib.ZSTD_HASHLOG_MAX
 HASHLOG3_MAX = lib.ZSTD_HASHLOG3_MAX
 MINMATCH_MIN = lib.ZSTD_MINMATCH_MIN
 MINMATCH_MAX = lib.ZSTD_MINMATCH_MAX
 SEARCHLOG_MIN = lib.ZSTD_SEARCHLOG_MIN
 SEARCHLOG_MAX = lib.ZSTD_SEARCHLOG_MAX
 SEARCHLENGTH_MIN = lib.ZSTD_MINMATCH_MIN
 SEARCHLENGTH_MAX = lib.ZSTD_MINMATCH_MAX
 TARGETLENGTH_MIN = lib.ZSTD_TARGETLENGTH_MIN
 TARGETLENGTH_MAX = lib.ZSTD_TARGETLENGTH_MAX
 LDM_MINMATCH_MIN = lib.ZSTD_LDM_MINMATCH_MIN
 LDM_MINMATCH_MAX = lib.ZSTD_LDM_MINMATCH_MAX
 LDM_BUCKETSIZELOG_MAX = lib.ZSTD_LDM_BUCKETSIZELOG_MAX
 
 STRATEGY_FAST = lib.ZSTD_fast
 STRATEGY_DFAST = lib.ZSTD_dfast
 STRATEGY_GREEDY = lib.ZSTD_greedy
 STRATEGY_LAZY = lib.ZSTD_lazy
 STRATEGY_LAZY2 = lib.ZSTD_lazy2
 STRATEGY_BTLAZY2 = lib.ZSTD_btlazy2
 STRATEGY_BTOPT = lib.ZSTD_btopt
 STRATEGY_BTULTRA = lib.ZSTD_btultra
 STRATEGY_BTULTRA2 = lib.ZSTD_btultra2
 
 DICT_TYPE_AUTO = lib.ZSTD_dct_auto
 DICT_TYPE_RAWCONTENT = lib.ZSTD_dct_rawContent
 DICT_TYPE_FULLDICT = lib.ZSTD_dct_fullDict
 
 FORMAT_ZSTD1 = lib.ZSTD_f_zstd1
 FORMAT_ZSTD1_MAGICLESS = lib.ZSTD_f_zstd1_magicless
 
 FLUSH_BLOCK = 0
 FLUSH_FRAME = 1
 
 COMPRESSOBJ_FLUSH_FINISH = 0
 COMPRESSOBJ_FLUSH_BLOCK = 1
 
 
 def _cpu_count():
     # os.cpu_count() was introducd in Python 3.4.
     try:
         return os.cpu_count() or 0
     except AttributeError:
         pass
 
     # Linux.
     try:
         if sys.version_info[0] == 2:
             return os.sysconf(b'SC_NPROCESSORS_ONLN')
         else:
             return os.sysconf(u'SC_NPROCESSORS_ONLN')
     except (AttributeError, ValueError):
         pass
 
     # TODO implement on other platforms.
     return 0
 
 
 class ZstdError(Exception):
     pass
 
 
 def _zstd_error(zresult):
     # Resolves to bytes on Python 2 and 3. We use the string for formatting
     # into error messages, which will be literal unicode. So convert it to
     # unicode.
     return ffi.string(lib.ZSTD_getErrorName(zresult)).decode('utf-8')
 
 def _make_cctx_params(params):
     res = lib.ZSTD_createCCtxParams()
     if res == ffi.NULL:
         raise MemoryError()
 
     res = ffi.gc(res, lib.ZSTD_freeCCtxParams)
 
     attrs = [
         (lib.ZSTD_c_format, params.format),
         (lib.ZSTD_c_compressionLevel, params.compression_level),
         (lib.ZSTD_c_windowLog, params.window_log),
         (lib.ZSTD_c_hashLog, params.hash_log),
         (lib.ZSTD_c_chainLog, params.chain_log),
         (lib.ZSTD_c_searchLog, params.search_log),
         (lib.ZSTD_c_minMatch, params.min_match),
         (lib.ZSTD_c_targetLength, params.target_length),
         (lib.ZSTD_c_strategy, params.compression_strategy),
         (lib.ZSTD_c_contentSizeFlag, params.write_content_size),
         (lib.ZSTD_c_checksumFlag, params.write_checksum),
         (lib.ZSTD_c_dictIDFlag, params.write_dict_id),
         (lib.ZSTD_c_nbWorkers, params.threads),
         (lib.ZSTD_c_jobSize, params.job_size),
         (lib.ZSTD_c_overlapLog, params.overlap_log),
         (lib.ZSTD_c_forceMaxWindow, params.force_max_window),
         (lib.ZSTD_c_enableLongDistanceMatching, params.enable_ldm),
         (lib.ZSTD_c_ldmHashLog, params.ldm_hash_log),
         (lib.ZSTD_c_ldmMinMatch, params.ldm_min_match),
         (lib.ZSTD_c_ldmBucketSizeLog, params.ldm_bucket_size_log),
         (lib.ZSTD_c_ldmHashRateLog, params.ldm_hash_rate_log),
     ]
 
     for param, value in attrs:
         _set_compression_parameter(res, param, value)
 
     return res
 
 class ZstdCompressionParameters(object):
     @staticmethod
     def from_level(level, source_size=0, dict_size=0, **kwargs):
         params = lib.ZSTD_getCParams(level, source_size, dict_size)
 
         args = {
             'window_log': 'windowLog',
             'chain_log': 'chainLog',
             'hash_log': 'hashLog',
             'search_log': 'searchLog',
             'min_match': 'minMatch',
             'target_length': 'targetLength',
             'compression_strategy': 'strategy',
         }
 
         for arg, attr in args.items():
             if arg not in kwargs:
                 kwargs[arg] = getattr(params, attr)
 
         return ZstdCompressionParameters(**kwargs)
 
     def __init__(self, format=0, compression_level=0, window_log=0, hash_log=0,
                  chain_log=0, search_log=0, min_match=0, target_length=0,
                  strategy=-1, compression_strategy=-1,
                  write_content_size=1, write_checksum=0,
                  write_dict_id=0, job_size=0, overlap_log=-1,
                  overlap_size_log=-1, force_max_window=0, enable_ldm=0,
                  ldm_hash_log=0, ldm_min_match=0, ldm_bucket_size_log=0,
                  ldm_hash_rate_log=-1, ldm_hash_every_log=-1, threads=0):
 
         params = lib.ZSTD_createCCtxParams()
         if params == ffi.NULL:
             raise MemoryError()
 
         params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
 
         self._params = params
 
         if threads < 0:
             threads = _cpu_count()
 
         # We need to set ZSTD_c_nbWorkers before ZSTD_c_jobSize and ZSTD_c_overlapLog
         # because setting ZSTD_c_nbWorkers resets the other parameters.
         _set_compression_parameter(params, lib.ZSTD_c_nbWorkers, threads)
 
         _set_compression_parameter(params, lib.ZSTD_c_format, format)
         _set_compression_parameter(params, lib.ZSTD_c_compressionLevel, compression_level)
         _set_compression_parameter(params, lib.ZSTD_c_windowLog, window_log)
         _set_compression_parameter(params, lib.ZSTD_c_hashLog, hash_log)
         _set_compression_parameter(params, lib.ZSTD_c_chainLog, chain_log)
         _set_compression_parameter(params, lib.ZSTD_c_searchLog, search_log)
         _set_compression_parameter(params, lib.ZSTD_c_minMatch, min_match)
         _set_compression_parameter(params, lib.ZSTD_c_targetLength, target_length)
 
         if strategy != -1 and compression_strategy != -1:
             raise ValueError('cannot specify both compression_strategy and strategy')
 
         if compression_strategy != -1:
             strategy = compression_strategy
         elif strategy == -1:
             strategy = 0
 
         _set_compression_parameter(params, lib.ZSTD_c_strategy, strategy)
         _set_compression_parameter(params, lib.ZSTD_c_contentSizeFlag, write_content_size)
         _set_compression_parameter(params, lib.ZSTD_c_checksumFlag, write_checksum)
         _set_compression_parameter(params, lib.ZSTD_c_dictIDFlag, write_dict_id)
         _set_compression_parameter(params, lib.ZSTD_c_jobSize, job_size)
 
         if overlap_log != -1 and overlap_size_log != -1:
             raise ValueError('cannot specify both overlap_log and overlap_size_log')
 
         if overlap_size_log != -1:
             overlap_log = overlap_size_log
         elif overlap_log == -1:
             overlap_log = 0
 
         _set_compression_parameter(params, lib.ZSTD_c_overlapLog, overlap_log)
         _set_compression_parameter(params, lib.ZSTD_c_forceMaxWindow, force_max_window)
         _set_compression_parameter(params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm)
         _set_compression_parameter(params, lib.ZSTD_c_ldmHashLog, ldm_hash_log)
         _set_compression_parameter(params, lib.ZSTD_c_ldmMinMatch, ldm_min_match)
         _set_compression_parameter(params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log)
 
         if ldm_hash_rate_log != -1 and ldm_hash_every_log != -1:
             raise ValueError('cannot specify both ldm_hash_rate_log and ldm_hash_every_log')
 
         if ldm_hash_every_log != -1:
             ldm_hash_rate_log = ldm_hash_every_log
         elif ldm_hash_rate_log == -1:
             ldm_hash_rate_log = 0
 
         _set_compression_parameter(params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log)
 
     @property
     def format(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_format)
 
     @property
     def compression_level(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_compressionLevel)
 
     @property
     def window_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_windowLog)
 
     @property
     def hash_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_hashLog)
 
     @property
     def chain_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_chainLog)
 
     @property
     def search_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_searchLog)
 
     @property
     def min_match(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_minMatch)
 
     @property
     def target_length(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_targetLength)
 
     @property
     def compression_strategy(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_strategy)
 
     @property
     def write_content_size(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_contentSizeFlag)
 
     @property
     def write_checksum(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_checksumFlag)
 
     @property
     def write_dict_id(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_dictIDFlag)
 
     @property
     def job_size(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_jobSize)
 
     @property
     def overlap_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_overlapLog)
 
     @property
     def overlap_size_log(self):
         return self.overlap_log
 
     @property
     def force_max_window(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_forceMaxWindow)
 
     @property
     def enable_ldm(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_enableLongDistanceMatching)
 
     @property
     def ldm_hash_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashLog)
 
     @property
     def ldm_min_match(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_ldmMinMatch)
 
     @property
     def ldm_bucket_size_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_ldmBucketSizeLog)
 
     @property
     def ldm_hash_rate_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashRateLog)
 
     @property
     def ldm_hash_every_log(self):
         return self.ldm_hash_rate_log
 
     @property
     def threads(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_nbWorkers)
 
     def estimated_compression_context_size(self):
         return lib.ZSTD_estimateCCtxSize_usingCCtxParams(self._params)
 
 CompressionParameters = ZstdCompressionParameters
 
 def estimate_decompression_context_size():
     return lib.ZSTD_estimateDCtxSize()
 
 
 def _set_compression_parameter(params, param, value):
-    zresult = lib.ZSTD_CCtxParam_setParameter(params, param, value)
+    zresult = lib.ZSTD_CCtxParams_setParameter(params, param, value)
     if lib.ZSTD_isError(zresult):
         raise ZstdError('unable to set compression context parameter: %s' %
                         _zstd_error(zresult))
 
 
 def _get_compression_parameter(params, param):
     result = ffi.new('int *')
 
-    zresult = lib.ZSTD_CCtxParam_getParameter(params, param, result)
+    zresult = lib.ZSTD_CCtxParams_getParameter(params, param, result)
     if lib.ZSTD_isError(zresult):
         raise ZstdError('unable to get compression context parameter: %s' %
                         _zstd_error(zresult))
 
     return result[0]
 
 
 class ZstdCompressionWriter(object):
     def __init__(self, compressor, writer, source_size, write_size,
                  write_return_read):
         self._compressor = compressor
         self._writer = writer
         self._write_size = write_size
         self._write_return_read = bool(write_return_read)
         self._entered = False
         self._closed = False
         self._bytes_compressed = 0
 
         self._dst_buffer = ffi.new('char[]', write_size)
         self._out_buffer = ffi.new('ZSTD_outBuffer *')
         self._out_buffer.dst = self._dst_buffer
         self._out_buffer.size = len(self._dst_buffer)
         self._out_buffer.pos = 0
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx,
                                                   source_size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
     def __enter__(self):
         if self._closed:
             raise ValueError('stream is closed')
 
         if self._entered:
             raise ZstdError('cannot __enter__ multiple times')
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
 
         if not exc_type and not exc_value and not exc_tb:
             self.close()
 
         self._compressor = None
 
         return False
 
     def memory_size(self):
         return lib.ZSTD_sizeof_CCtx(self._compressor._cctx)
 
     def fileno(self):
         f = getattr(self._writer, 'fileno', None)
         if f:
             return f()
         else:
             raise OSError('fileno not available on underlying writer')
 
     def close(self):
         if self._closed:
             return
 
         try:
             self.flush(FLUSH_FRAME)
         finally:
             self._closed = True
 
         # Call close() on underlying stream as well.
         f = getattr(self._writer, 'close', None)
         if f:
             f()
 
     @property
     def closed(self):
         return self._closed
 
     def isatty(self):
         return False
 
     def readable(self):
         return False
 
     def readline(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readlines(self, hint=-1):
         raise io.UnsupportedOperation()
 
     def seek(self, offset, whence=None):
         raise io.UnsupportedOperation()
 
     def seekable(self):
         return False
 
     def truncate(self, size=None):
         raise io.UnsupportedOperation()
 
     def writable(self):
         return True
 
     def writelines(self, lines):
         raise NotImplementedError('writelines() is not yet implemented')
 
     def read(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readall(self):
         raise io.UnsupportedOperation()
 
     def readinto(self, b):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         if self._closed:
             raise ValueError('stream is closed')
 
         total_write = 0
 
         data_buffer = ffi.from_buffer(data)
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         out_buffer = self._out_buffer
         out_buffer.pos = 0
 
         while in_buffer.pos < in_buffer.size:
             zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                                out_buffer, in_buffer,
                                                lib.ZSTD_e_continue)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' %
                                 _zstd_error(zresult))
 
             if out_buffer.pos:
                 self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
                 total_write += out_buffer.pos
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
 
         if self._write_return_read:
             return in_buffer.pos
         else:
             return total_write
 
     def flush(self, flush_mode=FLUSH_BLOCK):
         if flush_mode == FLUSH_BLOCK:
             flush = lib.ZSTD_e_flush
         elif flush_mode == FLUSH_FRAME:
             flush = lib.ZSTD_e_end
         else:
             raise ValueError('unknown flush_mode: %r' % flush_mode)
 
         if self._closed:
             raise ValueError('stream is closed')
 
         total_write = 0
 
         out_buffer = self._out_buffer
         out_buffer.pos = 0
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         while True:
             zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                                out_buffer, in_buffer,
                                                flush)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' %
                                 _zstd_error(zresult))
 
             if out_buffer.pos:
                 self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
                 total_write += out_buffer.pos
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
 
             if not zresult:
                 break
 
         return total_write
 
     def tell(self):
         return self._bytes_compressed
 
 
 class ZstdCompressionObj(object):
     def compress(self, data):
         if self._finished:
             raise ZstdError('cannot call compress() after compressor finished')
 
         data_buffer = ffi.from_buffer(data)
         source = ffi.new('ZSTD_inBuffer *')
         source.src = data_buffer
         source.size = len(data_buffer)
         source.pos = 0
 
         chunks = []
 
         while source.pos < len(data):
             zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                                self._out,
                                                source,
                                                lib.ZSTD_e_continue)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' %
                                 _zstd_error(zresult))
 
             if self._out.pos:
                 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
                 self._out.pos = 0
 
         return b''.join(chunks)
 
     def flush(self, flush_mode=COMPRESSOBJ_FLUSH_FINISH):
         if flush_mode not in (COMPRESSOBJ_FLUSH_FINISH, COMPRESSOBJ_FLUSH_BLOCK):
             raise ValueError('flush mode not recognized')
 
         if self._finished:
             raise ZstdError('compressor object already finished')
 
         if flush_mode == COMPRESSOBJ_FLUSH_BLOCK:
             z_flush_mode = lib.ZSTD_e_flush
         elif flush_mode == COMPRESSOBJ_FLUSH_FINISH:
             z_flush_mode = lib.ZSTD_e_end
             self._finished = True
         else:
             raise ZstdError('unhandled flush mode')
 
         assert self._out.pos == 0
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         chunks = []
 
         while True:
             zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                                self._out,
                                                in_buffer,
                                                z_flush_mode)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('error ending compression stream: %s' %
                                 _zstd_error(zresult))
 
             if self._out.pos:
                 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
                 self._out.pos = 0
 
             if not zresult:
                 break
 
         return b''.join(chunks)
 
 
 class ZstdCompressionChunker(object):
     def __init__(self, compressor, chunk_size):
         self._compressor = compressor
         self._out = ffi.new('ZSTD_outBuffer *')
         self._dst_buffer = ffi.new('char[]', chunk_size)
         self._out.dst = self._dst_buffer
         self._out.size = chunk_size
         self._out.pos = 0
 
         self._in = ffi.new('ZSTD_inBuffer *')
         self._in.src = ffi.NULL
         self._in.size = 0
         self._in.pos = 0
         self._finished = False
 
     def compress(self, data):
         if self._finished:
             raise ZstdError('cannot call compress() after compression finished')
 
         if self._in.src != ffi.NULL:
             raise ZstdError('cannot perform operation before consuming output '
                             'from previous operation')
 
         data_buffer = ffi.from_buffer(data)
 
         if not len(data_buffer):
             return
 
         self._in.src = data_buffer
         self._in.size = len(data_buffer)
         self._in.pos = 0
 
         while self._in.pos < self._in.size:
             zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                                self._out,
                                                self._in,
                                                lib.ZSTD_e_continue)
 
             if self._in.pos == self._in.size:
                 self._in.src = ffi.NULL
                 self._in.size = 0
                 self._in.pos = 0
 
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' %
                                 _zstd_error(zresult))
 
             if self._out.pos == self._out.size:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
     def flush(self):
         if self._finished:
             raise ZstdError('cannot call flush() after compression finished')
 
         if self._in.src != ffi.NULL:
             raise ZstdError('cannot call flush() before consuming output from '
                             'previous operation')
 
         while True:
             zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                                self._out, self._in,
                                                lib.ZSTD_e_flush)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' % _zstd_error(zresult))
 
             if self._out.pos:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
             if not zresult:
                 return
 
     def finish(self):
         if self._finished:
             raise ZstdError('cannot call finish() after compression finished')
 
         if self._in.src != ffi.NULL:
             raise ZstdError('cannot call finish() before consuming output from '
                             'previous operation')
 
         while True:
             zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                                self._out, self._in,
                                                lib.ZSTD_e_end)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' % _zstd_error(zresult))
 
             if self._out.pos:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
             if not zresult:
                 self._finished = True
                 return
 
 
 class ZstdCompressionReader(object):
     def __init__(self, compressor, source, read_size):
         self._compressor = compressor
         self._source = source
         self._read_size = read_size
         self._entered = False
         self._closed = False
         self._bytes_compressed = 0
         self._finished_input = False
         self._finished_output = False
 
         self._in_buffer = ffi.new('ZSTD_inBuffer *')
         # Holds a ref so backing bytes in self._in_buffer stay alive.
         self._source_buffer = None
 
     def __enter__(self):
         if self._entered:
             raise ValueError('cannot __enter__ multiple times')
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self._closed = True
         self._source = None
         self._compressor = None
 
         return False
 
     def readable(self):
         return True
 
     def writable(self):
         return False
 
     def seekable(self):
         return False
 
     def readline(self):
         raise io.UnsupportedOperation()
 
     def readlines(self):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         raise OSError('stream is not writable')
 
     def writelines(self, ignored):
         raise OSError('stream is not writable')
 
     def isatty(self):
         return False
 
     def flush(self):
         return None
 
     def close(self):
         self._closed = True
         return None
 
     @property
     def closed(self):
         return self._closed
 
     def tell(self):
         return self._bytes_compressed
 
     def readall(self):
         chunks = []
 
         while True:
             chunk = self.read(1048576)
             if not chunk:
                 break
 
             chunks.append(chunk)
 
         return b''.join(chunks)
 
     def __iter__(self):
         raise io.UnsupportedOperation()
 
     def __next__(self):
         raise io.UnsupportedOperation()
 
     next = __next__
 
     def _read_input(self):
         if self._finished_input:
             return
 
         if hasattr(self._source, 'read'):
             data = self._source.read(self._read_size)
 
             if not data:
                 self._finished_input = True
                 return
 
             self._source_buffer = ffi.from_buffer(data)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
         else:
             self._source_buffer = ffi.from_buffer(self._source)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
 
     def _compress_into_buffer(self, out_buffer):
         if self._in_buffer.pos >= self._in_buffer.size:
             return
 
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                            out_buffer, self._in_buffer,
                                            lib.ZSTD_e_continue)
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if self._in_buffer.pos == self._in_buffer.size:
             self._in_buffer.src = ffi.NULL
             self._in_buffer.pos = 0
             self._in_buffer.size = 0
             self._source_buffer = None
 
             if not hasattr(self._source, 'read'):
                 self._finished_input = True
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError('zstd compress error: %s',
                             _zstd_error(zresult))
 
         return out_buffer.pos and out_buffer.pos == out_buffer.size
 
     def read(self, size=-1):
         if self._closed:
             raise ValueError('stream is closed')
 
         if size < -1:
             raise ValueError('cannot read negative amounts less than -1')
 
         if size == -1:
             return self.readall()
 
         if self._finished_output or size == 0:
             return b''
 
         # Need a dedicated ref to dest buffer otherwise it gets collected.
         dst_buffer = ffi.new('char[]', size)
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         if self._compress_into_buffer(out_buffer):
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
 
             if self._compress_into_buffer(out_buffer):
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         # EOF
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                            out_buffer, self._in_buffer,
                                            lib.ZSTD_e_end)
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error ending compression stream: %s',
                             _zstd_error(zresult))
 
         if zresult == 0:
             self._finished_output = True
 
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def read1(self, size=-1):
         if self._closed:
             raise ValueError('stream is closed')
 
         if size < -1:
             raise ValueError('cannot read negative amounts less than -1')
 
         if self._finished_output or size == 0:
             return b''
 
         # -1 returns arbitrary number of bytes.
         if size == -1:
             size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
 
         dst_buffer = ffi.new('char[]', size)
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         # read1() dictates that we can perform at most 1 call to the
         # underlying stream to get input. However, we can't satisfy this
         # restriction with compression because not all input generates output.
         # It is possible to perform a block flush in order to ensure output.
         # But this may not be desirable behavior. So we allow multiple read()
         # to the underlying stream. But unlike read(), we stop once we have
         # any output.
 
         self._compress_into_buffer(out_buffer)
         if out_buffer.pos:
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
 
             # If we've filled the output buffer, return immediately.
             if self._compress_into_buffer(out_buffer):
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
             # If we've populated the output buffer and we're not at EOF,
             # also return, as we've satisfied the read1() limits.
             if out_buffer.pos and not self._finished_input:
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
             # Else if we're at EOS and we have room left in the buffer,
             # fall through to below and try to add more data to the output.
 
         # EOF.
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                            out_buffer, self._in_buffer,
                                            lib.ZSTD_e_end)
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error ending compression stream: %s' %
                             _zstd_error(zresult))
 
         if zresult == 0:
             self._finished_output = True
 
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto(self, b):
         if self._closed:
             raise ValueError('stream is closed')
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b'', 0)
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         if self._compress_into_buffer(out_buffer):
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
             if self._compress_into_buffer(out_buffer):
                 return out_buffer.pos
 
         # EOF.
         old_pos = out_buffer.pos
         zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                            out_buffer, self._in_buffer,
                                            lib.ZSTD_e_end)
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error ending compression stream: %s',
                             _zstd_error(zresult))
 
         if zresult == 0:
             self._finished_output = True
 
         return out_buffer.pos
 
     def readinto1(self, b):
         if self._closed:
             raise ValueError('stream is closed')
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b'', 0)
 
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         self._compress_into_buffer(out_buffer)
         if out_buffer.pos:
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
 
             if self._compress_into_buffer(out_buffer):
                 return out_buffer.pos
 
             if out_buffer.pos and not self._finished_input:
                 return out_buffer.pos
 
         # EOF.
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
                                            out_buffer, self._in_buffer,
                                            lib.ZSTD_e_end)
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error ending compression stream: %s' %
                             _zstd_error(zresult))
 
         if zresult == 0:
             self._finished_output = True
 
         return out_buffer.pos
 
 
 class ZstdCompressor(object):
     def __init__(self, level=3, dict_data=None, compression_params=None,
                  write_checksum=None, write_content_size=None,
                  write_dict_id=None, threads=0):
         if level > lib.ZSTD_maxCLevel():
             raise ValueError('level must be less than %d' % lib.ZSTD_maxCLevel())
 
         if threads < 0:
             threads = _cpu_count()
 
         if compression_params and write_checksum is not None:
             raise ValueError('cannot define compression_params and '
                              'write_checksum')
 
         if compression_params and write_content_size is not None:
             raise ValueError('cannot define compression_params and '
                              'write_content_size')
 
         if compression_params and write_dict_id is not None:
             raise ValueError('cannot define compression_params and '
                              'write_dict_id')
 
         if compression_params and threads:
             raise ValueError('cannot define compression_params and threads')
 
         if compression_params:
             self._params = _make_cctx_params(compression_params)
         else:
             if write_dict_id is None:
                 write_dict_id = True
 
             params = lib.ZSTD_createCCtxParams()
             if params == ffi.NULL:
                 raise MemoryError()
 
             self._params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
 
             _set_compression_parameter(self._params,
                                        lib.ZSTD_c_compressionLevel,
                                        level)
 
             _set_compression_parameter(
                 self._params,
                 lib.ZSTD_c_contentSizeFlag,
                 write_content_size if write_content_size is not None else 1)
 
             _set_compression_parameter(self._params,
                                        lib.ZSTD_c_checksumFlag,
                                        1 if write_checksum else 0)
 
             _set_compression_parameter(self._params,
                                        lib.ZSTD_c_dictIDFlag,
                                        1 if write_dict_id else 0)
 
             if threads:
                 _set_compression_parameter(self._params,
                                            lib.ZSTD_c_nbWorkers,
                                            threads)
 
         cctx = lib.ZSTD_createCCtx()
         if cctx == ffi.NULL:
             raise MemoryError()
 
         self._cctx = cctx
         self._dict_data = dict_data
 
         # We defer setting up garbage collection until after calling
         # _setup_cctx() to ensure the memory size estimate is more accurate.
         try:
             self._setup_cctx()
         finally:
             self._cctx = ffi.gc(cctx, lib.ZSTD_freeCCtx,
                                 size=lib.ZSTD_sizeof_CCtx(cctx))
 
     def _setup_cctx(self):
         zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(self._cctx,
                                                              self._params)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('could not set compression parameters: %s' %
                             _zstd_error(zresult))
 
         dict_data = self._dict_data
 
         if dict_data:
             if dict_data._cdict:
                 zresult = lib.ZSTD_CCtx_refCDict(self._cctx, dict_data._cdict)
             else:
                 zresult = lib.ZSTD_CCtx_loadDictionary_advanced(
                     self._cctx, dict_data.as_bytes(), len(dict_data),
                     lib.ZSTD_dlm_byRef, dict_data._dict_type)
 
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('could not load compression dictionary: %s' %
                                 _zstd_error(zresult))
 
     def memory_size(self):
         return lib.ZSTD_sizeof_CCtx(self._cctx)
 
     def compress(self, data):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         data_buffer = ffi.from_buffer(data)
 
         dest_size = lib.ZSTD_compressBound(len(data_buffer))
         out = new_nonzero('char[]', dest_size)
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, len(data_buffer))
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
         out_buffer = ffi.new('ZSTD_outBuffer *')
         in_buffer = ffi.new('ZSTD_inBuffer *')
 
         out_buffer.dst = out
         out_buffer.size = dest_size
         out_buffer.pos = 0
 
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_compressStream2(self._cctx,
                                            out_buffer,
                                            in_buffer,
                                            lib.ZSTD_e_end)
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError('cannot compress: %s' %
                             _zstd_error(zresult))
         elif zresult:
             raise ZstdError('unexpected partial frame flush')
 
         return ffi.buffer(out, out_buffer.pos)[:]
 
     def compressobj(self, size=-1):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
         cobj = ZstdCompressionObj()
         cobj._out = ffi.new('ZSTD_outBuffer *')
         cobj._dst_buffer = ffi.new('char[]', COMPRESSION_RECOMMENDED_OUTPUT_SIZE)
         cobj._out.dst = cobj._dst_buffer
         cobj._out.size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
         cobj._out.pos = 0
         cobj._compressor = self
         cobj._finished = False
 
         return cobj
 
     def chunker(self, size=-1, chunk_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
         return ZstdCompressionChunker(self, chunk_size=chunk_size)
 
     def copy_stream(self, ifh, ofh, size=-1,
                     read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE,
                     write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
 
         if not hasattr(ifh, 'read'):
             raise ValueError('first argument must have a read() method')
         if not hasattr(ofh, 'write'):
             raise ValueError('second argument must have a write() method')
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         out_buffer = ffi.new('ZSTD_outBuffer *')
 
         dst_buffer = ffi.new('char[]', write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         total_read, total_write = 0, 0
 
         while True:
             data = ifh.read(read_size)
             if not data:
                 break
 
             data_buffer = ffi.from_buffer(data)
             total_read += len(data_buffer)
             in_buffer.src = data_buffer
             in_buffer.size = len(data_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 zresult = lib.ZSTD_compressStream2(self._cctx,
                                                    out_buffer,
                                                    in_buffer,
                                                    lib.ZSTD_e_continue)
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError('zstd compress error: %s' %
                                     _zstd_error(zresult))
 
                 if out_buffer.pos:
                     ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                     total_write += out_buffer.pos
                     out_buffer.pos = 0
 
         # We've finished reading. Flush the compressor.
         while True:
             zresult = lib.ZSTD_compressStream2(self._cctx,
                                                out_buffer,
                                                in_buffer,
                                                lib.ZSTD_e_end)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('error ending compression stream: %s' %
                                 _zstd_error(zresult))
 
             if out_buffer.pos:
                 ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                 total_write += out_buffer.pos
                 out_buffer.pos = 0
 
             if zresult == 0:
                 break
 
         return total_read, total_write
 
     def stream_reader(self, source, size=-1,
                       read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         try:
             size = len(source)
         except Exception:
             pass
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
         return ZstdCompressionReader(self, source, read_size)
 
     def stream_writer(self, writer, size=-1,
                  write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE,
                  write_return_read=False):
 
         if not hasattr(writer, 'write'):
             raise ValueError('must pass an object with a write() method')
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         return ZstdCompressionWriter(self, writer, size, write_size,
                                      write_return_read)
 
     write_to = stream_writer
 
     def read_to_iter(self, reader, size=-1,
                      read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE,
                      write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         if hasattr(reader, 'read'):
             have_read = True
         elif hasattr(reader, '__getitem__'):
             have_read = False
             buffer_offset = 0
             size = len(reader)
         else:
             raise ValueError('must pass an object with a read() method or '
                              'conforms to buffer protocol')
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         out_buffer = ffi.new('ZSTD_outBuffer *')
 
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         dst_buffer = ffi.new('char[]', write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         while True:
             # We should never have output data sitting around after a previous
             # iteration.
             assert out_buffer.pos == 0
 
             # Collect input data.
             if have_read:
                 read_result = reader.read(read_size)
             else:
                 remaining = len(reader) - buffer_offset
                 slice_size = min(remaining, read_size)
                 read_result = reader[buffer_offset:buffer_offset + slice_size]
                 buffer_offset += slice_size
 
             # No new input data. Break out of the read loop.
             if not read_result:
                 break
 
             # Feed all read data into the compressor and emit output until
             # exhausted.
             read_buffer = ffi.from_buffer(read_result)
             in_buffer.src = read_buffer
             in_buffer.size = len(read_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 zresult = lib.ZSTD_compressStream2(self._cctx, out_buffer, in_buffer,
                                                    lib.ZSTD_e_continue)
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError('zstd compress error: %s' %
                                     _zstd_error(zresult))
 
                 if out_buffer.pos:
                     data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                     out_buffer.pos = 0
                     yield data
 
             assert out_buffer.pos == 0
 
             # And repeat the loop to collect more data.
             continue
 
         # If we get here, input is exhausted. End the stream and emit what
         # remains.
         while True:
             assert out_buffer.pos == 0
             zresult = lib.ZSTD_compressStream2(self._cctx,
                                                out_buffer,
                                                in_buffer,
                                                lib.ZSTD_e_end)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('error ending compression stream: %s' %
                                 _zstd_error(zresult))
 
             if out_buffer.pos:
                 data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                 out_buffer.pos = 0
                 yield data
 
             if zresult == 0:
                 break
 
     read_from = read_to_iter
 
     def frame_progression(self):
         progression = lib.ZSTD_getFrameProgression(self._cctx)
 
         return progression.ingested, progression.consumed, progression.produced
 
 
 class FrameParameters(object):
     def __init__(self, fparams):
         self.content_size = fparams.frameContentSize
         self.window_size = fparams.windowSize
         self.dict_id = fparams.dictID
         self.has_checksum = bool(fparams.checksumFlag)
 
 
 def frame_content_size(data):
     data_buffer = ffi.from_buffer(data)
 
     size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer))
 
     if size == lib.ZSTD_CONTENTSIZE_ERROR:
         raise ZstdError('error when determining content size')
     elif size == lib.ZSTD_CONTENTSIZE_UNKNOWN:
         return -1
     else:
         return size
 
 
 def frame_header_size(data):
     data_buffer = ffi.from_buffer(data)
 
     zresult = lib.ZSTD_frameHeaderSize(data_buffer, len(data_buffer))
     if lib.ZSTD_isError(zresult):
         raise ZstdError('could not determine frame header size: %s' %
                         _zstd_error(zresult))
 
     return zresult
 
 
 def get_frame_parameters(data):
     params = ffi.new('ZSTD_frameHeader *')
 
     data_buffer = ffi.from_buffer(data)
     zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer))
     if lib.ZSTD_isError(zresult):
         raise ZstdError('cannot get frame parameters: %s' %
                         _zstd_error(zresult))
 
     if zresult:
         raise ZstdError('not enough data for frame parameters; need %d bytes' %
                         zresult)
 
     return FrameParameters(params[0])
 
 
 class ZstdCompressionDict(object):
     def __init__(self, data, dict_type=DICT_TYPE_AUTO, k=0, d=0):
         assert isinstance(data, bytes_type)
         self._data = data
         self.k = k
         self.d = d
 
         if dict_type not in (DICT_TYPE_AUTO, DICT_TYPE_RAWCONTENT,
                              DICT_TYPE_FULLDICT):
             raise ValueError('invalid dictionary load mode: %d; must use '
                              'DICT_TYPE_* constants')
 
         self._dict_type = dict_type
         self._cdict = None
 
     def __len__(self):
         return len(self._data)
 
     def dict_id(self):
         return int_type(lib.ZDICT_getDictID(self._data, len(self._data)))
 
     def as_bytes(self):
         return self._data
 
     def precompute_compress(self, level=0, compression_params=None):
         if level and compression_params:
             raise ValueError('must only specify one of level or '
                              'compression_params')
 
         if not level and not compression_params:
             raise ValueError('must specify one of level or compression_params')
 
         if level:
             cparams = lib.ZSTD_getCParams(level, 0, len(self._data))
         else:
             cparams = ffi.new('ZSTD_compressionParameters')
             cparams.chainLog = compression_params.chain_log
             cparams.hashLog = compression_params.hash_log
             cparams.minMatch = compression_params.min_match
             cparams.searchLog = compression_params.search_log
             cparams.strategy = compression_params.compression_strategy
             cparams.targetLength = compression_params.target_length
             cparams.windowLog = compression_params.window_log
 
         cdict = lib.ZSTD_createCDict_advanced(self._data, len(self._data),
                                               lib.ZSTD_dlm_byRef,
                                               self._dict_type,
                                               cparams,
                                               lib.ZSTD_defaultCMem)
         if cdict == ffi.NULL:
             raise ZstdError('unable to precompute dictionary')
 
         self._cdict = ffi.gc(cdict, lib.ZSTD_freeCDict,
                              size=lib.ZSTD_sizeof_CDict(cdict))
 
     @property
     def _ddict(self):
         ddict = lib.ZSTD_createDDict_advanced(self._data, len(self._data),
                                               lib.ZSTD_dlm_byRef,
                                               self._dict_type,
                                               lib.ZSTD_defaultCMem)
 
         if ddict == ffi.NULL:
             raise ZstdError('could not create decompression dict')
 
         ddict = ffi.gc(ddict, lib.ZSTD_freeDDict,
                        size=lib.ZSTD_sizeof_DDict(ddict))
         self.__dict__['_ddict'] = ddict
 
         return ddict
 
 def train_dictionary(dict_size, samples, k=0, d=0, notifications=0, dict_id=0,
                      level=0, steps=0, threads=0):
     if not isinstance(samples, list):
         raise TypeError('samples must be a list')
 
     if threads < 0:
         threads = _cpu_count()
 
     total_size = sum(map(len, samples))
 
     samples_buffer = new_nonzero('char[]', total_size)
     sample_sizes = new_nonzero('size_t[]', len(samples))
 
     offset = 0
     for i, sample in enumerate(samples):
         if not isinstance(sample, bytes_type):
             raise ValueError('samples must be bytes')
 
         l = len(sample)
         ffi.memmove(samples_buffer + offset, sample, l)
         offset += l
         sample_sizes[i] = l
 
     dict_data = new_nonzero('char[]', dict_size)
 
     dparams = ffi.new('ZDICT_cover_params_t *')[0]
     dparams.k = k
     dparams.d = d
     dparams.steps = steps
     dparams.nbThreads = threads
     dparams.zParams.notificationLevel = notifications
     dparams.zParams.dictID = dict_id
     dparams.zParams.compressionLevel = level
 
     if (not dparams.k and not dparams.d and not dparams.steps
         and not dparams.nbThreads and not dparams.zParams.notificationLevel
         and not dparams.zParams.dictID
         and not dparams.zParams.compressionLevel):
         zresult = lib.ZDICT_trainFromBuffer(
             ffi.addressof(dict_data), dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0), len(samples))
     elif dparams.steps or dparams.nbThreads:
         zresult = lib.ZDICT_optimizeTrainFromBuffer_cover(
             ffi.addressof(dict_data), dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0), len(samples),
             ffi.addressof(dparams))
     else:
         zresult = lib.ZDICT_trainFromBuffer_cover(
             ffi.addressof(dict_data), dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0), len(samples),
             dparams)
 
     if lib.ZDICT_isError(zresult):
         msg = ffi.string(lib.ZDICT_getErrorName(zresult)).decode('utf-8')
         raise ZstdError('cannot train dict: %s' % msg)
 
     return ZstdCompressionDict(ffi.buffer(dict_data, zresult)[:],
                                dict_type=DICT_TYPE_FULLDICT,
                                k=dparams.k, d=dparams.d)
 
 
 class ZstdDecompressionObj(object):
     def __init__(self, decompressor, write_size):
         self._decompressor = decompressor
         self._write_size = write_size
         self._finished = False
 
     def decompress(self, data):
         if self._finished:
             raise ZstdError('cannot use a decompressobj multiple times')
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         out_buffer = ffi.new('ZSTD_outBuffer *')
 
         data_buffer = ffi.from_buffer(data)
 
         if len(data_buffer) == 0:
             return b''
 
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         dst_buffer = ffi.new('char[]', self._write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         chunks = []
 
         while True:
             zresult = lib.ZSTD_decompressStream(self._decompressor._dctx,
                                                 out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd decompressor error: %s' %
                                 _zstd_error(zresult))
 
             if zresult == 0:
                 self._finished = True
                 self._decompressor = None
 
             if out_buffer.pos:
                 chunks.append(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
 
             if (zresult == 0 or
                     (in_buffer.pos == in_buffer.size and out_buffer.pos == 0)):
                 break
 
             out_buffer.pos = 0
 
         return b''.join(chunks)
 
     def flush(self, length=0):
         pass
 
 
 class ZstdDecompressionReader(object):
     def __init__(self, decompressor, source, read_size, read_across_frames):
         self._decompressor = decompressor
         self._source = source
         self._read_size = read_size
         self._read_across_frames = bool(read_across_frames)
         self._entered = False
         self._closed = False
         self._bytes_decompressed = 0
         self._finished_input = False
         self._finished_output = False
         self._in_buffer = ffi.new('ZSTD_inBuffer *')
         # Holds a ref to self._in_buffer.src.
         self._source_buffer = None
 
     def __enter__(self):
         if self._entered:
             raise ValueError('cannot __enter__ multiple times')
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self._closed = True
         self._source = None
         self._decompressor = None
 
         return False
 
     def readable(self):
         return True
 
     def writable(self):
         return False
 
     def seekable(self):
         return True
 
     def readline(self):
         raise io.UnsupportedOperation()
 
     def readlines(self):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         raise io.UnsupportedOperation()
 
     def writelines(self, lines):
         raise io.UnsupportedOperation()
 
     def isatty(self):
         return False
 
     def flush(self):
         return None
 
     def close(self):
         self._closed = True
         return None
 
     @property
     def closed(self):
         return self._closed
 
     def tell(self):
         return self._bytes_decompressed
 
     def readall(self):
         chunks = []
 
         while True:
             chunk = self.read(1048576)
             if not chunk:
                 break
 
             chunks.append(chunk)
 
         return b''.join(chunks)
 
     def __iter__(self):
         raise io.UnsupportedOperation()
 
     def __next__(self):
         raise io.UnsupportedOperation()
 
     next = __next__
 
     def _read_input(self):
         # We have data left over in the input buffer. Use it.
         if self._in_buffer.pos < self._in_buffer.size:
             return
 
         # All input data exhausted. Nothing to do.
         if self._finished_input:
             return
 
         # Else populate the input buffer from our source.
         if hasattr(self._source, 'read'):
             data = self._source.read(self._read_size)
 
             if not data:
                 self._finished_input = True
                 return
 
             self._source_buffer = ffi.from_buffer(data)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
         else:
             self._source_buffer = ffi.from_buffer(self._source)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
 
     def _decompress_into_buffer(self, out_buffer):
         """Decompress available input into an output buffer.
 
         Returns True if data in output buffer should be emitted.
         """
         zresult = lib.ZSTD_decompressStream(self._decompressor._dctx,
                                             out_buffer, self._in_buffer)
 
         if self._in_buffer.pos == self._in_buffer.size:
             self._in_buffer.src = ffi.NULL
             self._in_buffer.pos = 0
             self._in_buffer.size = 0
             self._source_buffer = None
 
             if not hasattr(self._source, 'read'):
                 self._finished_input = True
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError('zstd decompress error: %s' %
                             _zstd_error(zresult))
 
         # Emit data if there is data AND either:
         # a) output buffer is full (read amount is satisfied)
         # b) we're at end of a frame and not in frame spanning mode
         return (out_buffer.pos and
                 (out_buffer.pos == out_buffer.size or
                  zresult == 0 and not self._read_across_frames))
 
     def read(self, size=-1):
         if self._closed:
             raise ValueError('stream is closed')
 
         if size < -1:
             raise ValueError('cannot read negative amounts less than -1')
 
         if size == -1:
             # This is recursive. But it gets the job done.
             return self.readall()
 
         if self._finished_output or size == 0:
             return b''
 
         # We /could/ call into readinto() here. But that introduces more
         # overhead.
         dst_buffer = ffi.new('char[]', size)
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         self._read_input()
         if self._decompress_into_buffer(out_buffer):
             self._bytes_decompressed += out_buffer.pos
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
             if self._decompress_into_buffer(out_buffer):
                 self._bytes_decompressed += out_buffer.pos
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         self._bytes_decompressed += out_buffer.pos
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto(self, b):
         if self._closed:
             raise ValueError('stream is closed')
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b'', 0)
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         self._read_input()
         if self._decompress_into_buffer(out_buffer):
             self._bytes_decompressed += out_buffer.pos
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
             if self._decompress_into_buffer(out_buffer):
                 self._bytes_decompressed += out_buffer.pos
                 return out_buffer.pos
 
         self._bytes_decompressed += out_buffer.pos
         return out_buffer.pos
 
     def read1(self, size=-1):
         if self._closed:
             raise ValueError('stream is closed')
 
         if size < -1:
             raise ValueError('cannot read negative amounts less than -1')
 
         if self._finished_output or size == 0:
             return b''
 
         # -1 returns arbitrary number of bytes.
         if size == -1:
             size = DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE
 
         dst_buffer = ffi.new('char[]', size)
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         # read1() dictates that we can perform at most 1 call to underlying
         # stream to get input. However, we can't satisfy this restriction with
         # decompression because not all input generates output. So we allow
         # multiple read(). But unlike read(), we stop once we have any output.
         while not self._finished_input:
             self._read_input()
             self._decompress_into_buffer(out_buffer)
 
             if out_buffer.pos:
                 break
 
         self._bytes_decompressed += out_buffer.pos
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto1(self, b):
         if self._closed:
             raise ValueError('stream is closed')
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b'', 0)
 
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         while not self._finished_input and not self._finished_output:
             self._read_input()
             self._decompress_into_buffer(out_buffer)
 
             if out_buffer.pos:
                 break
 
         self._bytes_decompressed += out_buffer.pos
         return out_buffer.pos
 
     def seek(self, pos, whence=os.SEEK_SET):
         if self._closed:
             raise ValueError('stream is closed')
 
         read_amount = 0
 
         if whence == os.SEEK_SET:
             if pos < 0:
                 raise ValueError('cannot seek to negative position with SEEK_SET')
 
             if pos < self._bytes_decompressed:
                 raise ValueError('cannot seek zstd decompression stream '
                                  'backwards')
 
             read_amount = pos - self._bytes_decompressed
 
         elif whence == os.SEEK_CUR:
             if pos < 0:
                 raise ValueError('cannot seek zstd decompression stream '
                                  'backwards')
 
             read_amount = pos
         elif whence == os.SEEK_END:
             raise ValueError('zstd decompression streams cannot be seeked '
                              'with SEEK_END')
 
         while read_amount:
             result = self.read(min(read_amount,
                                    DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE))
 
             if not result:
                 break
 
             read_amount -= len(result)
 
         return self._bytes_decompressed
 
 class ZstdDecompressionWriter(object):
     def __init__(self, decompressor, writer, write_size, write_return_read):
         decompressor._ensure_dctx()
 
         self._decompressor = decompressor
         self._writer = writer
         self._write_size = write_size
         self._write_return_read = bool(write_return_read)
         self._entered = False
         self._closed = False
 
     def __enter__(self):
         if self._closed:
             raise ValueError('stream is closed')
 
         if self._entered:
             raise ZstdError('cannot __enter__ multiple times')
 
         self._entered = True
 
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self.close()
 
     def memory_size(self):
         return lib.ZSTD_sizeof_DCtx(self._decompressor._dctx)
 
     def close(self):
         if self._closed:
             return
 
         try:
             self.flush()
         finally:
             self._closed = True
 
         f = getattr(self._writer, 'close', None)
         if f:
             f()
 
     @property
     def closed(self):
         return self._closed
 
     def fileno(self):
         f = getattr(self._writer, 'fileno', None)
         if f:
             return f()
         else:
             raise OSError('fileno not available on underlying writer')
 
     def flush(self):
         if self._closed:
             raise ValueError('stream is closed')
 
         f = getattr(self._writer, 'flush', None)
         if f:
             return f()
 
     def isatty(self):
         return False
 
     def readable(self):
         return False
 
     def readline(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readlines(self, hint=-1):
         raise io.UnsupportedOperation()
 
     def seek(self, offset, whence=None):
         raise io.UnsupportedOperation()
 
     def seekable(self):
         return False
 
     def tell(self):
         raise io.UnsupportedOperation()
 
     def truncate(self, size=None):
         raise io.UnsupportedOperation()
 
     def writable(self):
         return True
 
     def writelines(self, lines):
         raise io.UnsupportedOperation()
 
     def read(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readall(self):
         raise io.UnsupportedOperation()
 
     def readinto(self, b):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         if self._closed:
             raise ValueError('stream is closed')
 
         total_write = 0
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         out_buffer = ffi.new('ZSTD_outBuffer *')
 
         data_buffer = ffi.from_buffer(data)
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         dst_buffer = ffi.new('char[]', self._write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         dctx = self._decompressor._dctx
 
         while in_buffer.pos < in_buffer.size:
             zresult = lib.ZSTD_decompressStream(dctx, out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd decompress error: %s' %
                                 _zstd_error(zresult))
 
             if out_buffer.pos:
                 self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
                 total_write += out_buffer.pos
                 out_buffer.pos = 0
 
         if self._write_return_read:
             return in_buffer.pos
         else:
             return total_write
 
 
 class ZstdDecompressor(object):
     def __init__(self, dict_data=None, max_window_size=0, format=FORMAT_ZSTD1):
         self._dict_data = dict_data
         self._max_window_size = max_window_size
         self._format = format
 
         dctx = lib.ZSTD_createDCtx()
         if dctx == ffi.NULL:
             raise MemoryError()
 
         self._dctx = dctx
 
         # Defer setting up garbage collection until full state is loaded so
         # the memory size is more accurate.
         try:
             self._ensure_dctx()
         finally:
             self._dctx = ffi.gc(dctx, lib.ZSTD_freeDCtx,
                                 size=lib.ZSTD_sizeof_DCtx(dctx))
 
     def memory_size(self):
         return lib.ZSTD_sizeof_DCtx(self._dctx)
 
     def decompress(self, data, max_output_size=0):
         self._ensure_dctx()
 
         data_buffer = ffi.from_buffer(data)
 
         output_size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer))
 
         if output_size == lib.ZSTD_CONTENTSIZE_ERROR:
             raise ZstdError('error determining content size from frame header')
         elif output_size == 0:
             return b''
         elif output_size == lib.ZSTD_CONTENTSIZE_UNKNOWN:
             if not max_output_size:
                 raise ZstdError('could not determine content size in frame header')
 
             result_buffer = ffi.new('char[]', max_output_size)
             result_size = max_output_size
             output_size = 0
         else:
             result_buffer = ffi.new('char[]', output_size)
             result_size = output_size
 
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = result_buffer
         out_buffer.size = result_size
         out_buffer.pos = 0
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('decompression error: %s' %
                             _zstd_error(zresult))
         elif zresult:
             raise ZstdError('decompression error: did not decompress full frame')
         elif output_size and out_buffer.pos != output_size:
             raise ZstdError('decompression error: decompressed %d bytes; expected %d' %
                             (zresult, output_size))
 
         return ffi.buffer(result_buffer, out_buffer.pos)[:]
 
     def stream_reader(self, source, read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
                       read_across_frames=False):
         self._ensure_dctx()
         return ZstdDecompressionReader(self, source, read_size, read_across_frames)
 
     def decompressobj(self, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         if write_size < 1:
             raise ValueError('write_size must be positive')
 
         self._ensure_dctx()
         return ZstdDecompressionObj(self, write_size=write_size)
 
     def read_to_iter(self, reader, read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
                      write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
                      skip_bytes=0):
         if skip_bytes >= read_size:
             raise ValueError('skip_bytes must be smaller than read_size')
 
         if hasattr(reader, 'read'):
             have_read = True
         elif hasattr(reader, '__getitem__'):
             have_read = False
             buffer_offset = 0
             size = len(reader)
         else:
             raise ValueError('must pass an object with a read() method or '
                              'conforms to buffer protocol')
 
         if skip_bytes:
             if have_read:
                 reader.read(skip_bytes)
             else:
                 if skip_bytes > size:
                     raise ValueError('skip_bytes larger than first input chunk')
 
                 buffer_offset = skip_bytes
 
         self._ensure_dctx()
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         out_buffer = ffi.new('ZSTD_outBuffer *')
 
         dst_buffer = ffi.new('char[]', write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         while True:
             assert out_buffer.pos == 0
 
             if have_read:
                 read_result = reader.read(read_size)
             else:
                 remaining = size - buffer_offset
                 slice_size = min(remaining, read_size)
                 read_result = reader[buffer_offset:buffer_offset + slice_size]
                 buffer_offset += slice_size
 
             # No new input. Break out of read loop.
             if not read_result:
                 break
 
             # Feed all read data into decompressor and emit output until
             # exhausted.
             read_buffer = ffi.from_buffer(read_result)
             in_buffer.src = read_buffer
             in_buffer.size = len(read_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 assert out_buffer.pos == 0
 
                 zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError('zstd decompress error: %s' %
                                     _zstd_error(zresult))
 
                 if out_buffer.pos:
                     data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                     out_buffer.pos = 0
                     yield data
 
                 if zresult == 0:
                     return
 
             # Repeat loop to collect more input data.
             continue
 
         # If we get here, input is exhausted.
 
     read_from = read_to_iter
 
     def stream_writer(self, writer, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
                       write_return_read=False):
         if not hasattr(writer, 'write'):
             raise ValueError('must pass an object with a write() method')
 
         return ZstdDecompressionWriter(self, writer, write_size,
                                        write_return_read)
 
     write_to = stream_writer
 
     def copy_stream(self, ifh, ofh,
                     read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
                     write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         if not hasattr(ifh, 'read'):
             raise ValueError('first argument must have a read() method')
         if not hasattr(ofh, 'write'):
             raise ValueError('second argument must have a write() method')
 
         self._ensure_dctx()
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         out_buffer = ffi.new('ZSTD_outBuffer *')
 
         dst_buffer = ffi.new('char[]', write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         total_read, total_write = 0, 0
 
         # Read all available input.
         while True:
             data = ifh.read(read_size)
             if not data:
                 break
 
             data_buffer = ffi.from_buffer(data)
             total_read += len(data_buffer)
             in_buffer.src = data_buffer
             in_buffer.size = len(data_buffer)
             in_buffer.pos = 0
 
             # Flush all read data to output.
             while in_buffer.pos < in_buffer.size:
                 zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError('zstd decompressor error: %s' %
                                     _zstd_error(zresult))
 
                 if out_buffer.pos:
                     ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                     total_write += out_buffer.pos
                     out_buffer.pos = 0
 
             # Continue loop to keep reading.
 
         return total_read, total_write
 
     def decompress_content_dict_chain(self, frames):
         if not isinstance(frames, list):
             raise TypeError('argument must be a list')
 
         if not frames:
             raise ValueError('empty input chain')
 
         # First chunk should not be using a dictionary. We handle it specially.
         chunk = frames[0]
         if not isinstance(chunk, bytes_type):
             raise ValueError('chunk 0 must be bytes')
 
         # All chunks should be zstd frames and should have content size set.
         chunk_buffer = ffi.from_buffer(chunk)
         params = ffi.new('ZSTD_frameHeader *')
         zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer))
         if lib.ZSTD_isError(zresult):
             raise ValueError('chunk 0 is not a valid zstd frame')
         elif zresult:
             raise ValueError('chunk 0 is too small to contain a zstd frame')
 
         if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN:
             raise ValueError('chunk 0 missing content size in frame')
 
         self._ensure_dctx(load_dict=False)
 
         last_buffer = ffi.new('char[]', params.frameContentSize)
 
         out_buffer = ffi.new('ZSTD_outBuffer *')
         out_buffer.dst = last_buffer
         out_buffer.size = len(last_buffer)
         out_buffer.pos = 0
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
         in_buffer.src = chunk_buffer
         in_buffer.size = len(chunk_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('could not decompress chunk 0: %s' %
                             _zstd_error(zresult))
         elif zresult:
             raise ZstdError('chunk 0 did not decompress full frame')
 
         # Special case of chain length of 1
         if len(frames) == 1:
             return ffi.buffer(last_buffer, len(last_buffer))[:]
 
         i = 1
         while i < len(frames):
             chunk = frames[i]
             if not isinstance(chunk, bytes_type):
                 raise ValueError('chunk %d must be bytes' % i)
 
             chunk_buffer = ffi.from_buffer(chunk)
             zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer))
             if lib.ZSTD_isError(zresult):
                 raise ValueError('chunk %d is not a valid zstd frame' % i)
             elif zresult:
                 raise ValueError('chunk %d is too small to contain a zstd frame' % i)
 
             if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN:
                 raise ValueError('chunk %d missing content size in frame' % i)
 
             dest_buffer = ffi.new('char[]', params.frameContentSize)
 
             out_buffer.dst = dest_buffer
             out_buffer.size = len(dest_buffer)
             out_buffer.pos = 0
 
             in_buffer.src = chunk_buffer
             in_buffer.size = len(chunk_buffer)
             in_buffer.pos = 0
 
             zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('could not decompress chunk %d: %s' %
                                 _zstd_error(zresult))
             elif zresult:
                 raise ZstdError('chunk %d did not decompress full frame' % i)
 
             last_buffer = dest_buffer
             i += 1
 
         return ffi.buffer(last_buffer, len(last_buffer))[:]
 
     def _ensure_dctx(self, load_dict=True):
         lib.ZSTD_DCtx_reset(self._dctx, lib.ZSTD_reset_session_only)
 
         if self._max_window_size:
             zresult = lib.ZSTD_DCtx_setMaxWindowSize(self._dctx,
                                                      self._max_window_size)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('unable to set max window size: %s' %
                                 _zstd_error(zresult))
 
         zresult = lib.ZSTD_DCtx_setFormat(self._dctx, self._format)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('unable to set decoding format: %s' %
                             _zstd_error(zresult))
 
         if self._dict_data and load_dict:
             zresult = lib.ZSTD_DCtx_refDDict(self._dctx, self._dict_data._ddict)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('unable to reference prepared dictionary: %s' %
                                 _zstd_error(zresult))

contrib/python-zstandard/zstd.c

@@ -1,344 +1,344 b''
 /**
  * Copyright (c) 2016-present, Gregory Szorc
  * All rights reserved.
  *
  * This software may be modified and distributed under the terms
  * of the BSD license. See the LICENSE file for details.
  */
 
 /* A Python C extension for Zstandard. */
 
 #if defined(_WIN32)
 #define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__)
 #include <sys/types.h>
 #include <sys/sysctl.h>
 #endif
 
 #include "python-zstandard.h"
 
 PyObject *ZstdError;
 
 PyDoc_STRVAR(estimate_decompression_context_size__doc__,
 "estimate_decompression_context_size()\n"
 "\n"
 "Estimate the amount of memory allocated to a decompression context.\n"
 );
 
 static PyObject* estimate_decompression_context_size(PyObject* self) {
 	return PyLong_FromSize_t(ZSTD_estimateDCtxSize());
 }
 
 PyDoc_STRVAR(frame_content_size__doc__,
 "frame_content_size(data)\n"
 "\n"
 "Obtain the decompressed size of a frame."
 );
 
 static PyObject* frame_content_size(PyObject* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"source",
 		NULL
 	};
 
 	Py_buffer source;
 	PyObject* result = NULL;
 	unsigned long long size;
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:frame_content_size",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:frame_content_size",
 #endif
 		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	size = ZSTD_getFrameContentSize(source.buf, source.len);
 
 	if (size == ZSTD_CONTENTSIZE_ERROR) {
 		PyErr_SetString(ZstdError, "error when determining content size");
 	}
 	else if (size == ZSTD_CONTENTSIZE_UNKNOWN) {
 		result = PyLong_FromLong(-1);
 	}
 	else {
 		result = PyLong_FromUnsignedLongLong(size);
 	}
 
 finally:
 	PyBuffer_Release(&source);
 
 	return result;
 }
 
 PyDoc_STRVAR(frame_header_size__doc__,
 "frame_header_size(data)\n"
 "\n"
 "Obtain the size of a frame header.\n"
 );
 
 static PyObject* frame_header_size(PyObject* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"source",
 		NULL
 	};
 
 	Py_buffer source;
 	PyObject* result = NULL;
 	size_t zresult;
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:frame_header_size",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:frame_header_size",
 #endif
 		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	zresult = ZSTD_frameHeaderSize(source.buf, source.len);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "could not determine frame header size: %s",
 			ZSTD_getErrorName(zresult));
 	}
 	else {
 		result = PyLong_FromSize_t(zresult);
 	}
 
 finally:
 
 	PyBuffer_Release(&source);
 
 	return result;
 }
 
 PyDoc_STRVAR(get_frame_parameters__doc__,
 "get_frame_parameters(data)\n"
 "\n"
 "Obtains a ``FrameParameters`` instance by parsing data.\n");
 
 PyDoc_STRVAR(train_dictionary__doc__,
 "train_dictionary(dict_size, samples, k=None, d=None, steps=None,\n"
 "                 threads=None,notifications=0, dict_id=0, level=0)\n"
 "\n"
 "Train a dictionary from sample data using the COVER algorithm.\n"
 "\n"
 "A compression dictionary of size ``dict_size`` will be created from the\n"
 "iterable of ``samples``. The raw dictionary bytes will be returned.\n"
 "\n"
 "The COVER algorithm has 2 parameters: ``k`` and ``d``. These control the\n"
 "*segment size* and *dmer size*. A reasonable range for ``k`` is\n"
 "``[16, 2048+]``. A reasonable range for ``d`` is ``[6, 16]``.\n"
 "``d`` must be less than or equal to ``k``.\n"
 "\n"
 "``steps`` can be specified to control the number of steps through potential\n"
 "values of ``k`` and ``d`` to try. ``k`` and ``d`` will only be varied if\n"
 "those arguments are not defined. i.e. if ``d`` is ``8``, then only ``k``\n"
 "will be varied in this mode.\n"
 "\n"
 "``threads`` can specify how many threads to use to test various ``k`` and\n"
 "``d`` values. ``-1`` will use as many threads as available CPUs. By default,\n"
 "a single thread is used.\n"
 "\n"
 "When ``k`` and ``d`` are not defined, default values are used and the\n"
 "algorithm will perform multiple iterations - or steps - to try to find\n"
 "ideal parameters. If both ``k`` and ``d`` are specified, then those values\n"
 "will be used. ``steps`` or ``threads`` triggers optimization mode to test\n"
 "multiple ``k`` and ``d`` variations.\n"
 );
 
 static char zstd_doc[] = "Interface to zstandard";
 
 static PyMethodDef zstd_methods[] = {
 	{ "estimate_decompression_context_size", (PyCFunction)estimate_decompression_context_size,
 	METH_NOARGS, estimate_decompression_context_size__doc__ },
 	{ "frame_content_size", (PyCFunction)frame_content_size,
 	METH_VARARGS | METH_KEYWORDS, frame_content_size__doc__ },
 	{ "frame_header_size", (PyCFunction)frame_header_size,
 	METH_VARARGS | METH_KEYWORDS, frame_header_size__doc__ },
 	{ "get_frame_parameters", (PyCFunction)get_frame_parameters,
 	METH_VARARGS | METH_KEYWORDS, get_frame_parameters__doc__ },
 	{ "train_dictionary", (PyCFunction)train_dictionary,
 	METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ },
 	{ NULL, NULL }
 };
 
 void bufferutil_module_init(PyObject* mod);
 void compressobj_module_init(PyObject* mod);
 void compressor_module_init(PyObject* mod);
 void compressionparams_module_init(PyObject* mod);
 void constants_module_init(PyObject* mod);
 void compressionchunker_module_init(PyObject* mod);
 void compressiondict_module_init(PyObject* mod);
 void compressionreader_module_init(PyObject* mod);
 void compressionwriter_module_init(PyObject* mod);
 void compressoriterator_module_init(PyObject* mod);
 void decompressor_module_init(PyObject* mod);
 void decompressobj_module_init(PyObject* mod);
 void decompressionreader_module_init(PyObject *mod);
 void decompressionwriter_module_init(PyObject* mod);
 void decompressoriterator_module_init(PyObject* mod);
 void frameparams_module_init(PyObject* mod);
 
 void zstd_module_init(PyObject* m) {
 	/* python-zstandard relies on unstable zstd C API features. This means
 	   that changes in zstd may break expectations in python-zstandard.
 
 	   python-zstandard is distributed with a copy of the zstd sources.
 	   python-zstandard is only guaranteed to work with the bundled version
 	   of zstd.
 
 	   However, downstream redistributors or packagers may unbundle zstd
 	   from python-zstandard. This can result in a mismatch between zstd
 	   versions and API semantics. This essentially "voids the warranty"
 	   of python-zstandard and may cause undefined behavior.
 
 	   We detect this mismatch here and refuse to load the module if this
 	   scenario is detected.
 	*/
-	if (ZSTD_VERSION_NUMBER != 10308 || ZSTD_versionNumber() != 10308) {
+	if (ZSTD_VERSION_NUMBER != 10403 || ZSTD_versionNumber() != 10403) {
 		PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version");
 		return;
 	}
 
 	bufferutil_module_init(m);
 	compressionparams_module_init(m);
 	compressiondict_module_init(m);
 	compressobj_module_init(m);
 	compressor_module_init(m);
 	compressionchunker_module_init(m);
 	compressionreader_module_init(m);
 	compressionwriter_module_init(m);
 	compressoriterator_module_init(m);
 	constants_module_init(m);
 	decompressor_module_init(m);
 	decompressobj_module_init(m);
 	decompressionreader_module_init(m);
 	decompressionwriter_module_init(m);
 	decompressoriterator_module_init(m);
 	frameparams_module_init(m);
 }
 
 #if defined(__GNUC__) && (__GNUC__ >= 4)
 #  define PYTHON_ZSTD_VISIBILITY __attribute__ ((visibility ("default")))
 #else
 #  define PYTHON_ZSTD_VISIBILITY
 #endif
 
 #if PY_MAJOR_VERSION >= 3
 static struct PyModuleDef zstd_module = {
 	PyModuleDef_HEAD_INIT,
 	"zstd",
 	zstd_doc,
 	-1,
 	zstd_methods
 };
 
 PYTHON_ZSTD_VISIBILITY PyMODINIT_FUNC PyInit_zstd(void) {
 	PyObject *m = PyModule_Create(&zstd_module);
 	if (m) {
 		zstd_module_init(m);
 		if (PyErr_Occurred()) {
 			Py_DECREF(m);
 			m = NULL;
 		}
 	}
 	return m;
 }
 #else
 PYTHON_ZSTD_VISIBILITY PyMODINIT_FUNC initzstd(void) {
 	PyObject *m = Py_InitModule3("zstd", zstd_methods, zstd_doc);
 	if (m) {
 		zstd_module_init(m);
 	}
 }
 #endif
 
 /* Attempt to resolve the number of CPUs in the system. */
 int cpu_count() {
 	int count = 0;
 
 #if defined(_WIN32)
 	SYSTEM_INFO si;
 	si.dwNumberOfProcessors = 0;
 	GetSystemInfo(&si);
 	count = si.dwNumberOfProcessors;
 #elif defined(__APPLE__)
 	int num;
 	size_t size = sizeof(int);
 
 	if (0 == sysctlbyname("hw.logicalcpu", &num, &size, NULL, 0)) {
 		count = num;
 	}
 #elif defined(__linux__)
 	count = sysconf(_SC_NPROCESSORS_ONLN);
 #elif defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__)
 	int mib[2];
 	size_t len = sizeof(count);
 	mib[0] = CTL_HW;
 	mib[1] = HW_NCPU;
 	if (0 != sysctl(mib, 2, &count, &len, NULL, 0)) {
 		count = 0;
 	}
 #elif defined(__hpux)
 	count = mpctl(MPC_GETNUMSPUS, NULL, NULL);
 #endif
 
 	return count;
 }
 
 size_t roundpow2(size_t i) {
 	i--;
 	i |= i >> 1;
 	i |= i >> 2;
 	i |= i >> 4;
 	i |= i >> 8;
 	i |= i >> 16;
 	i++;
 
 	return i;
 }
 
 /* Safer version of _PyBytes_Resize().
  *
  * _PyBytes_Resize() only works if the refcount is 1. In some scenarios,
  * we can get an object with a refcount > 1, even if it was just created
  * with PyBytes_FromStringAndSize()! That's because (at least) CPython
  * pre-allocates PyBytes instances of size 1 for every possible byte value.
  *
  * If non-0 is returned, obj may or may not be NULL.
  */
 int safe_pybytes_resize(PyObject** obj, Py_ssize_t size) {
 	PyObject* tmp;
 
 	if ((*obj)->ob_refcnt == 1) {
 		return _PyBytes_Resize(obj, size);
 	}
 
 	tmp = PyBytes_FromStringAndSize(NULL, size);
 	if (!tmp) {
 		return -1;
 	}
 
 	memcpy(PyBytes_AS_STRING(tmp), PyBytes_AS_STRING(*obj),
 		PyBytes_GET_SIZE(*obj));
 
 	Py_DECREF(*obj);
 	*obj = tmp;
 
 	return 0;
 }
  No newline at end of file

contrib/python-zstandard/zstd/common/bitstream.h

@@ -1,455 +1,459 b''
 /* ******************************************************************
    bitstream
    Part of FSE library
    Copyright (C) 2013-present, Yann Collet.
 
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
    met:
 
        * Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
        * Redistributions in binary form must reproduce the above
    copyright notice, this list of conditions and the following disclaimer
    in the documentation and/or other materials provided with the
    distribution.
 
    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
 ****************************************************************** */
 #ifndef BITSTREAM_H_MODULE
 #define BITSTREAM_H_MODULE
 
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
 /*
 *  This API consists of small unitary functions, which must be inlined for best performance.
 *  Since link-time-optimization is not available for all compilers,
 *  these functions are defined into a .h to be included.
 */
 
 /*-****************************************
 *  Dependencies
 ******************************************/
 #include "mem.h"            /* unaligned access routines */
 #include "debug.h"          /* assert(), DEBUGLOG(), RAWLOG() */
 #include "error_private.h"  /* error codes and messages */
 
 
 /*=========================================
 *  Target specific
 =========================================*/
 #if defined(__BMI__) && defined(__GNUC__)
 #  include <immintrin.h>   /* support for bextr (experimental) */
+#elif defined(__ICCARM__)
+#  include <intrinsics.h>
 #endif
 
 #define STREAM_ACCUMULATOR_MIN_32  25
 #define STREAM_ACCUMULATOR_MIN_64  57
 #define STREAM_ACCUMULATOR_MIN    ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
 
 
 /*-******************************************
 *  bitStream encoding API (write forward)
 ********************************************/
 /* bitStream can mix input from multiple sources.
  * A critical property of these streams is that they encode and decode in **reverse** direction.
  * So the first bit sequence you add will be the last to be read, like a LIFO stack.
  */
 typedef struct {
     size_t bitContainer;
     unsigned bitPos;
     char*  startPtr;
     char*  ptr;
     char*  endPtr;
 } BIT_CStream_t;
 
 MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
 MEM_STATIC void   BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
 MEM_STATIC void   BIT_flushBits(BIT_CStream_t* bitC);
 MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 
 /* Start with initCStream, providing the size of buffer to write into.
 *  bitStream will never write outside of this buffer.
 *  `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
 *
 *  bits are first added to a local register.
 *  Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
 *  Writing data into memory is an explicit operation, performed by the flushBits function.
 *  Hence keep track how many bits are potentially stored into local register to avoid register overflow.
 *  After a flushBits, a maximum of 7 bits might still be stored into local register.
 *
 *  Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
 *
 *  Last operation is to close the bitStream.
 *  The function returns the final size of CStream in bytes.
 *  If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
 */
 
 
 /*-********************************************
 *  bitStream decoding API (read backward)
 **********************************************/
 typedef struct {
     size_t   bitContainer;
     unsigned bitsConsumed;
     const char* ptr;
     const char* start;
     const char* limitPtr;
 } BIT_DStream_t;
 
 typedef enum { BIT_DStream_unfinished = 0,
                BIT_DStream_endOfBuffer = 1,
                BIT_DStream_completed = 2,
                BIT_DStream_overflow = 3 } BIT_DStream_status;  /* result of BIT_reloadDStream() */
                /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
 
 MEM_STATIC size_t   BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
 MEM_STATIC size_t   BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
 
 
 /* Start by invoking BIT_initDStream().
 *  A chunk of the bitStream is then stored into a local register.
 *  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
 *  You can then retrieve bitFields stored into the local register, **in reverse order**.
 *  Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
 *  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
 *  Otherwise, it can be less than that, so proceed accordingly.
 *  Checking if DStream has reached its end can be performed with BIT_endOfDStream().
 */
 
 
 /*-****************************************
 *  unsafe API
 ******************************************/
 MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
 /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
 
 MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
 /* unsafe version; does not check buffer overflow */
 
 MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 /* faster, but works only if nbBits >= 1 */
 
 
 
 /*-**************************************************************
 *  Internal functions
 ****************************************************************/
 MEM_STATIC unsigned BIT_highbit32 (U32 val)
 {
     assert(val != 0);
     {
 #   if defined(_MSC_VER)   /* Visual */
         unsigned long r=0;
         _BitScanReverse ( &r, val );
         return (unsigned) r;
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */
         return 31 - __builtin_clz (val);
+#   elif defined(__ICCARM__)    /* IAR Intrinsic */
+        return 31 - __CLZ(val);
 #   else   /* Software version */
         static const unsigned DeBruijnClz[32] = { 0,  9,  1, 10, 13, 21,  2, 29,
                                                  11, 14, 16, 18, 22, 25,  3, 30,
                                                   8, 12, 20, 28, 15, 17, 24,  7,
                                                  19, 27, 23,  6, 26,  5,  4, 31 };
         U32 v = val;
         v |= v >> 1;
         v |= v >> 2;
         v |= v >> 4;
         v |= v >> 8;
         v |= v >> 16;
         return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
 #   endif
     }
 }
 
 /*=====    Local Constants   =====*/
 static const unsigned BIT_mask[] = {
     0,          1,         3,         7,         0xF,       0x1F,
     0x3F,       0x7F,      0xFF,      0x1FF,     0x3FF,     0x7FF,
     0xFFF,      0x1FFF,    0x3FFF,    0x7FFF,    0xFFFF,    0x1FFFF,
     0x3FFFF,    0x7FFFF,   0xFFFFF,   0x1FFFFF,  0x3FFFFF,  0x7FFFFF,
     0xFFFFFF,   0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
     0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
 #define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
 
 /*-**************************************************************
 *  bitStream encoding
 ****************************************************************/
 /*! BIT_initCStream() :
  *  `dstCapacity` must be > sizeof(size_t)
  *  @return : 0 if success,
  *            otherwise an error code (can be tested using ERR_isError()) */
 MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
                                   void* startPtr, size_t dstCapacity)
 {
     bitC->bitContainer = 0;
     bitC->bitPos = 0;
     bitC->startPtr = (char*)startPtr;
     bitC->ptr = bitC->startPtr;
     bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);
     if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);
     return 0;
 }
 
 /*! BIT_addBits() :
  *  can add up to 31 bits into `bitC`.
  *  Note : does not check for register overflow ! */
 MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
                             size_t value, unsigned nbBits)
 {
     MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32);
     assert(nbBits < BIT_MASK_SIZE);
     assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
     bitC->bitPos += nbBits;
 }
 
 /*! BIT_addBitsFast() :
  *  works only if `value` is _clean_,
  *  meaning all high bits above nbBits are 0 */
 MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
                                 size_t value, unsigned nbBits)
 {
     assert((value>>nbBits) == 0);
     assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     bitC->bitContainer |= value << bitC->bitPos;
     bitC->bitPos += nbBits;
 }
 
 /*! BIT_flushBitsFast() :
  *  assumption : bitContainer has not overflowed
  *  unsafe version; does not check buffer overflow */
 MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
 {
     size_t const nbBytes = bitC->bitPos >> 3;
     assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     MEM_writeLEST(bitC->ptr, bitC->bitContainer);
     bitC->ptr += nbBytes;
     assert(bitC->ptr <= bitC->endPtr);
     bitC->bitPos &= 7;
     bitC->bitContainer >>= nbBytes*8;
 }
 
 /*! BIT_flushBits() :
  *  assumption : bitContainer has not overflowed
  *  safe version; check for buffer overflow, and prevents it.
  *  note : does not signal buffer overflow.
  *  overflow will be revealed later on using BIT_closeCStream() */
 MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
 {
     size_t const nbBytes = bitC->bitPos >> 3;
     assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     MEM_writeLEST(bitC->ptr, bitC->bitContainer);
     bitC->ptr += nbBytes;
     if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
     bitC->bitPos &= 7;
     bitC->bitContainer >>= nbBytes*8;
 }
 
 /*! BIT_closeCStream() :
  *  @return : size of CStream, in bytes,
  *            or 0 if it could not fit into dstBuffer */
 MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
 {
     BIT_addBitsFast(bitC, 1, 1);   /* endMark */
     BIT_flushBits(bitC);
     if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
     return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
 }
 
 
 /*-********************************************************
 *  bitStream decoding
 **********************************************************/
 /*! BIT_initDStream() :
  *  Initialize a BIT_DStream_t.
  * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
  * `srcSize` must be the *exact* size of the bitStream, in bytes.
  * @return : size of stream (== srcSize), or an errorCode if a problem is detected
  */
 MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
 {
     if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
 
     bitD->start = (const char*)srcBuffer;
     bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
 
     if (srcSize >=  sizeof(bitD->bitContainer)) {  /* normal case */
         bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
         bitD->bitContainer = MEM_readLEST(bitD->ptr);
         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
           bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;  /* ensures bitsConsumed is always set */
           if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
     } else {
         bitD->ptr   = bitD->start;
         bitD->bitContainer = *(const BYTE*)(bitD->start);
         switch(srcSize)
         {
         case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
                 /* fall-through */
 
         case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
                 /* fall-through */
 
         case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
                 /* fall-through */
 
         case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
                 /* fall-through */
 
         case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
                 /* fall-through */
 
         case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
                 /* fall-through */
 
         default: break;
         }
         {   BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
             bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
             if (lastByte == 0) return ERROR(corruption_detected);  /* endMark not present */
         }
         bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
     }
 
     return srcSize;
 }
 
 MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
 {
     return bitContainer >> start;
 }
 
 MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
 {
     U32 const regMask = sizeof(bitContainer)*8 - 1;
     /* if start > regMask, bitstream is corrupted, and result is undefined */
     assert(nbBits < BIT_MASK_SIZE);
     return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
 }
 
 MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
 {
     assert(nbBits < BIT_MASK_SIZE);
     return bitContainer & BIT_mask[nbBits];
 }
 
 /*! BIT_lookBits() :
  *  Provides next n bits from local register.
  *  local register is not modified.
  *  On 32-bits, maxNbBits==24.
  *  On 64-bits, maxNbBits==56.
  * @return : value extracted */
 MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
 {
     /* arbitrate between double-shift and shift+mask */
 #if 1
     /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
      * bitstream is likely corrupted, and result is undefined */
     return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
 #else
     /* this code path is slower on my os-x laptop */
     U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
     return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
 #endif
 }
 
 /*! BIT_lookBitsFast() :
  *  unsafe version; only works if nbBits >= 1 */
 MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
 {
     U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
     assert(nbBits >= 1);
     return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
 }
 
 MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
 {
     bitD->bitsConsumed += nbBits;
 }
 
 /*! BIT_readBits() :
  *  Read (consume) next n bits from local register and update.
  *  Pay attention to not read more than nbBits contained into local register.
  * @return : extracted value. */
 MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBits(bitD, nbBits);
     BIT_skipBits(bitD, nbBits);
     return value;
 }
 
 /*! BIT_readBitsFast() :
  *  unsafe version; only works only if nbBits >= 1 */
 MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBitsFast(bitD, nbBits);
     assert(nbBits >= 1);
     BIT_skipBits(bitD, nbBits);
     return value;
 }
 
 /*! BIT_reloadDStream() :
  *  Refill `bitD` from buffer previously set in BIT_initDStream() .
  *  This function is safe, it guarantees it will not read beyond src buffer.
  * @return : status of `BIT_DStream_t` internal register.
  *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
     if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* overflow detected, like end of stream */
         return BIT_DStream_overflow;
 
     if (bitD->ptr >= bitD->limitPtr) {
         bitD->ptr -= bitD->bitsConsumed >> 3;
         bitD->bitsConsumed &= 7;
         bitD->bitContainer = MEM_readLEST(bitD->ptr);
         return BIT_DStream_unfinished;
     }
     if (bitD->ptr == bitD->start) {
         if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
         return BIT_DStream_completed;
     }
     /* start < ptr < limitPtr */
     {   U32 nbBytes = bitD->bitsConsumed >> 3;
         BIT_DStream_status result = BIT_DStream_unfinished;
         if (bitD->ptr - nbBytes < bitD->start) {
             nbBytes = (U32)(bitD->ptr - bitD->start);  /* ptr > start */
             result = BIT_DStream_endOfBuffer;
         }
         bitD->ptr -= nbBytes;
         bitD->bitsConsumed -= nbBytes*8;
         bitD->bitContainer = MEM_readLEST(bitD->ptr);   /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */
         return result;
     }
 }
 
 /*! BIT_endOfDStream() :
  * @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
  */
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
 {
     return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
 }
 
 #if defined (__cplusplus)
 }
 #endif
 
 #endif /* BITSTREAM_H_MODULE */

contrib/python-zstandard/zstd/common/compiler.h

@@ -1,140 +1,147 b''
 /*
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
  * You may select, at your option, one of the above-listed licenses.
  */
 
 #ifndef ZSTD_COMPILER_H
 #define ZSTD_COMPILER_H
 
 /*-*******************************************************
 *  Compiler specifics
 *********************************************************/
 /* force inlining */
 
 #if !defined(ZSTD_NO_INLINE)
 #if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #  define INLINE_KEYWORD inline
 #else
 #  define INLINE_KEYWORD
 #endif
 
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__ICCARM__)
 #  define FORCE_INLINE_ATTR __attribute__((always_inline))
 #elif defined(_MSC_VER)
 #  define FORCE_INLINE_ATTR __forceinline
 #else
 #  define FORCE_INLINE_ATTR
 #endif
 
 #else
 
 #define INLINE_KEYWORD
 #define FORCE_INLINE_ATTR
 
 #endif
 
 /**
  * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
- * parameters. They must be inlined for the compiler to elimininate the constant
+ * parameters. They must be inlined for the compiler to eliminate the constant
  * branches.
  */
 #define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
 /**
  * HINT_INLINE is used to help the compiler generate better code. It is *not*
  * used for "templates", so it can be tweaked based on the compilers
  * performance.
  *
  * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
  * always_inline attribute.
  *
  * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
  * attribute.
  */
 #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
 #  define HINT_INLINE static INLINE_KEYWORD
 #else
 #  define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
 #endif
 
 /* force no inlining */
 #ifdef _MSC_VER
 #  define FORCE_NOINLINE static __declspec(noinline)
 #else
-#  ifdef __GNUC__
+#  if defined(__GNUC__) || defined(__ICCARM__)
 #    define FORCE_NOINLINE static __attribute__((__noinline__))
 #  else
 #    define FORCE_NOINLINE static
 #  endif
 #endif
 
 /* target attribute */
 #ifndef __has_attribute
   #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
 #endif
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__ICCARM__)
 #  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
 #else
 #  define TARGET_ATTRIBUTE(target)
 #endif
 
 /* Enable runtime BMI2 dispatch based on the CPU.
  * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
  */
 #ifndef DYNAMIC_BMI2
   #if ((defined(__clang__) && __has_attribute(__target__)) \
       || (defined(__GNUC__) \
           && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
       && (defined(__x86_64__) || defined(_M_X86)) \
       && !defined(__BMI2__)
   #  define DYNAMIC_BMI2 1
   #else
   #  define DYNAMIC_BMI2 0
   #endif
 #endif
 
 /* prefetch
  * can be disabled, by declaring NO_PREFETCH build macro */
 #if defined(NO_PREFETCH)
 #  define PREFETCH_L1(ptr)  (void)(ptr)  /* disabled */
 #  define PREFETCH_L2(ptr)  (void)(ptr)  /* disabled */
 #else
 #  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
 #    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
 #    define PREFETCH_L1(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
 #    define PREFETCH_L2(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
 #  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
 #    define PREFETCH_L1(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
 #    define PREFETCH_L2(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
 #  else
 #    define PREFETCH_L1(ptr) (void)(ptr)  /* disabled */
 #    define PREFETCH_L2(ptr) (void)(ptr)  /* disabled */
 #  endif
 #endif  /* NO_PREFETCH */
 
 #define CACHELINE_SIZE 64
 
 #define PREFETCH_AREA(p, s)  {            \
     const char* const _ptr = (const char*)(p);  \
     size_t const _size = (size_t)(s);     \
     size_t _pos;                          \
     for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) {  \
         PREFETCH_L2(_ptr + _pos);         \
     }                                     \
 }
 
+/* vectorization */
+#if !defined(__clang__) && defined(__GNUC__)
+#  define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
+#else
+#  define DONT_VECTORIZE
+#endif
+
 /* disable warnings */
 #ifdef _MSC_VER    /* Visual Studio */
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
 #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
 #endif
 
 #endif /* ZSTD_COMPILER_H */

contrib/python-zstandard/zstd/common/fse.h

@@ -1,708 +1,708 b''
 /* ******************************************************************
    FSE : Finite State Entropy codec
    Public Prototypes declaration
    Copyright (C) 2013-2016, Yann Collet.
 
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
    met:
 
        * Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
        * Redistributions in binary form must reproduce the above
    copyright notice, this list of conditions and the following disclaimer
    in the documentation and/or other materials provided with the
    distribution.
 
    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
 ****************************************************************** */
 
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
 #ifndef FSE_H
 #define FSE_H
 
 
 /*-*****************************************
 *  Dependencies
 ******************************************/
 #include <stddef.h>    /* size_t, ptrdiff_t */
 
 
 /*-*****************************************
 *  FSE_PUBLIC_API : control library symbols visibility
 ******************************************/
 #if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
 #  define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
 #elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1)   /* Visual expected */
 #  define FSE_PUBLIC_API __declspec(dllexport)
 #elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
 #  define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
 #  define FSE_PUBLIC_API
 #endif
 
 /*------   Version   ------*/
 #define FSE_VERSION_MAJOR    0
 #define FSE_VERSION_MINOR    9
 #define FSE_VERSION_RELEASE  0
 
 #define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
 #define FSE_QUOTE(str) #str
 #define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
 #define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
 
 #define FSE_VERSION_NUMBER  (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
 FSE_PUBLIC_API unsigned FSE_versionNumber(void);   /**< library version number; to be used when checking dll version */
 
 
 /*-****************************************
 *  FSE simple functions
 ******************************************/
 /*! FSE_compress() :
     Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
     'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
     @return : size of compressed data (<= dstCapacity).
     Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
                      if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
                      if FSE_isError(return), compression failed (more details using FSE_getErrorName())
 */
 FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
                              const void* src, size_t srcSize);
 
 /*! FSE_decompress():
     Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
     into already allocated destination buffer 'dst', of size 'dstCapacity'.
     @return : size of regenerated data (<= maxDstSize),
               or an error code, which can be tested using FSE_isError() .
 
     ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
     Why ? : making this distinction requires a header.
     Header management is intentionally delegated to the user layer, which can better manage special cases.
 */
 FSE_PUBLIC_API size_t FSE_decompress(void* dst,  size_t dstCapacity,
                                const void* cSrc, size_t cSrcSize);
 
 
 /*-*****************************************
 *  Tool functions
 ******************************************/
 FSE_PUBLIC_API size_t FSE_compressBound(size_t size);       /* maximum compressed size */
 
 /* Error Management */
 FSE_PUBLIC_API unsigned    FSE_isError(size_t code);        /* tells if a return value is an error code */
 FSE_PUBLIC_API const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
 
 
 /*-*****************************************
 *  FSE advanced functions
 ******************************************/
 /*! FSE_compress2() :
     Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
     Both parameters can be defined as '0' to mean : use default value
     @return : size of compressed data
     Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
                      if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
                      if FSE_isError(return), it's an error code.
 */
 FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
 
 
 /*-*****************************************
 *  FSE detailed API
 ******************************************/
 /*!
 FSE_compress() does the following:
 1. count symbol occurrence from source[] into table count[] (see hist.h)
 2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
 3. save normalized counters to memory buffer using writeNCount()
 4. build encoding table 'CTable' from normalized counters
 5. encode the data stream using encoding table 'CTable'
 
 FSE_decompress() does the following:
 1. read normalized counters with readNCount()
 2. build decoding table 'DTable' from normalized counters
 3. decode the data stream using decoding table 'DTable'
 
 The following API allows targeting specific sub-functions for advanced tasks.
 For example, it's possible to compress several blocks using the same 'CTable',
 or to save and provide normalized distribution using external method.
 */
 
 /* *** COMPRESSION *** */
 
 /*! FSE_optimalTableLog():
     dynamically downsize 'tableLog' when conditions are met.
     It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
     @return : recommended tableLog (necessarily <= 'maxTableLog') */
 FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
 
 /*! FSE_normalizeCount():
     normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
     'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
     @return : tableLog,
               or an errorCode, which can be tested using FSE_isError() */
 FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
                     const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
 
 /*! FSE_NCountWriteBound():
     Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
     Typically useful for allocation purpose. */
 FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_writeNCount():
     Compactly save 'normalizedCounter' into 'buffer'.
     @return : size of the compressed table,
               or an errorCode, which can be tested using FSE_isError(). */
 FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
                                  const short* normalizedCounter,
                                  unsigned maxSymbolValue, unsigned tableLog);
 
 /*! Constructor and Destructor of FSE_CTable.
     Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
 typedef unsigned FSE_CTable;   /* don't allocate that. It's only meant to be more restrictive than void* */
 FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
 FSE_PUBLIC_API void        FSE_freeCTable (FSE_CTable* ct);
 
 /*! FSE_buildCTable():
     Builds `ct`, which must be already allocated, using FSE_createCTable().
     @return : 0, or an errorCode, which can be tested using FSE_isError() */
 FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_compress_usingCTable():
     Compress `src` using `ct` into `dst` which must be already allocated.
     @return : size of compressed data (<= `dstCapacity`),
               or 0 if compressed data could not fit into `dst`,
               or an errorCode, which can be tested using FSE_isError() */
 FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
 
 /*!
 Tutorial :
 ----------
 The first step is to count all symbols. FSE_count() does this job very fast.
 Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
 'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
 maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
 FSE_count() will return the number of occurrence of the most frequent symbol.
 This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
 If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
 
 The next step is to normalize the frequencies.
 FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
 It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
 You can use 'tableLog'==0 to mean "use default tableLog value".
 If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
 which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
 
 The result of FSE_normalizeCount() will be saved into a table,
 called 'normalizedCounter', which is a table of signed short.
 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
 The return value is tableLog if everything proceeded as expected.
 It is 0 if there is a single symbol within distribution.
 If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
 
 'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
 'buffer' must be already allocated.
 For guaranteed success, buffer size must be at least FSE_headerBound().
 The result of the function is the number of bytes written into 'buffer'.
 If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
 
 'normalizedCounter' can then be used to create the compression table 'CTable'.
 The space required by 'CTable' must be already allocated, using FSE_createCTable().
 You can then use FSE_buildCTable() to fill 'CTable'.
 If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
 
 'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
 Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
 The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
 If it returns '0', compressed data could not fit into 'dst'.
 If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
 */
 
 
 /* *** DECOMPRESSION *** */
 
 /*! FSE_readNCount():
     Read compactly saved 'normalizedCounter' from 'rBuffer'.
     @return : size read from 'rBuffer',
               or an errorCode, which can be tested using FSE_isError().
               maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
 FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
                            unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
                            const void* rBuffer, size_t rBuffSize);
 
 /*! Constructor and Destructor of FSE_DTable.
     Note that its size depends on 'tableLog' */
 typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be more restrictive than void* */
 FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
 FSE_PUBLIC_API void        FSE_freeDTable(FSE_DTable* dt);
 
 /*! FSE_buildDTable():
     Builds 'dt', which must be already allocated, using FSE_createDTable().
     return : 0, or an errorCode, which can be tested using FSE_isError() */
 FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_decompress_usingDTable():
     Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
     into `dst` which must be already allocated.
     @return : size of regenerated data (necessarily <= `dstCapacity`),
               or an errorCode, which can be tested using FSE_isError() */
 FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
 
 /*!
 Tutorial :
 ----------
 (Note : these functions only decompress FSE-compressed blocks.
  If block is uncompressed, use memcpy() instead
  If block is a single repeated byte, use memset() instead )
 
 The first step is to obtain the normalized frequencies of symbols.
 This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
 In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
 or size the table to handle worst case situations (typically 256).
 FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
 The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
 Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
 If there is an error, the function will return an error code, which can be tested using FSE_isError().
 
 The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
 This is performed by the function FSE_buildDTable().
 The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
 If there is an error, the function will return an error code, which can be tested using FSE_isError().
 
 `FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
 `cSrcSize` must be strictly correct, otherwise decompression will fail.
 FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
 If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
 */
 
 #endif  /* FSE_H */
 
 #if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
 #define FSE_H_FSE_STATIC_LINKING_ONLY
 
 /* *** Dependency *** */
 #include "bitstream.h"
 
 
 /* *****************************************
 *  Static allocation
 *******************************************/
 /* FSE buffer bounds */
 #define FSE_NCOUNTBOUND 512
 #define FSE_BLOCKBOUND(size) (size + (size>>7))
 #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size))   /* Macro version, useful for static allocation */
 
 /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
 #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue)   (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
 #define FSE_DTABLE_SIZE_U32(maxTableLog)                   (1 + (1<<maxTableLog))
 
 /* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
 #define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue)   (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
 #define FSE_DTABLE_SIZE(maxTableLog)                   (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
 
 
 /* *****************************************
  *  FSE advanced API
  ***************************************** */
 
 unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
 /**< same as FSE_optimalTableLog(), which used `minus==2` */
 
 /* FSE_compress_wksp() :
  * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
  * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
  */
 #define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)   ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
 size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
 
 size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
 /**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
 
 size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
 /**< build a fake FSE_CTable, designed to compress always the same symbolValue */
 
 /* FSE_buildCTable_wksp() :
  * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
  * `wkspSize` must be >= `(1<<tableLog)`.
  */
 size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
 
 size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
 /**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
 
 size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
 /**< build a fake FSE_DTable, designed to always generate the same symbolValue */
 
 size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog);
 /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
 
 typedef enum {
    FSE_repeat_none,  /**< Cannot use the previous table */
    FSE_repeat_check, /**< Can use the previous table but it must be checked */
-   FSE_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+   FSE_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
  } FSE_repeat;
 
 /* *****************************************
 *  FSE symbol compression API
 *******************************************/
 /*!
    This API consists of small unitary functions, which highly benefit from being inlined.
    Hence their body are included in next section.
 */
 typedef struct {
     ptrdiff_t   value;
     const void* stateTable;
     const void* symbolTT;
     unsigned    stateLog;
 } FSE_CState_t;
 
 static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
 
 static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
 
 static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
 
 /**<
 These functions are inner components of FSE_compress_usingCTable().
 They allow the creation of custom streams, mixing multiple tables and bit sources.
 
 A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
 So the first symbol you will encode is the last you will decode, like a LIFO stack.
 
 You will need a few variables to track your CStream. They are :
 
 FSE_CTable    ct;         // Provided by FSE_buildCTable()
 BIT_CStream_t bitStream;  // bitStream tracking structure
 FSE_CState_t  state;      // State tracking structure (can have several)
 
 
 The first thing to do is to init bitStream and state.
     size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
     FSE_initCState(&state, ct);
 
 Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
 You can then encode your input data, byte after byte.
 FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
 Remember decoding will be done in reverse direction.
     FSE_encodeByte(&bitStream, &state, symbol);
 
 At any time, you can also add any bit sequence.
 Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
     BIT_addBits(&bitStream, bitField, nbBits);
 
 The above methods don't commit data to memory, they just store it into local register, for speed.
 Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
 Writing data to memory is a manual operation, performed by the flushBits function.
     BIT_flushBits(&bitStream);
 
 Your last FSE encoding operation shall be to flush your last state value(s).
     FSE_flushState(&bitStream, &state);
 
 Finally, you must close the bitStream.
 The function returns the size of CStream in bytes.
 If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
 If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
     size_t size = BIT_closeCStream(&bitStream);
 */
 
 
 /* *****************************************
 *  FSE symbol decompression API
 *******************************************/
 typedef struct {
     size_t      state;
     const void* table;   /* precise table may vary, depending on U16 */
 } FSE_DState_t;
 
 
 static void     FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
 
 static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
 
 static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
 
 /**<
 Let's now decompose FSE_decompress_usingDTable() into its unitary components.
 You will decode FSE-encoded symbols from the bitStream,
 and also any other bitFields you put in, **in reverse order**.
 
 You will need a few variables to track your bitStream. They are :
 
 BIT_DStream_t DStream;    // Stream context
 FSE_DState_t  DState;     // State context. Multiple ones are possible
 FSE_DTable*   DTablePtr;  // Decoding table, provided by FSE_buildDTable()
 
 The first thing to do is to init the bitStream.
     errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
 
 You should then retrieve your initial state(s)
 (in reverse flushing order if you have several ones) :
     errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
 
 You can then decode your data, symbol after symbol.
 For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
 Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
     unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
 
 You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
 Note : maximum allowed nbBits is 25, for 32-bits compatibility
     size_t bitField = BIT_readBits(&DStream, nbBits);
 
 All above operations only read from local register (which size depends on size_t).
 Refueling the register from memory is manually performed by the reload method.
     endSignal = FSE_reloadDStream(&DStream);
 
 BIT_reloadDStream() result tells if there is still some more data to read from DStream.
 BIT_DStream_unfinished : there is still some data left into the DStream.
 BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
 BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
 BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
 
 When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
 to properly detect the exact end of stream.
 After each decoded symbol, check if DStream is fully consumed using this simple test :
     BIT_reloadDStream(&DStream) >= BIT_DStream_completed
 
 When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
 Checking if DStream has reached its end is performed by :
     BIT_endOfDStream(&DStream);
 Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
     FSE_endOfDState(&DState);
 */
 
 
 /* *****************************************
 *  FSE unsafe API
 *******************************************/
 static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
 /* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
 
 
 /* *****************************************
 *  Implementation of inlined functions
 *******************************************/
 typedef struct {
     int deltaFindState;
     U32 deltaNbBits;
 } FSE_symbolCompressionTransform; /* total 8 bytes */
 
 MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
 {
     const void* ptr = ct;
     const U16* u16ptr = (const U16*) ptr;
     const U32 tableLog = MEM_read16(ptr);
     statePtr->value = (ptrdiff_t)1<<tableLog;
     statePtr->stateTable = u16ptr+2;
     statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
     statePtr->stateLog = tableLog;
 }
 
 
 /*! FSE_initCState2() :
 *   Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
 *   uses the smallest state value possible, saving the cost of this symbol */
 MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
 {
     FSE_initCState(statePtr, ct);
     {   const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
         const U16* stateTable = (const U16*)(statePtr->stateTable);
         U32 nbBitsOut  = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
         statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
         statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
     }
 }
 
 MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
 {
     FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
     const U16* const stateTable = (const U16*)(statePtr->stateTable);
     U32 const nbBitsOut  = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
     BIT_addBits(bitC, statePtr->value, nbBitsOut);
     statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
 }
 
 MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
 {
     BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
     BIT_flushBits(bitC);
 }
 
 
 /* FSE_getMaxNbBits() :
  * Approximate maximum cost of a symbol, in bits.
  * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
  * note 1 : assume symbolValue is valid (<= maxSymbolValue)
  * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
 MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
 {
     const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
     return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
 }
 
 /* FSE_bitCost() :
  * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
  * note 1 : assume symbolValue is valid (<= maxSymbolValue)
  * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
 MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
 {
     const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
     U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
     U32 const threshold = (minNbBits+1) << 16;
     assert(tableLog < 16);
     assert(accuracyLog < 31-tableLog);  /* ensure enough room for renormalization double shift */
     {   U32 const tableSize = 1 << tableLog;
         U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
         U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog;   /* linear interpolation (very approximate) */
         U32 const bitMultiplier = 1 << accuracyLog;
         assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
         assert(normalizedDeltaFromThreshold <= bitMultiplier);
         return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
     }
 }
 
 
 /* ======    Decompression    ====== */
 
 typedef struct {
     U16 tableLog;
     U16 fastMode;
 } FSE_DTableHeader;   /* sizeof U32 */
 
 typedef struct
 {
     unsigned short newState;
     unsigned char  symbol;
     unsigned char  nbBits;
 } FSE_decode_t;   /* size == U32 */
 
 MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
 {
     const void* ptr = dt;
     const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
     DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
     BIT_reloadDStream(bitD);
     DStatePtr->table = dt + 1;
 }
 
 MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
 {
     FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
     return DInfo.symbol;
 }
 
 MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
 {
     FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
     U32 const nbBits = DInfo.nbBits;
     size_t const lowBits = BIT_readBits(bitD, nbBits);
     DStatePtr->state = DInfo.newState + lowBits;
 }
 
 MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
 {
     FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
     U32 const nbBits = DInfo.nbBits;
     BYTE const symbol = DInfo.symbol;
     size_t const lowBits = BIT_readBits(bitD, nbBits);
 
     DStatePtr->state = DInfo.newState + lowBits;
     return symbol;
 }
 
 /*! FSE_decodeSymbolFast() :
     unsafe, only works if no symbol has a probability > 50% */
 MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
 {
     FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
     U32 const nbBits = DInfo.nbBits;
     BYTE const symbol = DInfo.symbol;
     size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
 
     DStatePtr->state = DInfo.newState + lowBits;
     return symbol;
 }
 
 MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
 {
     return DStatePtr->state == 0;
 }
 
 
 
 #ifndef FSE_COMMONDEFS_ONLY
 
 /* **************************************************************
 *  Tuning parameters
 ****************************************************************/
 /*!MEMORY_USAGE :
 *  Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
 *  Increasing memory usage improves compression ratio
 *  Reduced memory usage can improve speed, due to cache effect
 *  Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
 #ifndef FSE_MAX_MEMORY_USAGE
 #  define FSE_MAX_MEMORY_USAGE 14
 #endif
 #ifndef FSE_DEFAULT_MEMORY_USAGE
 #  define FSE_DEFAULT_MEMORY_USAGE 13
 #endif
 
 /*!FSE_MAX_SYMBOL_VALUE :
 *  Maximum symbol value authorized.
 *  Required for proper stack allocation */
 #ifndef FSE_MAX_SYMBOL_VALUE
 #  define FSE_MAX_SYMBOL_VALUE 255
 #endif
 
 /* **************************************************************
 *  template functions type & suffix
 ****************************************************************/
 #define FSE_FUNCTION_TYPE BYTE
 #define FSE_FUNCTION_EXTENSION
 #define FSE_DECODE_TYPE FSE_decode_t
 
 
 #endif   /* !FSE_COMMONDEFS_ONLY */
 
 
 /* ***************************************************************
 *  Constants
 *****************************************************************/
 #define FSE_MAX_TABLELOG  (FSE_MAX_MEMORY_USAGE-2)
 #define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
 #define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
 #define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
 #define FSE_MIN_TABLELOG 5
 
 #define FSE_TABLELOG_ABSOLUTE_MAX 15
 #if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
 #  error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
 #endif
 
 #define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
 
 
 #endif /* FSE_STATIC_LINKING_ONLY */
 
 
 #if defined (__cplusplus)
 }
 #endif

contrib/python-zstandard/zstd/common/mem.h

@@ -1,380 +1,380 b''
 /*
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
  * You may select, at your option, one of the above-listed licenses.
  */
 
 #ifndef MEM_H_MODULE
 #define MEM_H_MODULE
 
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
 /*-****************************************
 *  Dependencies
 ******************************************/
 #include <stddef.h>     /* size_t, ptrdiff_t */
 #include <string.h>     /* memcpy */
 
 
 /*-****************************************
 *  Compiler specifics
 ******************************************/
 #if defined(_MSC_VER)   /* Visual Studio */
 #   include <stdlib.h>  /* _byteswap_ulong */
 #   include <intrin.h>  /* _byteswap_* */
 #endif
 #if defined(__GNUC__)
 #  define MEM_STATIC static __inline __attribute__((unused))
 #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #  define MEM_STATIC static inline
 #elif defined(_MSC_VER)
 #  define MEM_STATIC static __inline
 #else
 #  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
 #endif
 
 #ifndef __has_builtin
 #  define __has_builtin(x) 0  /* compat. with non-clang compilers */
 #endif
 
 /* code only tested on 32 and 64 bits systems */
 #define MEM_STATIC_ASSERT(c)   { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
 MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
 
 
 /*-**************************************************************
 *  Basic Types
 *****************************************************************/
 #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include <stdint.h>
   typedef   uint8_t BYTE;
   typedef  uint16_t U16;
   typedef   int16_t S16;
   typedef  uint32_t U32;
   typedef   int32_t S32;
   typedef  uint64_t U64;
   typedef   int64_t S64;
 #else
 # include <limits.h>
 #if CHAR_BIT != 8
 #  error "this implementation requires char to be exactly 8-bit type"
 #endif
   typedef unsigned char      BYTE;
 #if USHRT_MAX != 65535
 #  error "this implementation requires short to be exactly 16-bit type"
 #endif
   typedef unsigned short      U16;
   typedef   signed short      S16;
 #if UINT_MAX != 4294967295
 #  error "this implementation requires int to be exactly 32-bit type"
 #endif
   typedef unsigned int        U32;
   typedef   signed int        S32;
 /* note : there are no limits defined for long long type in C90.
  * limits exist in C99, however, in such case, <stdint.h> is preferred */
   typedef unsigned long long  U64;
   typedef   signed long long  S64;
 #endif
 
 
 /*-**************************************************************
 *  Memory I/O
 *****************************************************************/
 /* MEM_FORCE_MEMORY_ACCESS :
  * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
  * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
  * The below switch allow to select different access method for improved performance.
  * Method 0 (default) : use `memcpy()`. Safe and portable.
  * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
  *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
  *            In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
  * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || defined(__GNUC__)
+#  elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__)
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
 
 MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
 MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
 
 MEM_STATIC unsigned MEM_isLittleEndian(void)
 {
     const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
     return one.c[0];
 }
 
 #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
 
 /* violates C standard, by lying on structure alignment.
 Only use if no other choice to achieve best performance on target platform */
 MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
 MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
 MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
 MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
 MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
 MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
 
 #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
 
 /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
 /* currently only defined for gcc and icc */
 #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
     __pragma( pack(push, 1) )
     typedef struct { U16 v; } unalign16;
     typedef struct { U32 v; } unalign32;
     typedef struct { U64 v; } unalign64;
     typedef struct { size_t v; } unalignArch;
     __pragma( pack(pop) )
 #else
     typedef struct { U16 v; } __attribute__((packed)) unalign16;
     typedef struct { U32 v; } __attribute__((packed)) unalign32;
     typedef struct { U64 v; } __attribute__((packed)) unalign64;
     typedef struct { size_t v; } __attribute__((packed)) unalignArch;
 #endif
 
 MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
 MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
 MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
 MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
 MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
 MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
 
 #else
 
 /* default method, safe and standard.
    can sometimes prove slower */
 
 MEM_STATIC U16 MEM_read16(const void* memPtr)
 {
     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
 }
 
 MEM_STATIC U32 MEM_read32(const void* memPtr)
 {
     U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
 }
 
 MEM_STATIC U64 MEM_read64(const void* memPtr)
 {
     U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
 }
 
 MEM_STATIC size_t MEM_readST(const void* memPtr)
 {
     size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
 }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value)
 {
     memcpy(memPtr, &value, sizeof(value));
 }
 
 MEM_STATIC void MEM_write32(void* memPtr, U32 value)
 {
     memcpy(memPtr, &value, sizeof(value));
 }
 
 MEM_STATIC void MEM_write64(void* memPtr, U64 value)
 {
     memcpy(memPtr, &value, sizeof(value));
 }
 
 #endif /* MEM_FORCE_MEMORY_ACCESS */
 
 MEM_STATIC U32 MEM_swap32(U32 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
     return _byteswap_ulong(in);
 #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
   || (defined(__clang__) && __has_builtin(__builtin_bswap32))
     return __builtin_bswap32(in);
 #else
     return  ((in << 24) & 0xff000000 ) |
             ((in <<  8) & 0x00ff0000 ) |
             ((in >>  8) & 0x0000ff00 ) |
             ((in >> 24) & 0x000000ff );
 #endif
 }
 
 MEM_STATIC U64 MEM_swap64(U64 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
     return _byteswap_uint64(in);
 #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
   || (defined(__clang__) && __has_builtin(__builtin_bswap64))
     return __builtin_bswap64(in);
 #else
     return  ((in << 56) & 0xff00000000000000ULL) |
             ((in << 40) & 0x00ff000000000000ULL) |
             ((in << 24) & 0x0000ff0000000000ULL) |
             ((in << 8)  & 0x000000ff00000000ULL) |
             ((in >> 8)  & 0x00000000ff000000ULL) |
             ((in >> 24) & 0x0000000000ff0000ULL) |
             ((in >> 40) & 0x000000000000ff00ULL) |
             ((in >> 56) & 0x00000000000000ffULL);
 #endif
 }
 
 MEM_STATIC size_t MEM_swapST(size_t in)
 {
     if (MEM_32bits())
         return (size_t)MEM_swap32((U32)in);
     else
         return (size_t)MEM_swap64((U64)in);
 }
 
 /*=== Little endian r/w ===*/
 
 MEM_STATIC U16 MEM_readLE16(const void* memPtr)
 {
     if (MEM_isLittleEndian())
         return MEM_read16(memPtr);
     else {
         const BYTE* p = (const BYTE*)memPtr;
         return (U16)(p[0] + (p[1]<<8));
     }
 }
 
 MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
 {
     if (MEM_isLittleEndian()) {
         MEM_write16(memPtr, val);
     } else {
         BYTE* p = (BYTE*)memPtr;
         p[0] = (BYTE)val;
         p[1] = (BYTE)(val>>8);
     }
 }
 
 MEM_STATIC U32 MEM_readLE24(const void* memPtr)
 {
     return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
 }
 
 MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
 {
     MEM_writeLE16(memPtr, (U16)val);
     ((BYTE*)memPtr)[2] = (BYTE)(val>>16);
 }
 
 MEM_STATIC U32 MEM_readLE32(const void* memPtr)
 {
     if (MEM_isLittleEndian())
         return MEM_read32(memPtr);
     else
         return MEM_swap32(MEM_read32(memPtr));
 }
 
 MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
 {
     if (MEM_isLittleEndian())
         MEM_write32(memPtr, val32);
     else
         MEM_write32(memPtr, MEM_swap32(val32));
 }
 
 MEM_STATIC U64 MEM_readLE64(const void* memPtr)
 {
     if (MEM_isLittleEndian())
         return MEM_read64(memPtr);
     else
         return MEM_swap64(MEM_read64(memPtr));
 }
 
 MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
 {
     if (MEM_isLittleEndian())
         MEM_write64(memPtr, val64);
     else
         MEM_write64(memPtr, MEM_swap64(val64));
 }
 
 MEM_STATIC size_t MEM_readLEST(const void* memPtr)
 {
     if (MEM_32bits())
         return (size_t)MEM_readLE32(memPtr);
     else
         return (size_t)MEM_readLE64(memPtr);
 }
 
 MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
 {
     if (MEM_32bits())
         MEM_writeLE32(memPtr, (U32)val);
     else
         MEM_writeLE64(memPtr, (U64)val);
 }
 
 /*=== Big endian r/w ===*/
 
 MEM_STATIC U32 MEM_readBE32(const void* memPtr)
 {
     if (MEM_isLittleEndian())
         return MEM_swap32(MEM_read32(memPtr));
     else
         return MEM_read32(memPtr);
 }
 
 MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
 {
     if (MEM_isLittleEndian())
         MEM_write32(memPtr, MEM_swap32(val32));
     else
         MEM_write32(memPtr, val32);
 }
 
 MEM_STATIC U64 MEM_readBE64(const void* memPtr)
 {
     if (MEM_isLittleEndian())
         return MEM_swap64(MEM_read64(memPtr));
     else
         return MEM_read64(memPtr);
 }
 
 MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
 {
     if (MEM_isLittleEndian())
         MEM_write64(memPtr, MEM_swap64(val64));
     else
         MEM_write64(memPtr, val64);
 }
 
 MEM_STATIC size_t MEM_readBEST(const void* memPtr)
 {
     if (MEM_32bits())
         return (size_t)MEM_readBE32(memPtr);
     else
         return (size_t)MEM_readBE64(memPtr);
 }
 
 MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
 {
     if (MEM_32bits())
         MEM_writeBE32(memPtr, (U32)val);
     else
         MEM_writeBE64(memPtr, (U64)val);
 }
 
 
 #if defined (__cplusplus)
 }
 #endif
 
 #endif /* MEM_H_MODULE */

contrib/python-zstandard/zstd/common/threading.c

@@ -1,75 +1,75 b''
 /**
  * Copyright (c) 2016 Tino Reichardt
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
  *
  * You can contact the author at:
  * - zstdmt source repository: https://github.com/mcmilk/zstdmt
  */
 
 /**
  * This file will hold wrapper for systems, which do not support pthreads
  */
 
-/* create fake symbol to avoid empty trnaslation unit warning */
-int g_ZSTD_threading_useles_symbol;
+/* create fake symbol to avoid empty translation unit warning */
+int g_ZSTD_threading_useless_symbol;
 
 #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
 
 /**
  * Windows minimalist Pthread Wrapper, based on :
  * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
  */
 
 
 /* ===  Dependencies  === */
 #include <process.h>
 #include <errno.h>
 #include "threading.h"
 
 
 /* ===  Implementation  === */
 
 static unsigned __stdcall worker(void *arg)
 {
     ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
     thread->arg = thread->start_routine(thread->arg);
     return 0;
 }
 
 int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
             void* (*start_routine) (void*), void* arg)
 {
     (void)unused;
     thread->arg = arg;
     thread->start_routine = start_routine;
     thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
 
     if (!thread->handle)
         return errno;
     else
         return 0;
 }
 
 int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
 {
     DWORD result;
 
     if (!thread.handle) return 0;
 
     result = WaitForSingleObject(thread.handle, INFINITE);
     switch (result) {
     case WAIT_OBJECT_0:
         if (value_ptr) *value_ptr = thread.arg;
         return 0;
     case WAIT_ABANDONED:
         return EINVAL;
     default:
         return GetLastError();
     }
 }
 
 #endif   /* ZSTD_MULTITHREAD */

contrib/python-zstandard/zstd/common/xxhash.c

@@ -1,876 +1,882 b''
 /*
 *  xxHash - Fast Hash algorithm
 *  Copyright (C) 2012-2016, Yann Collet
 *
 *  BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are
 *  met:
 *
 *  * Redistributions of source code must retain the above copyright
 *  notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above
 *  copyright notice, this list of conditions and the following disclaimer
 *  in the documentation and/or other materials provided with the
 *  distribution.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *  You can contact the author at :
 *  - xxHash homepage: http://www.xxhash.com
 *  - xxHash source repository : https://github.com/Cyan4973/xxHash
 */
 
 
 /* *************************************
 *  Tuning parameters
 ***************************************/
 /*!XXH_FORCE_MEMORY_ACCESS :
  * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
  * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
  * The below switch allow to select different access method for improved performance.
  * Method 0 (default) : use `memcpy()`. Safe and portable.
  * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
  *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
  * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
  *            It can generate buggy code on targets which do not support unaligned memory accesses.
  *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
  * See http://stackoverflow.com/a/32095106/646947 for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define XXH_FORCE_MEMORY_ACCESS 2
 #  elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \
-  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \
+  defined(__ICCARM__)
 #    define XXH_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
 
 /*!XXH_ACCEPT_NULL_INPUT_POINTER :
  * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
  * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
  * By default, this option is disabled. To enable it, uncomment below define :
  */
 /* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
 
 /*!XXH_FORCE_NATIVE_FORMAT :
- * By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
+ * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
  * Results are therefore identical for little-endian and big-endian CPU.
  * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
- * Should endian-independance be of no importance for your application, you may set the #define below to 1,
+ * Should endian-independence be of no importance for your application, you may set the #define below to 1,
  * to improve speed for Big-endian CPU.
  * This option has no impact on Little_Endian CPU.
  */
 #ifndef XXH_FORCE_NATIVE_FORMAT   /* can be defined externally */
 #  define XXH_FORCE_NATIVE_FORMAT 0
 #endif
 
 /*!XXH_FORCE_ALIGN_CHECK :
  * This is a minor performance trick, only useful with lots of very small keys.
  * It means : check for aligned/unaligned input.
  * The check costs one initial branch per hash; set to 0 when the input data
  * is guaranteed to be aligned.
  */
 #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
 #  if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
 #    define XXH_FORCE_ALIGN_CHECK 0
 #  else
 #    define XXH_FORCE_ALIGN_CHECK 1
 #  endif
 #endif
 
 
 /* *************************************
 *  Includes & Memory related functions
 ***************************************/
 /* Modify the local functions below should you wish to use some other memory routines */
 /* for malloc(), free() */
 #include <stdlib.h>
 #include <stddef.h>     /* size_t */
 static void* XXH_malloc(size_t s) { return malloc(s); }
 static void  XXH_free  (void* p)  { free(p); }
 /* for memcpy() */
 #include <string.h>
 static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
 
 #ifndef XXH_STATIC_LINKING_ONLY
 #  define XXH_STATIC_LINKING_ONLY
 #endif
 #include "xxhash.h"
 
 
 /* *************************************
 *  Compiler Specific Options
 ***************************************/
 #if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #  define INLINE_KEYWORD inline
 #else
 #  define INLINE_KEYWORD
 #endif
 
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__ICCARM__)
 #  define FORCE_INLINE_ATTR __attribute__((always_inline))
 #elif defined(_MSC_VER)
 #  define FORCE_INLINE_ATTR __forceinline
 #else
 #  define FORCE_INLINE_ATTR
 #endif
 
 #define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
 
 
 #ifdef _MSC_VER
 #  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
 #endif
 
 
 /* *************************************
 *  Basic Types
 ***************************************/
 #ifndef MEM_MODULE
 # define MEM_MODULE
 # if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 #   include <stdint.h>
     typedef uint8_t  BYTE;
     typedef uint16_t U16;
     typedef uint32_t U32;
     typedef  int32_t S32;
     typedef uint64_t U64;
 #  else
     typedef unsigned char      BYTE;
     typedef unsigned short     U16;
     typedef unsigned int       U32;
     typedef   signed int       S32;
     typedef unsigned long long U64;   /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
 #  endif
 #endif
 
 
 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
 
 /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
 static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
 static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
 
 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
 
 /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
 /* currently only defined for gcc and icc */
 typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign;
 
 static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
 static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
 
 #else
 
 /* portable and safe solution. Generally efficient.
  * see : http://stackoverflow.com/a/32095106/646947
  */
 
 static U32 XXH_read32(const void* memPtr)
 {
     U32 val;
     memcpy(&val, memPtr, sizeof(val));
     return val;
 }
 
 static U64 XXH_read64(const void* memPtr)
 {
     U64 val;
     memcpy(&val, memPtr, sizeof(val));
     return val;
 }
 
 #endif   /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
 
 
 /* ****************************************
 *  Compiler-specific Functions and Macros
 ******************************************/
 #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
 
 /* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
 #if defined(_MSC_VER)
 #  define XXH_rotl32(x,r) _rotl(x,r)
 #  define XXH_rotl64(x,r) _rotl64(x,r)
 #else
+#if defined(__ICCARM__)
+#  include <intrinsics.h>
+#  define XXH_rotl32(x,r) __ROR(x,(32 - r))
+#else
 #  define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#endif
 #  define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
 #endif
 
 #if defined(_MSC_VER)     /* Visual Studio */
 #  define XXH_swap32 _byteswap_ulong
 #  define XXH_swap64 _byteswap_uint64
 #elif GCC_VERSION >= 403
 #  define XXH_swap32 __builtin_bswap32
 #  define XXH_swap64 __builtin_bswap64
 #else
 static U32 XXH_swap32 (U32 x)
 {
     return  ((x << 24) & 0xff000000 ) |
             ((x <<  8) & 0x00ff0000 ) |
             ((x >>  8) & 0x0000ff00 ) |
             ((x >> 24) & 0x000000ff );
 }
 static U64 XXH_swap64 (U64 x)
 {
     return  ((x << 56) & 0xff00000000000000ULL) |
             ((x << 40) & 0x00ff000000000000ULL) |
             ((x << 24) & 0x0000ff0000000000ULL) |
             ((x << 8)  & 0x000000ff00000000ULL) |
             ((x >> 8)  & 0x00000000ff000000ULL) |
             ((x >> 24) & 0x0000000000ff0000ULL) |
             ((x >> 40) & 0x000000000000ff00ULL) |
             ((x >> 56) & 0x00000000000000ffULL);
 }
 #endif
 
 
 /* *************************************
 *  Architecture Macros
 ***************************************/
 typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
 
 /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
 #ifndef XXH_CPU_LITTLE_ENDIAN
     static const int g_one = 1;
 #   define XXH_CPU_LITTLE_ENDIAN   (*(const char*)(&g_one))
 #endif
 
 
 /* ***************************
 *  Memory reads
 *****************************/
 typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
 
 FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
 {
     if (align==XXH_unaligned)
         return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
     else
         return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
 }
 
 FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
 {
     return XXH_readLE32_align(ptr, endian, XXH_unaligned);
 }
 
 static U32 XXH_readBE32(const void* ptr)
 {
     return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
 }
 
 FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
 {
     if (align==XXH_unaligned)
         return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
     else
         return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
 }
 
 FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
 {
     return XXH_readLE64_align(ptr, endian, XXH_unaligned);
 }
 
 static U64 XXH_readBE64(const void* ptr)
 {
     return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
 }
 
 
 /* *************************************
 *  Macros
 ***************************************/
 #define XXH_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(int)(!!(c)) }; }    /* use only *after* variable declarations */
 
 
 /* *************************************
 *  Constants
 ***************************************/
 static const U32 PRIME32_1 = 2654435761U;
 static const U32 PRIME32_2 = 2246822519U;
 static const U32 PRIME32_3 = 3266489917U;
 static const U32 PRIME32_4 =  668265263U;
 static const U32 PRIME32_5 =  374761393U;
 
 static const U64 PRIME64_1 = 11400714785074694791ULL;
 static const U64 PRIME64_2 = 14029467366897019727ULL;
 static const U64 PRIME64_3 =  1609587929392839161ULL;
 static const U64 PRIME64_4 =  9650029242287828579ULL;
 static const U64 PRIME64_5 =  2870177450012600261ULL;
 
 XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
 
 
 /* **************************
 *  Utils
 ****************************/
 XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState)
 {
     memcpy(dstState, srcState, sizeof(*dstState));
 }
 
 XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState)
 {
     memcpy(dstState, srcState, sizeof(*dstState));
 }
 
 
 /* ***************************
 *  Simple Hash Functions
 *****************************/
 
 static U32 XXH32_round(U32 seed, U32 input)
 {
     seed += input * PRIME32_2;
     seed  = XXH_rotl32(seed, 13);
     seed *= PRIME32_1;
     return seed;
 }
 
 FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
 {
     const BYTE* p = (const BYTE*)input;
     const BYTE* bEnd = p + len;
     U32 h32;
 #define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
 
 #ifdef XXH_ACCEPT_NULL_INPUT_POINTER
     if (p==NULL) {
         len=0;
         bEnd=p=(const BYTE*)(size_t)16;
     }
 #endif
 
     if (len>=16) {
         const BYTE* const limit = bEnd - 16;
         U32 v1 = seed + PRIME32_1 + PRIME32_2;
         U32 v2 = seed + PRIME32_2;
         U32 v3 = seed + 0;
         U32 v4 = seed - PRIME32_1;
 
         do {
             v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
             v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
             v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
             v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
         } while (p<=limit);
 
         h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
     } else {
         h32  = seed + PRIME32_5;
     }
 
     h32 += (U32) len;
 
     while (p+4<=bEnd) {
         h32 += XXH_get32bits(p) * PRIME32_3;
         h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
         p+=4;
     }
 
     while (p<bEnd) {
         h32 += (*p) * PRIME32_5;
         h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
         p++;
     }
 
     h32 ^= h32 >> 15;
     h32 *= PRIME32_2;
     h32 ^= h32 >> 13;
     h32 *= PRIME32_3;
     h32 ^= h32 >> 16;
 
     return h32;
 }
 
 
 XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
 {
 #if 0
     /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
     XXH32_CREATESTATE_STATIC(state);
     XXH32_reset(state, seed);
     XXH32_update(state, input, len);
     return XXH32_digest(state);
 #else
     XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
 
     if (XXH_FORCE_ALIGN_CHECK) {
         if ((((size_t)input) & 3) == 0) {   /* Input is 4-bytes aligned, leverage the speed benefit */
             if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
                 return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
             else
                 return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
     }   }
 
     if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
         return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
     else
         return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
 #endif
 }
 
 
 static U64 XXH64_round(U64 acc, U64 input)
 {
     acc += input * PRIME64_2;
     acc  = XXH_rotl64(acc, 31);
     acc *= PRIME64_1;
     return acc;
 }
 
 static U64 XXH64_mergeRound(U64 acc, U64 val)
 {
     val  = XXH64_round(0, val);
     acc ^= val;
     acc  = acc * PRIME64_1 + PRIME64_4;
     return acc;
 }
 
 FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
 {
     const BYTE* p = (const BYTE*)input;
     const BYTE* const bEnd = p + len;
     U64 h64;
 #define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
 
 #ifdef XXH_ACCEPT_NULL_INPUT_POINTER
     if (p==NULL) {
         len=0;
         bEnd=p=(const BYTE*)(size_t)32;
     }
 #endif
 
     if (len>=32) {
         const BYTE* const limit = bEnd - 32;
         U64 v1 = seed + PRIME64_1 + PRIME64_2;
         U64 v2 = seed + PRIME64_2;
         U64 v3 = seed + 0;
         U64 v4 = seed - PRIME64_1;
 
         do {
             v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
             v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
             v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
             v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
         } while (p<=limit);
 
         h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
         h64 = XXH64_mergeRound(h64, v1);
         h64 = XXH64_mergeRound(h64, v2);
         h64 = XXH64_mergeRound(h64, v3);
         h64 = XXH64_mergeRound(h64, v4);
 
     } else {
         h64  = seed + PRIME64_5;
     }
 
     h64 += (U64) len;
 
     while (p+8<=bEnd) {
         U64 const k1 = XXH64_round(0, XXH_get64bits(p));
         h64 ^= k1;
         h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
         p+=8;
     }
 
     if (p+4<=bEnd) {
         h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
         h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
         p+=4;
     }
 
     while (p<bEnd) {
         h64 ^= (*p) * PRIME64_5;
         h64 = XXH_rotl64(h64, 11) * PRIME64_1;
         p++;
     }
 
     h64 ^= h64 >> 33;
     h64 *= PRIME64_2;
     h64 ^= h64 >> 29;
     h64 *= PRIME64_3;
     h64 ^= h64 >> 32;
 
     return h64;
 }
 
 
 XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
 {
 #if 0
     /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
     XXH64_CREATESTATE_STATIC(state);
     XXH64_reset(state, seed);
     XXH64_update(state, input, len);
     return XXH64_digest(state);
 #else
     XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
 
     if (XXH_FORCE_ALIGN_CHECK) {
         if ((((size_t)input) & 7)==0) {  /* Input is aligned, let's leverage the speed advantage */
             if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
                 return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
             else
                 return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
     }   }
 
     if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
         return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
     else
         return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
 #endif
 }
 
 
 /* **************************************************
 *  Advanced Hash Functions
 ****************************************************/
 
 XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
 {
     return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
 }
 XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
 {
     XXH_free(statePtr);
     return XXH_OK;
 }
 
 XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
 {
     return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
 }
 XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
 {
     XXH_free(statePtr);
     return XXH_OK;
 }
 
 
 /*** Hash feed ***/
 
 XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
 {
     XXH32_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
     memset(&state, 0, sizeof(state)-4);   /* do not write into reserved, for future removal */
     state.v1 = seed + PRIME32_1 + PRIME32_2;
     state.v2 = seed + PRIME32_2;
     state.v3 = seed + 0;
     state.v4 = seed - PRIME32_1;
     memcpy(statePtr, &state, sizeof(state));
     return XXH_OK;
 }
 
 
 XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
 {
     XXH64_state_t state;   /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
     memset(&state, 0, sizeof(state)-8);   /* do not write into reserved, for future removal */
     state.v1 = seed + PRIME64_1 + PRIME64_2;
     state.v2 = seed + PRIME64_2;
     state.v3 = seed + 0;
     state.v4 = seed - PRIME64_1;
     memcpy(statePtr, &state, sizeof(state));
     return XXH_OK;
 }
 
 
 FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
 {
     const BYTE* p = (const BYTE*)input;
     const BYTE* const bEnd = p + len;
 
 #ifdef XXH_ACCEPT_NULL_INPUT_POINTER
     if (input==NULL) return XXH_ERROR;
 #endif
 
     state->total_len_32 += (unsigned)len;
     state->large_len |= (len>=16) | (state->total_len_32>=16);
 
     if (state->memsize + len < 16)  {   /* fill in tmp buffer */
         XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
         state->memsize += (unsigned)len;
         return XXH_OK;
     }
 
     if (state->memsize) {   /* some data left from previous update */
         XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
         {   const U32* p32 = state->mem32;
             state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
             state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
             state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
             state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
         }
         p += 16-state->memsize;
         state->memsize = 0;
     }
 
     if (p <= bEnd-16) {
         const BYTE* const limit = bEnd - 16;
         U32 v1 = state->v1;
         U32 v2 = state->v2;
         U32 v3 = state->v3;
         U32 v4 = state->v4;
 
         do {
             v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
             v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
             v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
             v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
         } while (p<=limit);
 
         state->v1 = v1;
         state->v2 = v2;
         state->v3 = v3;
         state->v4 = v4;
     }
 
     if (p < bEnd) {
         XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
         state->memsize = (unsigned)(bEnd-p);
     }
 
     return XXH_OK;
 }
 
 XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
 {
     XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
 
     if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
         return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
     else
         return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
 }
 
 
 
 FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
 {
     const BYTE * p = (const BYTE*)state->mem32;
     const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
     U32 h32;
 
     if (state->large_len) {
         h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
     } else {
         h32 = state->v3 /* == seed */ + PRIME32_5;
     }
 
     h32 += state->total_len_32;
 
     while (p+4<=bEnd) {
         h32 += XXH_readLE32(p, endian) * PRIME32_3;
         h32  = XXH_rotl32(h32, 17) * PRIME32_4;
         p+=4;
     }
 
     while (p<bEnd) {
         h32 += (*p) * PRIME32_5;
         h32  = XXH_rotl32(h32, 11) * PRIME32_1;
         p++;
     }
 
     h32 ^= h32 >> 15;
     h32 *= PRIME32_2;
     h32 ^= h32 >> 13;
     h32 *= PRIME32_3;
     h32 ^= h32 >> 16;
 
     return h32;
 }
 
 
 XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
 {
     XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
 
     if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
         return XXH32_digest_endian(state_in, XXH_littleEndian);
     else
         return XXH32_digest_endian(state_in, XXH_bigEndian);
 }
 
 
 
 /* **** XXH64 **** */
 
 FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
 {
     const BYTE* p = (const BYTE*)input;
     const BYTE* const bEnd = p + len;
 
 #ifdef XXH_ACCEPT_NULL_INPUT_POINTER
     if (input==NULL) return XXH_ERROR;
 #endif
 
     state->total_len += len;
 
     if (state->memsize + len < 32) {  /* fill in tmp buffer */
         XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
         state->memsize += (U32)len;
         return XXH_OK;
     }
 
     if (state->memsize) {   /* tmp buffer is full */
         XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
         state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
         state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
         state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
         state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
         p += 32-state->memsize;
         state->memsize = 0;
     }
 
     if (p+32 <= bEnd) {
         const BYTE* const limit = bEnd - 32;
         U64 v1 = state->v1;
         U64 v2 = state->v2;
         U64 v3 = state->v3;
         U64 v4 = state->v4;
 
         do {
             v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
             v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
             v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
             v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
         } while (p<=limit);
 
         state->v1 = v1;
         state->v2 = v2;
         state->v3 = v3;
         state->v4 = v4;
     }
 
     if (p < bEnd) {
         XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
         state->memsize = (unsigned)(bEnd-p);
     }
 
     return XXH_OK;
 }
 
 XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
 {
     XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
 
     if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
         return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
     else
         return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
 }
 
 
 
 FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
 {
     const BYTE * p = (const BYTE*)state->mem64;
     const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
     U64 h64;
 
     if (state->total_len >= 32) {
         U64 const v1 = state->v1;
         U64 const v2 = state->v2;
         U64 const v3 = state->v3;
         U64 const v4 = state->v4;
 
         h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
         h64 = XXH64_mergeRound(h64, v1);
         h64 = XXH64_mergeRound(h64, v2);
         h64 = XXH64_mergeRound(h64, v3);
         h64 = XXH64_mergeRound(h64, v4);
     } else {
         h64  = state->v3 + PRIME64_5;
     }
 
     h64 += (U64) state->total_len;
 
     while (p+8<=bEnd) {
         U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
         h64 ^= k1;
         h64  = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
         p+=8;
     }
 
     if (p+4<=bEnd) {
         h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
         h64  = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
         p+=4;
     }
 
     while (p<bEnd) {
         h64 ^= (*p) * PRIME64_5;
         h64  = XXH_rotl64(h64, 11) * PRIME64_1;
         p++;
     }
 
     h64 ^= h64 >> 33;
     h64 *= PRIME64_2;
     h64 ^= h64 >> 29;
     h64 *= PRIME64_3;
     h64 ^= h64 >> 32;
 
     return h64;
 }
 
 
 XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
 {
     XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
 
     if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
         return XXH64_digest_endian(state_in, XXH_littleEndian);
     else
         return XXH64_digest_endian(state_in, XXH_bigEndian);
 }
 
 
 /* **************************
 *  Canonical representation
 ****************************/
 
 /*! Default XXH result types are basic unsigned 32 and 64 bits.
 *   The canonical representation follows human-readable write convention, aka big-endian (large digits first).
 *   These functions allow transformation of hash result into and from its canonical format.
 *   This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
 */
 
 XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
 {
     XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
     if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
     memcpy(dst, &hash, sizeof(*dst));
 }
 
 XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
 {
     XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
     if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
     memcpy(dst, &hash, sizeof(*dst));
 }
 
 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
 {
     return XXH_readBE32(src);
 }
 
 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
 {
     return XXH_readBE64(src);
 }

contrib/python-zstandard/zstd/common/zstd_internal.h

@@ -1,266 +1,373 b''
 /*
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
  * You may select, at your option, one of the above-listed licenses.
  */
 
 #ifndef ZSTD_CCOMMON_H_MODULE
 #define ZSTD_CCOMMON_H_MODULE
 
 /* this module contains definitions which must be identical
  * across compression, decompression and dictBuilder.
  * It also contains a few functions useful to at least 2 of them
  * and which benefit from being inlined */
 
 /*-*************************************
 *  Dependencies
 ***************************************/
 #include "compiler.h"
 #include "mem.h"
 #include "debug.h"                 /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
 #include "error_private.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
 #ifndef XXH_STATIC_LINKING_ONLY
 #  define XXH_STATIC_LINKING_ONLY  /* XXH64_state_t */
 #endif
 #include "xxhash.h"                /* XXH_reset, update, digest */
 
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
 /* ---- static assert (debug) --- */
 #define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
 #define ZSTD_isError ERR_isError   /* for inlining */
 #define FSE_isError  ERR_isError
 #define HUF_isError  ERR_isError
 
 
 /*-*************************************
 *  shared macros
 ***************************************/
 #undef MIN
 #undef MAX
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define MAX(a,b) ((a)>(b) ? (a) : (b))
-#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; }  /* check and Forward error code */
-#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); }  /* check and send Error code */
+
+/**
+ * Return the specified error if the condition evaluates to true.
+ *
+ * In debug modes, prints additional information.
+ * In order to do that (particularly, printing the conditional that failed),
+ * this can't just wrap RETURN_ERROR().
+ */
+#define RETURN_ERROR_IF(cond, err, ...) \
+  if (cond) { \
+    RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \
+    RAWLOG(3, ": " __VA_ARGS__); \
+    RAWLOG(3, "\n"); \
+    return ERROR(err); \
+  }
+
+/**
+ * Unconditionally return the specified error.
+ *
+ * In debug modes, prints additional information.
+ */
+#define RETURN_ERROR(err, ...) \
+  do { \
+    RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \
+    RAWLOG(3, ": " __VA_ARGS__); \
+    RAWLOG(3, "\n"); \
+    return ERROR(err); \
+  } while(0);
+
+/**
+ * If the provided expression evaluates to an error code, returns that error code.
+ *
+ * In debug modes, prints additional information.
+ */
+#define FORWARD_IF_ERROR(err, ...) \
+  do { \
+    size_t const err_code = (err); \
+    if (ERR_isError(err_code)) { \
+      RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \
+      RAWLOG(3, ": " __VA_ARGS__); \
+      RAWLOG(3, "\n"); \
+      return err_code; \
+    } \
+  } while(0);
 
 
 /*-*************************************
 *  Common constants
 ***************************************/
 #define ZSTD_OPT_NUM    (1<<12)
 
 #define ZSTD_REP_NUM      3                 /* number of repcodes */
 #define ZSTD_REP_MOVE     (ZSTD_REP_NUM-1)
 static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
 #define BIT7 128
 #define BIT6  64
 #define BIT5  32
 #define BIT4  16
 #define BIT1   2
 #define BIT0   1
 
 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
 static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
 static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
 
 #define ZSTD_FRAMEIDSIZE 4   /* magic number size */
 
 #define ZSTD_BLOCKHEADERSIZE 3   /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
 static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
 typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 
 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
 #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */)   /* for a non-null block */
 
 #define HufLog 12
 typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
 
 #define LONGNBSEQ 0x7F00
 
 #define MINMATCH 3
 
 #define Litbits  8
 #define MaxLit ((1<<Litbits) - 1)
 #define MaxML   52
 #define MaxLL   35
 #define DefaultMaxOff 28
 #define MaxOff  31
 #define MaxSeq MAX(MaxLL, MaxML)   /* Assumption : MaxOff < MaxLL,MaxML */
 #define MLFSELog    9
 #define LLFSELog    9
 #define OffFSELog   8
 #define MaxFSELog  MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
 
 static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
                                       0, 0, 0, 0, 0, 0, 0, 0,
                                       1, 1, 1, 1, 2, 2, 3, 3,
                                       4, 6, 7, 8, 9,10,11,12,
                                      13,14,15,16 };
 static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2,
                                              2, 2, 2, 2, 2, 1, 1, 1,
                                              2, 2, 2, 2, 2, 2, 2, 2,
                                              2, 3, 2, 1, 1, 1, 1, 1,
                                             -1,-1,-1,-1 };
 #define LL_DEFAULTNORMLOG 6  /* for static allocation */
 static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
 
 static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
                                       0, 0, 0, 0, 0, 0, 0, 0,
                                       0, 0, 0, 0, 0, 0, 0, 0,
                                       0, 0, 0, 0, 0, 0, 0, 0,
                                       1, 1, 1, 1, 2, 2, 3, 3,
                                       4, 4, 5, 7, 8, 9,10,11,
                                      12,13,14,15,16 };
 static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2,
                                              2, 1, 1, 1, 1, 1, 1, 1,
                                              1, 1, 1, 1, 1, 1, 1, 1,
                                              1, 1, 1, 1, 1, 1, 1, 1,
                                              1, 1, 1, 1, 1, 1, 1, 1,
                                              1, 1, 1, 1, 1, 1,-1,-1,
                                             -1,-1,-1,-1,-1 };
 #define ML_DEFAULTNORMLOG 6  /* for static allocation */
 static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
 
 static const S16 OF_defaultNorm[DefaultMaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2,
                                                      2, 1, 1, 1, 1, 1, 1, 1,
                                                      1, 1, 1, 1, 1, 1, 1, 1,
                                                     -1,-1,-1,-1,-1 };
 #define OF_DEFAULTNORMLOG 5  /* for static allocation */
 static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
 
 
 /*-*******************************************
 *  Shared functions to include for inlining
 *********************************************/
 static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
+
 #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
+static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); }
+#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
+
+#define WILDCOPY_OVERLENGTH 8
+#define VECLEN 16
+
+typedef enum {
+    ZSTD_no_overlap,
+    ZSTD_overlap_src_before_dst,
+    /*  ZSTD_overlap_dst_before_src, */
+} ZSTD_overlap_e;
 
 /*! ZSTD_wildcopy() :
  *  custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */
-#define WILDCOPY_OVERLENGTH 8
-MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
+MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
+void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype)
 {
+    ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
     const BYTE* ip = (const BYTE*)src;
     BYTE* op = (BYTE*)dst;
     BYTE* const oend = op + length;
-    do
-        COPY8(op, ip)
-    while (op < oend);
+
+    assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8));
+    if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) {
+      do
+          COPY8(op, ip)
+      while (op < oend);
+    }
+    else {
+      if ((length & 8) == 0)
+        COPY8(op, ip);
+      do {
+        COPY16(op, ip);
+      }
+      while (op < oend);
+    }
+}
+
+/*! ZSTD_wildcopy_16min() :
+ *  same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */
+MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
+void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype)
+{
+    ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
+    const BYTE* ip = (const BYTE*)src;
+    BYTE* op = (BYTE*)dst;
+    BYTE* const oend = op + length;
+
+    assert(length >= 8);
+    assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8));
+
+    if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) {
+      do
+          COPY8(op, ip)
+      while (op < oend);
+    }
+    else {
+      if ((length & 8) == 0)
+        COPY8(op, ip);
+      do {
+        COPY16(op, ip);
+      }
+      while (op < oend);
+    }
 }
 
 MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd)   /* should be faster for decoding, but strangely, not verified on all platform */
 {
     const BYTE* ip = (const BYTE*)src;
     BYTE* op = (BYTE*)dst;
     BYTE* const oend = (BYTE*)dstEnd;
     do
         COPY8(op, ip)
     while (op < oend);
 }
 
 
 /*-*******************************************
 *  Private declarations
 *********************************************/
 typedef struct seqDef_s {
     U32 offset;
     U16 litLength;
     U16 matchLength;
 } seqDef;
 
 typedef struct {
     seqDef* sequencesStart;
     seqDef* sequences;
     BYTE* litStart;
     BYTE* lit;
     BYTE* llCode;
     BYTE* mlCode;
     BYTE* ofCode;
     size_t maxNbSeq;
     size_t maxNbLit;
     U32   longLengthID;   /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
     U32   longLengthPos;
 } seqStore_t;
 
+/**
+ * Contains the compressed frame size and an upper-bound for the decompressed frame size.
+ * Note: before using `compressedSize`, check for errors using ZSTD_isError().
+ *       similarly, before using `decompressedBound`, check for errors using:
+ *          `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
+ */
+typedef struct {
+    size_t compressedSize;
+    unsigned long long decompressedBound;
+} ZSTD_frameSizeInfo;   /* decompress & legacy */
+
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
 void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
 
 /* custom memory allocation functions */
 void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
 void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
 void ZSTD_free(void* ptr, ZSTD_customMem customMem);
 
 
 MEM_STATIC U32 ZSTD_highbit32(U32 val)   /* compress, dictBuilder, decodeCorpus */
 {
     assert(val != 0);
     {
 #   if defined(_MSC_VER)   /* Visual */
         unsigned long r=0;
         _BitScanReverse(&r, val);
         return (unsigned)r;
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* GCC Intrinsic */
         return 31 - __builtin_clz(val);
+#   elif defined(__ICCARM__)    /* IAR Intrinsic */
+        return 31 - __CLZ(val);
 #   else   /* Software version */
         static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
         U32 v = val;
         v |= v >> 1;
         v |= v >> 2;
         v |= v >> 4;
         v |= v >> 8;
         v |= v >> 16;
         return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];
 #   endif
     }
 }
 
 
 /* ZSTD_invalidateRepCodes() :
  * ensures next compression will not use repcodes from previous block.
  * Note : only works with regular variant;
  *        do not use with extDict variant ! */
 void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);   /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
 
 
 typedef struct {
     blockType_e blockType;
     U32 lastBlock;
     U32 origSize;
 } blockProperties_t;   /* declared here for decompress and fullbench */
 
 /*! ZSTD_getcBlockSize() :
  *  Provides the size of compressed block from block header `src` */
 /* Used by: decompress, fullbench (does not get its definition from here) */
 size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
                           blockProperties_t* bpPtr);
 
 /*! ZSTD_decodeSeqHeaders() :
  *  decode sequence header from src */
 /* Used by: decompress, fullbench (does not get its definition from here) */
 size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
                        const void* src, size_t srcSize);
 
 
 #if defined (__cplusplus)
 }
 #endif
 
 #endif   /* ZSTD_CCOMMON_H_MODULE */

contrib/python-zstandard/zstd/compress/fse_compress.c

@@ -1,721 +1,721 b''
 /* ******************************************************************
    FSE : Finite State Entropy encoder
    Copyright (C) 2013-present, Yann Collet.
 
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are
    met:
 
        * Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
        * Redistributions in binary form must reproduce the above
    copyright notice, this list of conditions and the following disclaimer
    in the documentation and/or other materials provided with the
    distribution.
 
    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
     - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
     - Public forum : https://groups.google.com/forum/#!forum/lz4c
 ****************************************************************** */
 
 /* **************************************************************
 *  Includes
 ****************************************************************/
 #include <stdlib.h>     /* malloc, free, qsort */
 #include <string.h>     /* memcpy, memset */
 #include "compiler.h"
 #include "mem.h"        /* U32, U16, etc. */
 #include "debug.h"      /* assert, DEBUGLOG */
 #include "hist.h"       /* HIST_count_wksp */
 #include "bitstream.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
 #include "error_private.h"
 
 
 /* **************************************************************
 *  Error Management
 ****************************************************************/
 #define FSE_isError ERR_isError
 
 
 /* **************************************************************
 *  Templates
 ****************************************************************/
 /*
   designed to be included
   for type-specific functions (template emulation in C)
   Objective is to write these functions only once, for improved maintenance
 */
 
 /* safety checks */
 #ifndef FSE_FUNCTION_EXTENSION
 #  error "FSE_FUNCTION_EXTENSION must be defined"
 #endif
 #ifndef FSE_FUNCTION_TYPE
 #  error "FSE_FUNCTION_TYPE must be defined"
 #endif
 
 /* Function names */
 #define FSE_CAT(X,Y) X##Y
 #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
 #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
 
 
 /* Function templates */
 
 /* FSE_buildCTable_wksp() :
  * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
  * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
  * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
  */
 size_t FSE_buildCTable_wksp(FSE_CTable* ct,
                       const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
                             void* workSpace, size_t wkspSize)
 {
     U32 const tableSize = 1 << tableLog;
     U32 const tableMask = tableSize - 1;
     void* const ptr = ct;
     U16* const tableU16 = ( (U16*) ptr) + 2;
     void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
     FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
     U32 const step = FSE_TABLESTEP(tableSize);
     U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
 
     FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace;
     U32 highThreshold = tableSize-1;
 
     /* CTable header */
     if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
     tableU16[-2] = (U16) tableLog;
     tableU16[-1] = (U16) maxSymbolValue;
     assert(tableLog < 16);   /* required for threshold strategy to work */
 
     /* For explanations on how to distribute symbol values over the table :
      * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
 
      #ifdef __clang_analyzer__
      memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize);   /* useless initialization, just to keep scan-build happy */
      #endif
 
     /* symbol start positions */
     {   U32 u;
         cumul[0] = 0;
         for (u=1; u <= maxSymbolValue+1; u++) {
             if (normalizedCounter[u-1]==-1) {  /* Low proba symbol */
                 cumul[u] = cumul[u-1] + 1;
                 tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
             } else {
                 cumul[u] = cumul[u-1] + normalizedCounter[u-1];
         }   }
         cumul[maxSymbolValue+1] = tableSize+1;
     }
 
     /* Spread symbols */
     {   U32 position = 0;
         U32 symbol;
         for (symbol=0; symbol<=maxSymbolValue; symbol++) {
-            int nbOccurences;
+            int nbOccurrences;
             int const freq = normalizedCounter[symbol];
-            for (nbOccurences=0; nbOccurences<freq; nbOccurences++) {
+            for (nbOccurrences=0; nbOccurrences<freq; nbOccurrences++) {
                 tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
                 position = (position + step) & tableMask;
                 while (position > highThreshold)
                     position = (position + step) & tableMask;   /* Low proba area */
         }   }
 
         assert(position==0);  /* Must have initialized all positions */
     }
 
     /* Build table */
     {   U32 u; for (u=0; u<tableSize; u++) {
         FSE_FUNCTION_TYPE s = tableSymbol[u];   /* note : static analyzer may not understand tableSymbol is properly initialized */
         tableU16[cumul[s]++] = (U16) (tableSize+u);   /* TableU16 : sorted by symbol order; gives next state value */
     }   }
 
     /* Build Symbol Transformation Table */
     {   unsigned total = 0;
         unsigned s;
         for (s=0; s<=maxSymbolValue; s++) {
             switch (normalizedCounter[s])
             {
             case  0:
                 /* filling nonetheless, for compatibility with FSE_getMaxNbBits() */
                 symbolTT[s].deltaNbBits = ((tableLog+1) << 16) - (1<<tableLog);
                 break;
 
             case -1:
             case  1:
                 symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
                 symbolTT[s].deltaFindState = total - 1;
                 total ++;
                 break;
             default :
                 {
                     U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
                     U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
                     symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
                     symbolTT[s].deltaFindState = total - normalizedCounter[s];
                     total +=  normalizedCounter[s];
     }   }   }   }
 
 #if 0  /* debug : symbol costs */
     DEBUGLOG(5, "\n --- table statistics : ");
     {   U32 symbol;
         for (symbol=0; symbol<=maxSymbolValue; symbol++) {
             DEBUGLOG(5, "%3u: w=%3i,   maxBits=%u, fracBits=%.2f",
                 symbol, normalizedCounter[symbol],
                 FSE_getMaxNbBits(symbolTT, symbol),
                 (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);
         }
     }
 #endif
 
     return 0;
 }
 
 
 size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
 {
     FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE];   /* memset() is not necessary, even if static analyzer complain about it */
     return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
 }
 
 
 
 #ifndef FSE_COMMONDEFS_ONLY
 
 
 /*-**************************************************************
 *  FSE NCount encoding
 ****************************************************************/
 size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
 {
     size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
     return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND;  /* maxSymbolValue==0 ? use default */
 }
 
 static size_t
 FSE_writeNCount_generic (void* header, size_t headerBufferSize,
                    const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
                          unsigned writeIsSafe)
 {
     BYTE* const ostart = (BYTE*) header;
     BYTE* out = ostart;
     BYTE* const oend = ostart + headerBufferSize;
     int nbBits;
     const int tableSize = 1 << tableLog;
     int remaining;
     int threshold;
     U32 bitStream = 0;
     int bitCount = 0;
     unsigned symbol = 0;
     unsigned const alphabetSize = maxSymbolValue + 1;
     int previousIs0 = 0;
 
     /* Table Size */
     bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
     bitCount  += 4;
 
     /* Init */
     remaining = tableSize+1;   /* +1 for extra accuracy */
     threshold = tableSize;
     nbBits = tableLog+1;
 
     while ((symbol < alphabetSize) && (remaining>1)) {  /* stops at 1 */
         if (previousIs0) {
             unsigned start = symbol;
             while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++;
             if (symbol == alphabetSize) break;   /* incorrect distribution */
             while (symbol >= start+24) {
                 start+=24;
                 bitStream += 0xFFFFU << bitCount;
                 if ((!writeIsSafe) && (out > oend-2))
                     return ERROR(dstSize_tooSmall);   /* Buffer overflow */
                 out[0] = (BYTE) bitStream;
                 out[1] = (BYTE)(bitStream>>8);
                 out+=2;
                 bitStream>>=16;
             }
             while (symbol >= start+3) {
                 start+=3;
                 bitStream += 3 << bitCount;
                 bitCount += 2;
             }
             bitStream += (symbol-start) << bitCount;
             bitCount += 2;
             if (bitCount>16) {
                 if ((!writeIsSafe) && (out > oend - 2))
                     return ERROR(dstSize_tooSmall);   /* Buffer overflow */
                 out[0] = (BYTE)bitStream;
                 out[1] = (BYTE)(bitStream>>8);
                 out += 2;
                 bitStream >>= 16;
                 bitCount -= 16;
         }   }
         {   int count = normalizedCounter[symbol++];
             int const max = (2*threshold-1) - remaining;
             remaining -= count < 0 ? -count : count;
             count++;   /* +1 for extra accuracy */
             if (count>=threshold)
                 count += max;   /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
             bitStream += count << bitCount;
             bitCount  += nbBits;
             bitCount  -= (count<max);
             previousIs0  = (count==1);
             if (remaining<1) return ERROR(GENERIC);
             while (remaining<threshold) { nbBits--; threshold>>=1; }
         }
         if (bitCount>16) {
             if ((!writeIsSafe) && (out > oend - 2))
                 return ERROR(dstSize_tooSmall);   /* Buffer overflow */
             out[0] = (BYTE)bitStream;
             out[1] = (BYTE)(bitStream>>8);
             out += 2;
             bitStream >>= 16;
             bitCount -= 16;
     }   }
 
     if (remaining != 1)
         return ERROR(GENERIC);  /* incorrect normalized distribution */
     assert(symbol <= alphabetSize);
 
     /* flush remaining bitStream */
     if ((!writeIsSafe) && (out > oend - 2))
         return ERROR(dstSize_tooSmall);   /* Buffer overflow */
     out[0] = (BYTE)bitStream;
     out[1] = (BYTE)(bitStream>>8);
     out+= (bitCount+7) /8;
 
     return (out-ostart);
 }
 
 
 size_t FSE_writeNCount (void* buffer, size_t bufferSize,
                   const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
 {
     if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);   /* Unsupported */
     if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC);   /* Unsupported */
 
     if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
         return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
 
     return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */);
 }
 
 
 /*-**************************************************************
 *  FSE Compression Code
 ****************************************************************/
 
 FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
 {
     size_t size;
     if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
     size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
     return (FSE_CTable*)malloc(size);
 }
 
 void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
 
 /* provides the minimum logSize to safely represent a distribution */
 static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
 {
     U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1;
     U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
     U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
     return minBits;
 }
 
 unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
 {
     U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
     U32 tableLog = maxTableLog;
     U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
     if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
     if (maxBitsSrc < tableLog) tableLog = maxBitsSrc;   /* Accuracy can be reduced */
     if (minBits > tableLog) tableLog = minBits;   /* Need a minimum to safely represent all symbol values */
     if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
     if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
     return tableLog;
 }
 
 unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
 {
     return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
 }
 
 
 /* Secondary normalization method.
    To be used when primary method fails. */
 
 static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
 {
     short const NOT_YET_ASSIGNED = -2;
     U32 s;
     U32 distributed = 0;
     U32 ToDistribute;
 
     /* Init */
     U32 const lowThreshold = (U32)(total >> tableLog);
     U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
 
     for (s=0; s<=maxSymbolValue; s++) {
         if (count[s] == 0) {
             norm[s]=0;
             continue;
         }
         if (count[s] <= lowThreshold) {
             norm[s] = -1;
             distributed++;
             total -= count[s];
             continue;
         }
         if (count[s] <= lowOne) {
             norm[s] = 1;
             distributed++;
             total -= count[s];
             continue;
         }
 
         norm[s]=NOT_YET_ASSIGNED;
     }
     ToDistribute = (1 << tableLog) - distributed;
 
     if (ToDistribute == 0)
         return 0;
 
     if ((total / ToDistribute) > lowOne) {
         /* risk of rounding to zero */
         lowOne = (U32)((total * 3) / (ToDistribute * 2));
         for (s=0; s<=maxSymbolValue; s++) {
             if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
                 norm[s] = 1;
                 distributed++;
                 total -= count[s];
                 continue;
         }   }
         ToDistribute = (1 << tableLog) - distributed;
     }
 
     if (distributed == maxSymbolValue+1) {
         /* all values are pretty poor;
            probably incompressible data (should have already been detected);
            find max, then give all remaining points to max */
         U32 maxV = 0, maxC = 0;
         for (s=0; s<=maxSymbolValue; s++)
             if (count[s] > maxC) { maxV=s; maxC=count[s]; }
         norm[maxV] += (short)ToDistribute;
         return 0;
     }
 
     if (total == 0) {
         /* all of the symbols were low enough for the lowOne or lowThreshold */
         for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
             if (norm[s] > 0) { ToDistribute--; norm[s]++; }
         return 0;
     }
 
     {   U64 const vStepLog = 62 - tableLog;
         U64 const mid = (1ULL << (vStepLog-1)) - 1;
         U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total;   /* scale on remaining */
         U64 tmpTotal = mid;
         for (s=0; s<=maxSymbolValue; s++) {
             if (norm[s]==NOT_YET_ASSIGNED) {
                 U64 const end = tmpTotal + (count[s] * rStep);
                 U32 const sStart = (U32)(tmpTotal >> vStepLog);
                 U32 const sEnd = (U32)(end >> vStepLog);
                 U32 const weight = sEnd - sStart;
                 if (weight < 1)
                     return ERROR(GENERIC);
                 norm[s] = (short)weight;
                 tmpTotal = end;
     }   }   }
 
     return 0;
 }
 
 
 size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
                            const unsigned* count, size_t total,
                            unsigned maxSymbolValue)
 {
     /* Sanity checks */
     if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
     if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC);   /* Unsupported size */
     if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);   /* Unsupported size */
     if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC);   /* Too small tableLog, compression potentially impossible */
 
     {   static U32 const rtbTable[] = {     0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
         U64 const scale = 62 - tableLog;
         U64 const step = ((U64)1<<62) / total;   /* <== here, one division ! */
         U64 const vStep = 1ULL<<(scale-20);
         int stillToDistribute = 1<<tableLog;
         unsigned s;
         unsigned largest=0;
         short largestP=0;
         U32 lowThreshold = (U32)(total >> tableLog);
 
         for (s=0; s<=maxSymbolValue; s++) {
             if (count[s] == total) return 0;   /* rle special case */
             if (count[s] == 0) { normalizedCounter[s]=0; continue; }
             if (count[s] <= lowThreshold) {
                 normalizedCounter[s] = -1;
                 stillToDistribute--;
             } else {
                 short proba = (short)((count[s]*step) >> scale);
                 if (proba<8) {
                     U64 restToBeat = vStep * rtbTable[proba];
                     proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
                 }
                 if (proba > largestP) { largestP=proba; largest=s; }
                 normalizedCounter[s] = proba;
                 stillToDistribute -= proba;
         }   }
         if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
             /* corner case, need another normalization method */
             size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
             if (FSE_isError(errorCode)) return errorCode;
         }
         else normalizedCounter[largest] += (short)stillToDistribute;
     }
 
 #if 0
     {   /* Print Table (debug) */
         U32 s;
         U32 nTotal = 0;
         for (s=0; s<=maxSymbolValue; s++)
             RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]);
         for (s=0; s<=maxSymbolValue; s++)
             nTotal += abs(normalizedCounter[s]);
         if (nTotal != (1U<<tableLog))
             RAWLOG(2, "Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
         getchar();
     }
 #endif
 
     return tableLog;
 }
 
 
 /* fake FSE_CTable, for raw (uncompressed) input */
 size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
 {
     const unsigned tableSize = 1 << nbBits;
     const unsigned tableMask = tableSize - 1;
     const unsigned maxSymbolValue = tableMask;
     void* const ptr = ct;
     U16* const tableU16 = ( (U16*) ptr) + 2;
     void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1);   /* assumption : tableLog >= 1 */
     FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
     unsigned s;
 
     /* Sanity checks */
     if (nbBits < 1) return ERROR(GENERIC);             /* min size */
 
     /* header */
     tableU16[-2] = (U16) nbBits;
     tableU16[-1] = (U16) maxSymbolValue;
 
     /* Build table */
     for (s=0; s<tableSize; s++)
         tableU16[s] = (U16)(tableSize + s);
 
     /* Build Symbol Transformation Table */
     {   const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
         for (s=0; s<=maxSymbolValue; s++) {
             symbolTT[s].deltaNbBits = deltaNbBits;
             symbolTT[s].deltaFindState = s-1;
     }   }
 
     return 0;
 }
 
 /* fake FSE_CTable, for rle input (always same symbol) */
 size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
 {
     void* ptr = ct;
     U16* tableU16 = ( (U16*) ptr) + 2;
     void* FSCTptr = (U32*)ptr + 2;
     FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
 
     /* header */
     tableU16[-2] = (U16) 0;
     tableU16[-1] = (U16) symbolValue;
 
     /* Build table */
     tableU16[0] = 0;
     tableU16[1] = 0;   /* just in case */
 
     /* Build Symbol Transformation Table */
     symbolTT[symbolValue].deltaNbBits = 0;
     symbolTT[symbolValue].deltaFindState = 0;
 
     return 0;
 }
 
 
 static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
                            const void* src, size_t srcSize,
                            const FSE_CTable* ct, const unsigned fast)
 {
     const BYTE* const istart = (const BYTE*) src;
     const BYTE* const iend = istart + srcSize;
     const BYTE* ip=iend;
 
     BIT_CStream_t bitC;
     FSE_CState_t CState1, CState2;
 
     /* init */
     if (srcSize <= 2) return 0;
     { size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
       if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ }
 
 #define FSE_FLUSHBITS(s)  (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
 
     if (srcSize & 1) {
         FSE_initCState2(&CState1, ct, *--ip);
         FSE_initCState2(&CState2, ct, *--ip);
         FSE_encodeSymbol(&bitC, &CState1, *--ip);
         FSE_FLUSHBITS(&bitC);
     } else {
         FSE_initCState2(&CState2, ct, *--ip);
         FSE_initCState2(&CState1, ct, *--ip);
     }
 
     /* join to mod 4 */
     srcSize -= 2;
     if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) {  /* test bit 2 */
         FSE_encodeSymbol(&bitC, &CState2, *--ip);
         FSE_encodeSymbol(&bitC, &CState1, *--ip);
         FSE_FLUSHBITS(&bitC);
     }
 
     /* 2 or 4 encoding per loop */
     while ( ip>istart ) {
 
         FSE_encodeSymbol(&bitC, &CState2, *--ip);
 
         if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 )   /* this test must be static */
             FSE_FLUSHBITS(&bitC);
 
         FSE_encodeSymbol(&bitC, &CState1, *--ip);
 
         if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) {  /* this test must be static */
             FSE_encodeSymbol(&bitC, &CState2, *--ip);
             FSE_encodeSymbol(&bitC, &CState1, *--ip);
         }
 
         FSE_FLUSHBITS(&bitC);
     }
 
     FSE_flushCState(&bitC, &CState2);
     FSE_flushCState(&bitC, &CState1);
     return BIT_closeCStream(&bitC);
 }
 
 size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
                            const void* src, size_t srcSize,
                            const FSE_CTable* ct)
 {
     unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
 
     if (fast)
         return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
     else
         return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
 }
 
 
 size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
 
 #define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
 #define CHECK_F(f)   { CHECK_V_F(_var_err__, f); }
 
 /* FSE_compress_wksp() :
  * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
  * `wkspSize` size must be `(1<<tableLog)`.
  */
 size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
 {
     BYTE* const ostart = (BYTE*) dst;
     BYTE* op = ostart;
     BYTE* const oend = ostart + dstSize;
 
     unsigned count[FSE_MAX_SYMBOL_VALUE+1];
     S16   norm[FSE_MAX_SYMBOL_VALUE+1];
     FSE_CTable* CTable = (FSE_CTable*)workSpace;
     size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
     void* scratchBuffer = (void*)(CTable + CTableSize);
     size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
 
     /* init conditions */
     if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
     if (srcSize <= 1) return 0;  /* Not compressible */
     if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
     if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
 
     /* Scan input and build symbol stats */
     {   CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) );
         if (maxCount == srcSize) return 1;   /* only a single symbol in src : rle */
         if (maxCount == 1) return 0;         /* each symbol present maximum once => not compressible */
         if (maxCount < (srcSize >> 7)) return 0;   /* Heuristic : not compressible enough */
     }
 
     tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
     CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) );
 
     /* Write table description header */
     {   CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
         op += nc_err;
     }
 
     /* Compress */
     CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) );
     {   CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) );
         if (cSize == 0) return 0;   /* not enough space for compressed data */
         op += cSize;
     }
 
     /* check compressibility */
     if ( (size_t)(op-ostart) >= srcSize-1 ) return 0;
 
     return op-ostart;
 }
 
 typedef struct {
     FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
     BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
 } fseWkspMax_t;
 
 size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
 {
     fseWkspMax_t scratchBuffer;
     DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE));   /* compilation failures here means scratchBuffer is not large enough */
     if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
     return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
 }
 
 size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
 }
 
 
 #endif   /* FSE_COMMONDEFS_ONLY */