python-zstandard.h
346 lines
| 8.3 KiB
| text/x-c
|
CLexer
|
r30435 | /** | ||
* Copyright (c) 2016-present, Gregory Szorc | ||||
* All rights reserved. | ||||
* | ||||
* This software may be modified and distributed under the terms | ||||
* of the BSD license. See the LICENSE file for details. | ||||
*/ | ||||
#define PY_SSIZE_T_CLEAN | ||||
#include <Python.h> | ||||
|
r30895 | #include "structmember.h" | ||
|
r30435 | |||
#define ZSTD_STATIC_LINKING_ONLY | ||||
#define ZDICT_STATIC_LINKING_ONLY | ||||
|
r37513 | #include <zstd.h> | ||
#include <zdict.h> | ||||
|
r30435 | |||
|
r37513 | #define PYTHON_ZSTANDARD_VERSION "0.9.0" | ||
|
r30822 | |||
typedef enum { | ||||
compressorobj_flush_finish, | ||||
compressorobj_flush_block, | ||||
} CompressorObj_Flush; | ||||
|
r30435 | |||
|
r31796 | /* | ||
|
r37513 | Represents a ZstdCompressionParameters type. | ||
|
r31796 | |||
|
r37513 | This type holds all the low-level compression parameters that can be set. | ||
|
r31796 | */ | ||
|
r30435 | typedef struct { | ||
PyObject_HEAD | ||||
|
r37513 | ZSTD_CCtx_params* params; | ||
unsigned format; | ||||
int compressionLevel; | ||||
|
r30435 | unsigned windowLog; | ||
|
r37513 | unsigned hashLog; | ||
|
r30435 | unsigned chainLog; | ||
unsigned searchLog; | ||||
|
r37513 | unsigned minMatch; | ||
|
r30435 | unsigned targetLength; | ||
|
r37513 | unsigned compressionStrategy; | ||
unsigned contentSizeFlag; | ||||
unsigned checksumFlag; | ||||
unsigned dictIDFlag; | ||||
unsigned threads; | ||||
unsigned jobSize; | ||||
unsigned overlapSizeLog; | ||||
unsigned compressLiterals; | ||||
unsigned forceMaxWindow; | ||||
unsigned enableLongDistanceMatching; | ||||
unsigned ldmHashLog; | ||||
unsigned ldmMinMatch; | ||||
unsigned ldmBucketSizeLog; | ||||
unsigned ldmHashEveryLog; | ||||
} ZstdCompressionParametersObject; | ||||
|
r30435 | |||
|
r37513 | extern PyTypeObject ZstdCompressionParametersType; | ||
|
r30435 | |||
|
r31796 | /* | ||
Represents a FrameParameters type. | ||||
This type is basically a wrapper around ZSTD_frameParams. | ||||
*/ | ||||
|
r30435 | typedef struct { | ||
PyObject_HEAD | ||||
|
r30895 | unsigned long long frameContentSize; | ||
|
r37513 | unsigned long long windowSize; | ||
|
r30895 | unsigned dictID; | ||
char checksumFlag; | ||||
} FrameParametersObject; | ||||
extern PyTypeObject FrameParametersType; | ||||
|
r31796 | /* | ||
Represents a ZstdCompressionDict type. | ||||
|
r30435 | |||
|
r31796 | Instances hold data used for a zstd compression dictionary. | ||
*/ | ||||
|
r30435 | typedef struct { | ||
PyObject_HEAD | ||||
|
r31796 | /* Pointer to dictionary data. Owned by self. */ | ||
|
r30435 | void* dictData; | ||
|
r31796 | /* Size of dictionary data. */ | ||
|
r30435 | size_t dictSize; | ||
|
r37513 | ZSTD_dictContentType_e dictType; | ||
|
r31796 | /* k parameter for cover dictionaries. Only populated by train_cover_dict(). */ | ||
unsigned k; | ||||
/* d parameter for cover dictionaries. Only populated by train_cover_dict(). */ | ||||
unsigned d; | ||||
|
r37513 | /* Digested dictionary, suitable for reuse. */ | ||
ZSTD_CDict* cdict; | ||||
ZSTD_DDict* ddict; | ||||
|
r30435 | } ZstdCompressionDict; | ||
extern PyTypeObject ZstdCompressionDictType; | ||||
|
r31796 | /* | ||
Represents a ZstdCompressor type. | ||||
*/ | ||||
|
r30435 | typedef struct { | ||
PyObject_HEAD | ||||
|
r31796 | /* Number of threads to use for operations. */ | ||
unsigned int threads; | ||||
/* Pointer to compression dictionary to use. NULL if not using dictionary | ||||
compression. */ | ||||
|
r30435 | ZstdCompressionDict* dict; | ||
|
r37513 | /* Compression context to use. Populated during object construction. */ | ||
|
r30822 | ZSTD_CCtx* cctx; | ||
|
r37513 | /* Compression parameters in use. */ | ||
ZSTD_CCtx_params* params; | ||||
|
r30435 | } ZstdCompressor; | ||
extern PyTypeObject ZstdCompressorType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
ZstdCompressor* compressor; | ||||
ZSTD_outBuffer output; | ||||
|
r30822 | int finished; | ||
|
r30435 | } ZstdCompressionObj; | ||
extern PyTypeObject ZstdCompressionObjType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
ZstdCompressor* compressor; | ||||
PyObject* writer; | ||||
|
r37513 | unsigned long long sourceSize; | ||
|
r30435 | size_t outSize; | ||
int entered; | ||||
|
r37513 | unsigned long long bytesCompressed; | ||
|
r30435 | } ZstdCompressionWriter; | ||
extern PyTypeObject ZstdCompressionWriterType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
ZstdCompressor* compressor; | ||||
PyObject* reader; | ||||
|
r37513 | Py_buffer buffer; | ||
|
r30435 | Py_ssize_t bufferOffset; | ||
size_t inSize; | ||||
size_t outSize; | ||||
ZSTD_inBuffer input; | ||||
ZSTD_outBuffer output; | ||||
int finishedOutput; | ||||
int finishedInput; | ||||
PyObject* readResult; | ||||
} ZstdCompressorIterator; | ||||
extern PyTypeObject ZstdCompressorIteratorType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
|
r37513 | ZstdCompressor* compressor; | ||
PyObject* reader; | ||||
Py_buffer buffer; | ||||
unsigned long long sourceSize; | ||||
size_t readSize; | ||||
int entered; | ||||
int closed; | ||||
unsigned long long bytesCompressed; | ||||
ZSTD_inBuffer input; | ||||
ZSTD_outBuffer output; | ||||
int finishedInput; | ||||
int finishedOutput; | ||||
PyObject* readResult; | ||||
} ZstdCompressionReader; | ||||
extern PyTypeObject ZstdCompressionReaderType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
|
r30895 | ZSTD_DCtx* dctx; | ||
|
r30435 | ZstdCompressionDict* dict; | ||
|
r37513 | size_t maxWindowSize; | ||
ZSTD_format_e format; | ||||
|
r30435 | } ZstdDecompressor; | ||
extern PyTypeObject ZstdDecompressorType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
ZstdDecompressor* decompressor; | ||||
|
r37513 | size_t outSize; | ||
|
r30435 | int finished; | ||
} ZstdDecompressionObj; | ||||
extern PyTypeObject ZstdDecompressionObjType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
|
r37513 | /* Parent decompressor to which this object is associated. */ | ||
ZstdDecompressor* decompressor; | ||||
/* Object to read() from (if reading from a stream). */ | ||||
PyObject* reader; | ||||
/* Size for read() operations on reader. */ | ||||
size_t readSize; | ||||
/* Buffer to read from (if reading from a buffer). */ | ||||
Py_buffer buffer; | ||||
/* Whether the context manager is active. */ | ||||
int entered; | ||||
/* Whether we've closed the stream. */ | ||||
int closed; | ||||
/* Number of bytes decompressed and returned to user. */ | ||||
unsigned long long bytesDecompressed; | ||||
/* Tracks data going into decompressor. */ | ||||
ZSTD_inBuffer input; | ||||
/* Holds output from read() operation on reader. */ | ||||
PyObject* readResult; | ||||
/* Whether all input has been sent to the decompressor. */ | ||||
int finishedInput; | ||||
/* Whether all output has been flushed from the decompressor. */ | ||||
int finishedOutput; | ||||
} ZstdDecompressionReader; | ||||
extern PyTypeObject ZstdDecompressionReaderType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
|
r30435 | ZstdDecompressor* decompressor; | ||
PyObject* writer; | ||||
size_t outSize; | ||||
int entered; | ||||
} ZstdDecompressionWriter; | ||||
extern PyTypeObject ZstdDecompressionWriterType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
ZstdDecompressor* decompressor; | ||||
PyObject* reader; | ||||
|
r37513 | Py_buffer buffer; | ||
|
r30435 | Py_ssize_t bufferOffset; | ||
size_t inSize; | ||||
size_t outSize; | ||||
size_t skipBytes; | ||||
ZSTD_inBuffer input; | ||||
ZSTD_outBuffer output; | ||||
Py_ssize_t readCount; | ||||
int finishedInput; | ||||
int finishedOutput; | ||||
} ZstdDecompressorIterator; | ||||
extern PyTypeObject ZstdDecompressorIteratorType; | ||||
typedef struct { | ||||
int errored; | ||||
PyObject* chunk; | ||||
} DecompressorIteratorResult; | ||||
|
r31796 | typedef struct { | ||
|
r37513 | /* The public API is that these are 64-bit unsigned integers. So these can't | ||
* be size_t, even though values larger than SIZE_MAX or PY_SSIZE_T_MAX may | ||||
* be nonsensical for this platform. */ | ||||
|
r31796 | unsigned long long offset; | ||
unsigned long long length; | ||||
} BufferSegment; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
PyObject* parent; | ||||
BufferSegment* segments; | ||||
Py_ssize_t segmentCount; | ||||
} ZstdBufferSegments; | ||||
extern PyTypeObject ZstdBufferSegmentsType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
PyObject* parent; | ||||
void* data; | ||||
Py_ssize_t dataSize; | ||||
unsigned long long offset; | ||||
} ZstdBufferSegment; | ||||
extern PyTypeObject ZstdBufferSegmentType; | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
Py_buffer parent; | ||||
void* data; | ||||
unsigned long long dataSize; | ||||
BufferSegment* segments; | ||||
Py_ssize_t segmentCount; | ||||
int useFree; | ||||
} ZstdBufferWithSegments; | ||||
extern PyTypeObject ZstdBufferWithSegmentsType; | ||||
/** | ||||
* An ordered collection of BufferWithSegments exposed as a squashed collection. | ||||
* | ||||
* This type provides a virtual view spanning multiple BufferWithSegments | ||||
* instances. It allows multiple instances to be "chained" together and | ||||
* exposed as a single collection. e.g. if there are 2 buffers holding | ||||
* 10 segments each, then o[14] will access the 5th segment in the 2nd buffer. | ||||
*/ | ||||
typedef struct { | ||||
PyObject_HEAD | ||||
/* An array of buffers that should be exposed through this instance. */ | ||||
ZstdBufferWithSegments** buffers; | ||||
/* Number of elements in buffers array. */ | ||||
Py_ssize_t bufferCount; | ||||
/* Array of first offset in each buffer instance. 0th entry corresponds | ||||
to number of elements in the 0th buffer. 1st entry corresponds to the | ||||
sum of elements in 0th and 1st buffers. */ | ||||
Py_ssize_t* firstElements; | ||||
} ZstdBufferWithSegmentsCollection; | ||||
extern PyTypeObject ZstdBufferWithSegmentsCollectionType; | ||||
|
r37513 | int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value); | ||
int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj); | ||||
FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args, PyObject* kwargs); | ||||
int ensure_ddict(ZstdCompressionDict* dict); | ||||
int ensure_dctx(ZstdDecompressor* decompressor, int loadDict); | ||||
|
r30435 | ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs); | ||
|
r31796 | ZstdBufferWithSegments* BufferWithSegments_FromMemory(void* data, unsigned long long dataSize, BufferSegment* segments, Py_ssize_t segmentsSize); | ||
Py_ssize_t BufferWithSegmentsCollection_length(ZstdBufferWithSegmentsCollection*); | ||||
int cpu_count(void); | ||||
size_t roundpow2(size_t); | ||||
|
r37513 | int safe_pybytes_resize(PyObject** obj, Py_ssize_t size); | ||