compressionparams.c
502 lines
| 15.5 KiB
| text/x-c
|
CLexer
|
r30435 | /** | ||
* Copyright (c) 2016-present, Gregory Szorc | ||||
* All rights reserved. | ||||
* | ||||
* This software may be modified and distributed under the terms | ||||
* of the BSD license. See the LICENSE file for details. | ||||
*/ | ||||
#include "python-zstandard.h" | ||||
|
r37513 | extern PyObject* ZstdError; | ||
|
r30435 | |||
|
r37513 | int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value) { | ||
size_t zresult = ZSTD_CCtxParam_setParameter(params, param, value); | ||||
if (ZSTD_isError(zresult)) { | ||||
PyErr_Format(ZstdError, "unable to set compression context parameter: %s", | ||||
ZSTD_getErrorName(zresult)); | ||||
return 1; | ||||
|
r30435 | } | ||
|
r37513 | return 0; | ||
} | ||||
#define TRY_SET_PARAMETER(params, param, value) if (set_parameter(params, param, value)) return -1; | ||||
|
r30435 | |||
|
r37513 | int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj) { | ||
TRY_SET_PARAMETER(params, ZSTD_p_format, obj->format); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_compressionLevel, (unsigned)obj->compressionLevel); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_windowLog, obj->windowLog); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_hashLog, obj->hashLog); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_chainLog, obj->chainLog); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_searchLog, obj->searchLog); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_minMatch, obj->minMatch); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_targetLength, obj->targetLength); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_compressionStrategy, obj->compressionStrategy); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_contentSizeFlag, obj->contentSizeFlag); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_checksumFlag, obj->checksumFlag); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_dictIDFlag, obj->dictIDFlag); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_nbWorkers, obj->threads); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_jobSize, obj->jobSize); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_overlapSizeLog, obj->overlapSizeLog); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_compressLiterals, obj->compressLiterals); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_forceMaxWindow, obj->forceMaxWindow); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_enableLongDistanceMatching, obj->enableLongDistanceMatching); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_ldmHashLog, obj->ldmHashLog); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_ldmMinMatch, obj->ldmMinMatch); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_ldmBucketSizeLog, obj->ldmBucketSizeLog); | ||||
TRY_SET_PARAMETER(params, ZSTD_p_ldmHashEveryLog, obj->ldmHashEveryLog); | ||||
return 0; | ||||
} | ||||
int reset_params(ZstdCompressionParametersObject* params) { | ||||
if (params->params) { | ||||
ZSTD_CCtxParams_reset(params->params); | ||||
} | ||||
else { | ||||
params->params = ZSTD_createCCtxParams(); | ||||
if (!params->params) { | ||||
PyErr_NoMemory(); | ||||
return 1; | ||||
} | ||||
|
r30435 | } | ||
|
r37513 | return set_parameters(params->params, params); | ||
|
r30435 | } | ||
|
r37513 | static int ZstdCompressionParameters_init(ZstdCompressionParametersObject* self, PyObject* args, PyObject* kwargs) { | ||
|
r30895 | static char* kwlist[] = { | ||
|
r37513 | "format", | ||
"compression_level", | ||||
|
r30895 | "window_log", | ||
"hash_log", | ||||
|
r37513 | "chain_log", | ||
|
r30895 | "search_log", | ||
|
r37513 | "min_match", | ||
|
r30895 | "target_length", | ||
|
r37513 | "compression_strategy", | ||
"write_content_size", | ||||
"write_checksum", | ||||
"write_dict_id", | ||||
"job_size", | ||||
"overlap_size_log", | ||||
"force_max_window", | ||||
"enable_ldm", | ||||
"ldm_hash_log", | ||||
"ldm_min_match", | ||||
"ldm_bucket_size_log", | ||||
"ldm_hash_every_log", | ||||
"threads", | ||||
"compress_literals", | ||||
|
r30895 | NULL | ||
}; | ||||
|
r37513 | unsigned format = 0; | ||
int compressionLevel = 0; | ||||
unsigned windowLog = 0; | ||||
unsigned hashLog = 0; | ||||
unsigned chainLog = 0; | ||||
unsigned searchLog = 0; | ||||
unsigned minMatch = 0; | ||||
unsigned targetLength = 0; | ||||
unsigned compressionStrategy = 0; | ||||
unsigned contentSizeFlag = 1; | ||||
unsigned checksumFlag = 0; | ||||
unsigned dictIDFlag = 0; | ||||
unsigned jobSize = 0; | ||||
unsigned overlapSizeLog = 0; | ||||
unsigned forceMaxWindow = 0; | ||||
unsigned enableLDM = 0; | ||||
unsigned ldmHashLog = 0; | ||||
unsigned ldmMinMatch = 0; | ||||
unsigned ldmBucketSizeLog = 0; | ||||
unsigned ldmHashEveryLog = 0; | ||||
int threads = 0; | ||||
|
r30895 | |||
|
r37513 | /* Setting value 0 has the effect of disabling. So we use -1 as a default | ||
* to detect whether to set. Then we automatically derive the expected value | ||||
* based on the level, just like zstandard does itself. */ | ||||
int compressLiterals = -1; | ||||
|
r30895 | |||
|
r37513 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, | ||
"|IiIIIIIIIIIIIIIIIIIIii:CompressionParameters", | ||||
kwlist, &format, &compressionLevel, &windowLog, &hashLog, &chainLog, | ||||
&searchLog, &minMatch, &targetLength, &compressionStrategy, | ||||
&contentSizeFlag, &checksumFlag, &dictIDFlag, &jobSize, &overlapSizeLog, | ||||
&forceMaxWindow, &enableLDM, &ldmHashLog, &ldmMinMatch, &ldmBucketSizeLog, | ||||
&ldmHashEveryLog, &threads, &compressLiterals)) { | ||||
|
r30895 | return -1; | ||
} | ||||
|
r37513 | if (threads < 0) { | ||
threads = cpu_count(); | ||||
|
r30895 | } | ||
|
r37513 | if (compressLiterals < 0) { | ||
compressLiterals = compressionLevel >= 0; | ||||
|
r30895 | } | ||
|
r37513 | self->format = format; | ||
self->compressionLevel = compressionLevel; | ||||
|
r30895 | self->windowLog = windowLog; | ||
|
r37513 | self->hashLog = hashLog; | ||
|
r30895 | self->chainLog = chainLog; | ||
self->searchLog = searchLog; | ||||
|
r37513 | self->minMatch = minMatch; | ||
|
r30895 | self->targetLength = targetLength; | ||
|
r37513 | self->compressionStrategy = compressionStrategy; | ||
self->contentSizeFlag = contentSizeFlag; | ||||
self->checksumFlag = checksumFlag; | ||||
self->dictIDFlag = dictIDFlag; | ||||
self->threads = threads; | ||||
self->jobSize = jobSize; | ||||
self->overlapSizeLog = overlapSizeLog; | ||||
self->compressLiterals = compressLiterals; | ||||
self->forceMaxWindow = forceMaxWindow; | ||||
self->enableLongDistanceMatching = enableLDM; | ||||
self->ldmHashLog = ldmHashLog; | ||||
self->ldmMinMatch = ldmMinMatch; | ||||
self->ldmBucketSizeLog = ldmBucketSizeLog; | ||||
self->ldmHashEveryLog = ldmHashEveryLog; | ||||
|
r30895 | |||
|
r37513 | if (reset_params(self)) { | ||
|
r31796 | return -1; | ||
} | ||||
|
r30895 | return 0; | ||
} | ||||
|
r37513 | PyDoc_STRVAR(ZstdCompressionParameters_from_level__doc__, | ||
"Create a CompressionParameters from a compression level and target sizes\n" | ||||
); | ||||
ZstdCompressionParametersObject* CompressionParameters_from_level(PyObject* undef, PyObject* args, PyObject* kwargs) { | ||||
int managedKwargs = 0; | ||||
int level; | ||||
PyObject* sourceSize = NULL; | ||||
PyObject* dictSize = NULL; | ||||
unsigned PY_LONG_LONG iSourceSize = 0; | ||||
Py_ssize_t iDictSize = 0; | ||||
PyObject* val; | ||||
ZSTD_compressionParameters params; | ||||
ZstdCompressionParametersObject* result = NULL; | ||||
int res; | ||||
if (!PyArg_ParseTuple(args, "i:from_level", | ||||
&level)) { | ||||
return NULL; | ||||
} | ||||
if (!kwargs) { | ||||
kwargs = PyDict_New(); | ||||
if (!kwargs) { | ||||
return NULL; | ||||
} | ||||
managedKwargs = 1; | ||||
} | ||||
sourceSize = PyDict_GetItemString(kwargs, "source_size"); | ||||
if (sourceSize) { | ||||
#if PY_MAJOR_VERSION >= 3 | ||||
iSourceSize = PyLong_AsUnsignedLongLong(sourceSize); | ||||
if (iSourceSize == (unsigned PY_LONG_LONG)(-1)) { | ||||
goto cleanup; | ||||
} | ||||
#else | ||||
iSourceSize = PyInt_AsUnsignedLongLongMask(sourceSize); | ||||
#endif | ||||
PyDict_DelItemString(kwargs, "source_size"); | ||||
} | ||||
dictSize = PyDict_GetItemString(kwargs, "dict_size"); | ||||
if (dictSize) { | ||||
#if PY_MAJOR_VERSION >= 3 | ||||
iDictSize = PyLong_AsSsize_t(dictSize); | ||||
#else | ||||
iDictSize = PyInt_AsSsize_t(dictSize); | ||||
#endif | ||||
if (iDictSize == -1) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_DelItemString(kwargs, "dict_size"); | ||||
} | ||||
params = ZSTD_getCParams(level, iSourceSize, iDictSize); | ||||
/* Values derived from the input level and sizes are passed along to the | ||||
constructor. But only if a value doesn't already exist. */ | ||||
val = PyDict_GetItemString(kwargs, "window_log"); | ||||
if (!val) { | ||||
val = PyLong_FromUnsignedLong(params.windowLog); | ||||
if (!val) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_SetItemString(kwargs, "window_log", val); | ||||
Py_DECREF(val); | ||||
} | ||||
val = PyDict_GetItemString(kwargs, "chain_log"); | ||||
if (!val) { | ||||
val = PyLong_FromUnsignedLong(params.chainLog); | ||||
if (!val) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_SetItemString(kwargs, "chain_log", val); | ||||
Py_DECREF(val); | ||||
} | ||||
val = PyDict_GetItemString(kwargs, "hash_log"); | ||||
if (!val) { | ||||
val = PyLong_FromUnsignedLong(params.hashLog); | ||||
if (!val) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_SetItemString(kwargs, "hash_log", val); | ||||
Py_DECREF(val); | ||||
} | ||||
val = PyDict_GetItemString(kwargs, "search_log"); | ||||
if (!val) { | ||||
val = PyLong_FromUnsignedLong(params.searchLog); | ||||
if (!val) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_SetItemString(kwargs, "search_log", val); | ||||
Py_DECREF(val); | ||||
} | ||||
val = PyDict_GetItemString(kwargs, "min_match"); | ||||
if (!val) { | ||||
val = PyLong_FromUnsignedLong(params.searchLength); | ||||
if (!val) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_SetItemString(kwargs, "min_match", val); | ||||
Py_DECREF(val); | ||||
} | ||||
val = PyDict_GetItemString(kwargs, "target_length"); | ||||
if (!val) { | ||||
val = PyLong_FromUnsignedLong(params.targetLength); | ||||
if (!val) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_SetItemString(kwargs, "target_length", val); | ||||
Py_DECREF(val); | ||||
} | ||||
val = PyDict_GetItemString(kwargs, "compression_strategy"); | ||||
if (!val) { | ||||
val = PyLong_FromUnsignedLong(params.strategy); | ||||
if (!val) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_SetItemString(kwargs, "compression_strategy", val); | ||||
Py_DECREF(val); | ||||
} | ||||
val = PyDict_GetItemString(kwargs, "compress_literals"); | ||||
if (!val) { | ||||
val = PyLong_FromLong(level >= 0 ? 1 : 0); | ||||
if (!val) { | ||||
goto cleanup; | ||||
} | ||||
PyDict_SetItemString(kwargs, "compress_literals", val); | ||||
Py_DECREF(val); | ||||
} | ||||
result = PyObject_New(ZstdCompressionParametersObject, &ZstdCompressionParametersType); | ||||
if (!result) { | ||||
goto cleanup; | ||||
} | ||||
result->params = NULL; | ||||
val = PyTuple_New(0); | ||||
if (!val) { | ||||
Py_CLEAR(result); | ||||
goto cleanup; | ||||
} | ||||
res = ZstdCompressionParameters_init(result, val, kwargs); | ||||
Py_DECREF(val); | ||||
if (res) { | ||||
Py_CLEAR(result); | ||||
goto cleanup; | ||||
} | ||||
cleanup: | ||||
if (managedKwargs) { | ||||
Py_DECREF(kwargs); | ||||
} | ||||
return result; | ||||
} | ||||
PyDoc_STRVAR(ZstdCompressionParameters_estimated_compression_context_size__doc__, | ||||
|
r31796 | "Estimate the size in bytes of a compression context for compression parameters\n" | ||
); | ||||
|
r37513 | PyObject* ZstdCompressionParameters_estimated_compression_context_size(ZstdCompressionParametersObject* self) { | ||
return PyLong_FromSize_t(ZSTD_estimateCCtxSize_usingCCtxParams(self->params)); | ||||
|
r31796 | } | ||
|
r37513 | PyDoc_STRVAR(ZstdCompressionParameters__doc__, | ||
"ZstdCompressionParameters: low-level control over zstd compression"); | ||||
|
r30435 | |||
|
r37513 | static void ZstdCompressionParameters_dealloc(ZstdCompressionParametersObject* self) { | ||
if (self->params) { | ||||
ZSTD_freeCCtxParams(self->params); | ||||
self->params = NULL; | ||||
|
r30435 | } | ||
PyObject_Del(self); | ||||
} | ||||
|
r37513 | static PyMethodDef ZstdCompressionParameters_methods[] = { | ||
{ | ||||
"from_level", | ||||
(PyCFunction)CompressionParameters_from_level, | ||||
METH_VARARGS | METH_KEYWORDS | METH_STATIC, | ||||
ZstdCompressionParameters_from_level__doc__ | ||||
}, | ||||
|
r31796 | { | ||
"estimated_compression_context_size", | ||||
|
r37513 | (PyCFunction)ZstdCompressionParameters_estimated_compression_context_size, | ||
|
r31796 | METH_NOARGS, | ||
|
r37513 | ZstdCompressionParameters_estimated_compression_context_size__doc__ | ||
|
r31796 | }, | ||
{ NULL, NULL } | ||||
}; | ||||
|
r37513 | static PyMemberDef ZstdCompressionParameters_members[] = { | ||
{ "format", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, format), READONLY, | ||||
"compression format" }, | ||||
{ "compression_level", T_INT, | ||||
offsetof(ZstdCompressionParametersObject, compressionLevel), READONLY, | ||||
"compression level" }, | ||||
|
r30895 | { "window_log", T_UINT, | ||
|
r37513 | offsetof(ZstdCompressionParametersObject, windowLog), READONLY, | ||
|
r30895 | "window log" }, | ||
{ "hash_log", T_UINT, | ||||
|
r37513 | offsetof(ZstdCompressionParametersObject, hashLog), READONLY, | ||
|
r30895 | "hash log" }, | ||
|
r37513 | { "chain_log", T_UINT, | ||
offsetof(ZstdCompressionParametersObject, chainLog), READONLY, | ||||
"chain log" }, | ||||
|
r30895 | { "search_log", T_UINT, | ||
|
r37513 | offsetof(ZstdCompressionParametersObject, searchLog), READONLY, | ||
|
r30895 | "search log" }, | ||
|
r37513 | { "min_match", T_UINT, | ||
offsetof(ZstdCompressionParametersObject, minMatch), READONLY, | ||||
|
r30895 | "search length" }, | ||
{ "target_length", T_UINT, | ||||
|
r37513 | offsetof(ZstdCompressionParametersObject, targetLength), READONLY, | ||
|
r30895 | "target length" }, | ||
|
r37513 | { "compression_strategy", T_UINT, | ||
offsetof(ZstdCompressionParametersObject, compressionStrategy), READONLY, | ||||
"compression strategy" }, | ||||
{ "write_content_size", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, contentSizeFlag), READONLY, | ||||
"whether to write content size in frames" }, | ||||
{ "write_checksum", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, checksumFlag), READONLY, | ||||
"whether to write checksum in frames" }, | ||||
{ "write_dict_id", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, dictIDFlag), READONLY, | ||||
"whether to write dictionary ID in frames" }, | ||||
{ "threads", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, threads), READONLY, | ||||
"number of threads to use" }, | ||||
{ "job_size", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, jobSize), READONLY, | ||||
"size of compression job when using multiple threads" }, | ||||
{ "overlap_size_log", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, overlapSizeLog), READONLY, | ||||
"Size of previous input reloaded at the beginning of each job" }, | ||||
{ "compress_literals", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, compressLiterals), READONLY, | ||||
"whether Huffman compression of literals is in use" }, | ||||
{ "force_max_window", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, forceMaxWindow), READONLY, | ||||
"force back references to remain smaller than window size" }, | ||||
{ "enable_ldm", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, enableLongDistanceMatching), READONLY, | ||||
"whether to enable long distance matching" }, | ||||
{ "ldm_hash_log", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, ldmHashLog), READONLY, | ||||
"Size of the table for long distance matching, as a power of 2" }, | ||||
{ "ldm_min_match", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, ldmMinMatch), READONLY, | ||||
"minimum size of searched matches for long distance matcher" }, | ||||
{ "ldm_bucket_size_log", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, ldmBucketSizeLog), READONLY, | ||||
"log size of each bucket in the LDM hash table for collision resolution" }, | ||||
{ "ldm_hash_every_log", T_UINT, | ||||
offsetof(ZstdCompressionParametersObject, ldmHashEveryLog), READONLY, | ||||
"frequency of inserting/looking up entries in the LDM hash table" }, | ||||
|
r30895 | { NULL } | ||
|
r30435 | }; | ||
|
r37513 | PyTypeObject ZstdCompressionParametersType = { | ||
|
r30435 | PyVarObject_HEAD_INIT(NULL, 0) | ||
|
r37513 | "ZstdCompressionParameters", /* tp_name */ | ||
sizeof(ZstdCompressionParametersObject), /* tp_basicsize */ | ||||
|
r30435 | 0, /* tp_itemsize */ | ||
|
r37513 | (destructor)ZstdCompressionParameters_dealloc, /* tp_dealloc */ | ||
|
r30435 | 0, /* tp_print */ | ||
0, /* tp_getattr */ | ||||
0, /* tp_setattr */ | ||||
0, /* tp_compare */ | ||||
0, /* tp_repr */ | ||||
0, /* tp_as_number */ | ||||
|
r30895 | 0, /* tp_as_sequence */ | ||
|
r30435 | 0, /* tp_as_mapping */ | ||
0, /* tp_hash */ | ||||
0, /* tp_call */ | ||||
0, /* tp_str */ | ||||
0, /* tp_getattro */ | ||||
0, /* tp_setattro */ | ||||
0, /* tp_as_buffer */ | ||||
|
r30895 | Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ | ||
|
r37513 | ZstdCompressionParameters__doc__, /* tp_doc */ | ||
|
r30435 | 0, /* tp_traverse */ | ||
0, /* tp_clear */ | ||||
0, /* tp_richcompare */ | ||||
0, /* tp_weaklistoffset */ | ||||
0, /* tp_iter */ | ||||
0, /* tp_iternext */ | ||||
|
r37513 | ZstdCompressionParameters_methods, /* tp_methods */ | ||
ZstdCompressionParameters_members, /* tp_members */ | ||||
|
r30435 | 0, /* tp_getset */ | ||
0, /* tp_base */ | ||||
0, /* tp_dict */ | ||||
0, /* tp_descr_get */ | ||||
0, /* tp_descr_set */ | ||||
0, /* tp_dictoffset */ | ||||
|
r37513 | (initproc)ZstdCompressionParameters_init, /* tp_init */ | ||
|
r30435 | 0, /* tp_alloc */ | ||
|
r30895 | PyType_GenericNew, /* tp_new */ | ||
|
r30435 | }; | ||
void compressionparams_module_init(PyObject* mod) { | ||||
|
r37513 | Py_TYPE(&ZstdCompressionParametersType) = &PyType_Type; | ||
if (PyType_Ready(&ZstdCompressionParametersType) < 0) { | ||||
|
r30435 | return; | ||
} | ||||
|
r37513 | Py_INCREF(&ZstdCompressionParametersType); | ||
PyModule_AddObject(mod, "ZstdCompressionParameters", | ||||
(PyObject*)&ZstdCompressionParametersType); | ||||
/* TODO remove deprecated alias. */ | ||||
Py_INCREF(&ZstdCompressionParametersType); | ||||
|
r30435 | PyModule_AddObject(mod, "CompressionParameters", | ||
|
r37513 | (PyObject*)&ZstdCompressionParametersType); | ||
|
r30435 | } | ||