Show More
@@ -1,788 +1,791 b'' | |||||
1 | /** |
|
1 | /** | |
2 | * Copyright (c) 2016-present, Gregory Szorc |
|
2 | * Copyright (c) 2016-present, Gregory Szorc | |
3 | * All rights reserved. |
|
3 | * All rights reserved. | |
4 | * |
|
4 | * | |
5 | * This software may be modified and distributed under the terms |
|
5 | * This software may be modified and distributed under the terms | |
6 | * of the BSD license. See the LICENSE file for details. |
|
6 | * of the BSD license. See the LICENSE file for details. | |
7 | */ |
|
7 | */ | |
8 |
|
8 | |||
9 | #include "python-zstandard.h" |
|
9 | #include "python-zstandard.h" | |
10 |
|
10 | |||
11 | extern PyObject* ZstdError; |
|
11 | extern PyObject* ZstdError; | |
12 |
|
12 | |||
13 | int populate_cdict(ZstdCompressor* compressor, void* dictData, size_t dictSize, ZSTD_parameters* zparams) { |
|
13 | int populate_cdict(ZstdCompressor* compressor, void* dictData, size_t dictSize, ZSTD_parameters* zparams) { | |
14 | ZSTD_customMem zmem; |
|
14 | ZSTD_customMem zmem; | |
15 | assert(!compressor->cdict); |
|
15 | assert(!compressor->cdict); | |
16 | Py_BEGIN_ALLOW_THREADS |
|
16 | Py_BEGIN_ALLOW_THREADS | |
17 | memset(&zmem, 0, sizeof(zmem)); |
|
17 | memset(&zmem, 0, sizeof(zmem)); | |
18 | compressor->cdict = ZSTD_createCDict_advanced(compressor->dict->dictData, |
|
18 | compressor->cdict = ZSTD_createCDict_advanced(compressor->dict->dictData, | |
19 | compressor->dict->dictSize, *zparams, zmem); |
|
19 | compressor->dict->dictSize, *zparams, zmem); | |
20 | Py_END_ALLOW_THREADS |
|
20 | Py_END_ALLOW_THREADS | |
21 |
|
21 | |||
22 | if (!compressor->cdict) { |
|
22 | if (!compressor->cdict) { | |
23 | PyErr_SetString(ZstdError, "could not create compression dictionary"); |
|
23 | PyErr_SetString(ZstdError, "could not create compression dictionary"); | |
24 | return 1; |
|
24 | return 1; | |
25 | } |
|
25 | } | |
26 |
|
26 | |||
27 | return 0; |
|
27 | return 0; | |
28 | } |
|
28 | } | |
29 |
|
29 | |||
30 | /** |
|
30 | /** | |
31 | * Initialize a zstd CStream from a ZstdCompressor instance. |
|
31 | * Initialize a zstd CStream from a ZstdCompressor instance. | |
32 | * |
|
32 | * | |
33 | * Returns a ZSTD_CStream on success or NULL on failure. If NULL, a Python |
|
33 | * Returns a ZSTD_CStream on success or NULL on failure. If NULL, a Python | |
34 | * exception will be set. |
|
34 | * exception will be set. | |
35 | */ |
|
35 | */ | |
36 | ZSTD_CStream* CStream_from_ZstdCompressor(ZstdCompressor* compressor, Py_ssize_t sourceSize) { |
|
36 | ZSTD_CStream* CStream_from_ZstdCompressor(ZstdCompressor* compressor, Py_ssize_t sourceSize) { | |
37 | ZSTD_CStream* cstream; |
|
37 | ZSTD_CStream* cstream; | |
38 | ZSTD_parameters zparams; |
|
38 | ZSTD_parameters zparams; | |
39 | void* dictData = NULL; |
|
39 | void* dictData = NULL; | |
40 | size_t dictSize = 0; |
|
40 | size_t dictSize = 0; | |
41 | size_t zresult; |
|
41 | size_t zresult; | |
42 |
|
42 | |||
43 | cstream = ZSTD_createCStream(); |
|
43 | cstream = ZSTD_createCStream(); | |
44 | if (!cstream) { |
|
44 | if (!cstream) { | |
45 | PyErr_SetString(ZstdError, "cannot create CStream"); |
|
45 | PyErr_SetString(ZstdError, "cannot create CStream"); | |
46 | return NULL; |
|
46 | return NULL; | |
47 | } |
|
47 | } | |
48 |
|
48 | |||
49 | if (compressor->dict) { |
|
49 | if (compressor->dict) { | |
50 | dictData = compressor->dict->dictData; |
|
50 | dictData = compressor->dict->dictData; | |
51 | dictSize = compressor->dict->dictSize; |
|
51 | dictSize = compressor->dict->dictSize; | |
52 | } |
|
52 | } | |
53 |
|
53 | |||
54 | memset(&zparams, 0, sizeof(zparams)); |
|
54 | memset(&zparams, 0, sizeof(zparams)); | |
55 | if (compressor->cparams) { |
|
55 | if (compressor->cparams) { | |
56 | ztopy_compression_parameters(compressor->cparams, &zparams.cParams); |
|
56 | ztopy_compression_parameters(compressor->cparams, &zparams.cParams); | |
57 | /* Do NOT call ZSTD_adjustCParams() here because the compression params |
|
57 | /* Do NOT call ZSTD_adjustCParams() here because the compression params | |
58 | come from the user. */ |
|
58 | come from the user. */ | |
59 | } |
|
59 | } | |
60 | else { |
|
60 | else { | |
61 | zparams.cParams = ZSTD_getCParams(compressor->compressionLevel, sourceSize, dictSize); |
|
61 | zparams.cParams = ZSTD_getCParams(compressor->compressionLevel, sourceSize, dictSize); | |
62 | } |
|
62 | } | |
63 |
|
63 | |||
64 | zparams.fParams = compressor->fparams; |
|
64 | zparams.fParams = compressor->fparams; | |
65 |
|
65 | |||
66 | zresult = ZSTD_initCStream_advanced(cstream, dictData, dictSize, zparams, sourceSize); |
|
66 | zresult = ZSTD_initCStream_advanced(cstream, dictData, dictSize, zparams, sourceSize); | |
67 |
|
67 | |||
68 | if (ZSTD_isError(zresult)) { |
|
68 | if (ZSTD_isError(zresult)) { | |
69 | ZSTD_freeCStream(cstream); |
|
69 | ZSTD_freeCStream(cstream); | |
70 | PyErr_Format(ZstdError, "cannot init CStream: %s", ZSTD_getErrorName(zresult)); |
|
70 | PyErr_Format(ZstdError, "cannot init CStream: %s", ZSTD_getErrorName(zresult)); | |
71 | return NULL; |
|
71 | return NULL; | |
72 | } |
|
72 | } | |
73 |
|
73 | |||
74 | return cstream; |
|
74 | return cstream; | |
75 | } |
|
75 | } | |
76 |
|
76 | |||
77 | PyDoc_STRVAR(ZstdCompressor__doc__, |
|
77 | PyDoc_STRVAR(ZstdCompressor__doc__, | |
78 | "ZstdCompressor(level=None, dict_data=None, compression_params=None)\n" |
|
78 | "ZstdCompressor(level=None, dict_data=None, compression_params=None)\n" | |
79 | "\n" |
|
79 | "\n" | |
80 | "Create an object used to perform Zstandard compression.\n" |
|
80 | "Create an object used to perform Zstandard compression.\n" | |
81 | "\n" |
|
81 | "\n" | |
82 | "An instance can compress data various ways. Instances can be used multiple\n" |
|
82 | "An instance can compress data various ways. Instances can be used multiple\n" | |
83 | "times. Each compression operation will use the compression parameters\n" |
|
83 | "times. Each compression operation will use the compression parameters\n" | |
84 | "defined at construction time.\n" |
|
84 | "defined at construction time.\n" | |
85 | "\n" |
|
85 | "\n" | |
86 | "Compression can be configured via the following names arguments:\n" |
|
86 | "Compression can be configured via the following names arguments:\n" | |
87 | "\n" |
|
87 | "\n" | |
88 | "level\n" |
|
88 | "level\n" | |
89 | " Integer compression level.\n" |
|
89 | " Integer compression level.\n" | |
90 | "dict_data\n" |
|
90 | "dict_data\n" | |
91 | " A ``ZstdCompressionDict`` to be used to compress with dictionary data.\n" |
|
91 | " A ``ZstdCompressionDict`` to be used to compress with dictionary data.\n" | |
92 | "compression_params\n" |
|
92 | "compression_params\n" | |
93 | " A ``CompressionParameters`` instance defining low-level compression" |
|
93 | " A ``CompressionParameters`` instance defining low-level compression" | |
94 | " parameters. If defined, this will overwrite the ``level`` argument.\n" |
|
94 | " parameters. If defined, this will overwrite the ``level`` argument.\n" | |
95 | "write_checksum\n" |
|
95 | "write_checksum\n" | |
96 | " If True, a 4 byte content checksum will be written with the compressed\n" |
|
96 | " If True, a 4 byte content checksum will be written with the compressed\n" | |
97 | " data, allowing the decompressor to perform content verification.\n" |
|
97 | " data, allowing the decompressor to perform content verification.\n" | |
98 | "write_content_size\n" |
|
98 | "write_content_size\n" | |
99 | " If True, the decompressed content size will be included in the header of\n" |
|
99 | " If True, the decompressed content size will be included in the header of\n" | |
100 | " the compressed data. This data will only be written if the compressor\n" |
|
100 | " the compressed data. This data will only be written if the compressor\n" | |
101 | " knows the size of the input data.\n" |
|
101 | " knows the size of the input data.\n" | |
102 | "write_dict_id\n" |
|
102 | "write_dict_id\n" | |
103 | " Determines whether the dictionary ID will be written into the compressed\n" |
|
103 | " Determines whether the dictionary ID will be written into the compressed\n" | |
104 | " data. Defaults to True. Only adds content to the compressed data if\n" |
|
104 | " data. Defaults to True. Only adds content to the compressed data if\n" | |
105 | " a dictionary is being used.\n" |
|
105 | " a dictionary is being used.\n" | |
106 | ); |
|
106 | ); | |
107 |
|
107 | |||
108 | static int ZstdCompressor_init(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { |
|
108 | static int ZstdCompressor_init(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { | |
109 | static char* kwlist[] = { |
|
109 | static char* kwlist[] = { | |
110 | "level", |
|
110 | "level", | |
111 | "dict_data", |
|
111 | "dict_data", | |
112 | "compression_params", |
|
112 | "compression_params", | |
113 | "write_checksum", |
|
113 | "write_checksum", | |
114 | "write_content_size", |
|
114 | "write_content_size", | |
115 | "write_dict_id", |
|
115 | "write_dict_id", | |
116 | NULL |
|
116 | NULL | |
117 | }; |
|
117 | }; | |
118 |
|
118 | |||
119 | int level = 3; |
|
119 | int level = 3; | |
120 | ZstdCompressionDict* dict = NULL; |
|
120 | ZstdCompressionDict* dict = NULL; | |
121 | CompressionParametersObject* params = NULL; |
|
121 | CompressionParametersObject* params = NULL; | |
122 | PyObject* writeChecksum = NULL; |
|
122 | PyObject* writeChecksum = NULL; | |
123 | PyObject* writeContentSize = NULL; |
|
123 | PyObject* writeContentSize = NULL; | |
124 | PyObject* writeDictID = NULL; |
|
124 | PyObject* writeDictID = NULL; | |
125 |
|
125 | |||
126 | self->cctx = NULL; |
|
126 | self->cctx = NULL; | |
127 | self->dict = NULL; |
|
127 | self->dict = NULL; | |
128 | self->cparams = NULL; |
|
128 | self->cparams = NULL; | |
129 | self->cdict = NULL; |
|
129 | self->cdict = NULL; | |
130 |
|
130 | |||
131 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!O!OOO", kwlist, |
|
131 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!O!OOO", kwlist, | |
132 | &level, &ZstdCompressionDictType, &dict, |
|
132 | &level, &ZstdCompressionDictType, &dict, | |
133 | &CompressionParametersType, ¶ms, |
|
133 | &CompressionParametersType, ¶ms, | |
134 | &writeChecksum, &writeContentSize, &writeDictID)) { |
|
134 | &writeChecksum, &writeContentSize, &writeDictID)) { | |
135 | return -1; |
|
135 | return -1; | |
136 | } |
|
136 | } | |
137 |
|
137 | |||
138 | if (level < 1) { |
|
138 | if (level < 1) { | |
139 | PyErr_SetString(PyExc_ValueError, "level must be greater than 0"); |
|
139 | PyErr_SetString(PyExc_ValueError, "level must be greater than 0"); | |
140 | return -1; |
|
140 | return -1; | |
141 | } |
|
141 | } | |
142 |
|
142 | |||
143 | if (level > ZSTD_maxCLevel()) { |
|
143 | if (level > ZSTD_maxCLevel()) { | |
144 | PyErr_Format(PyExc_ValueError, "level must be less than %d", |
|
144 | PyErr_Format(PyExc_ValueError, "level must be less than %d", | |
145 | ZSTD_maxCLevel() + 1); |
|
145 | ZSTD_maxCLevel() + 1); | |
146 | return -1; |
|
146 | return -1; | |
147 | } |
|
147 | } | |
148 |
|
148 | |||
149 | /* We create a ZSTD_CCtx for reuse among multiple operations to reduce the |
|
149 | /* We create a ZSTD_CCtx for reuse among multiple operations to reduce the | |
150 | overhead of each compression operation. */ |
|
150 | overhead of each compression operation. */ | |
151 | self->cctx = ZSTD_createCCtx(); |
|
151 | self->cctx = ZSTD_createCCtx(); | |
152 | if (!self->cctx) { |
|
152 | if (!self->cctx) { | |
153 | PyErr_NoMemory(); |
|
153 | PyErr_NoMemory(); | |
154 | return -1; |
|
154 | return -1; | |
155 | } |
|
155 | } | |
156 |
|
156 | |||
157 | self->compressionLevel = level; |
|
157 | self->compressionLevel = level; | |
158 |
|
158 | |||
159 | if (dict) { |
|
159 | if (dict) { | |
160 | self->dict = dict; |
|
160 | self->dict = dict; | |
161 | Py_INCREF(dict); |
|
161 | Py_INCREF(dict); | |
162 | } |
|
162 | } | |
163 |
|
163 | |||
164 | if (params) { |
|
164 | if (params) { | |
165 | self->cparams = params; |
|
165 | self->cparams = params; | |
166 | Py_INCREF(params); |
|
166 | Py_INCREF(params); | |
167 | } |
|
167 | } | |
168 |
|
168 | |||
169 | memset(&self->fparams, 0, sizeof(self->fparams)); |
|
169 | memset(&self->fparams, 0, sizeof(self->fparams)); | |
170 |
|
170 | |||
171 | if (writeChecksum && PyObject_IsTrue(writeChecksum)) { |
|
171 | if (writeChecksum && PyObject_IsTrue(writeChecksum)) { | |
172 | self->fparams.checksumFlag = 1; |
|
172 | self->fparams.checksumFlag = 1; | |
173 | } |
|
173 | } | |
174 | if (writeContentSize && PyObject_IsTrue(writeContentSize)) { |
|
174 | if (writeContentSize && PyObject_IsTrue(writeContentSize)) { | |
175 | self->fparams.contentSizeFlag = 1; |
|
175 | self->fparams.contentSizeFlag = 1; | |
176 | } |
|
176 | } | |
177 | if (writeDictID && PyObject_Not(writeDictID)) { |
|
177 | if (writeDictID && PyObject_Not(writeDictID)) { | |
178 | self->fparams.noDictIDFlag = 1; |
|
178 | self->fparams.noDictIDFlag = 1; | |
179 | } |
|
179 | } | |
180 |
|
180 | |||
181 | return 0; |
|
181 | return 0; | |
182 | } |
|
182 | } | |
183 |
|
183 | |||
184 | static void ZstdCompressor_dealloc(ZstdCompressor* self) { |
|
184 | static void ZstdCompressor_dealloc(ZstdCompressor* self) { | |
185 | Py_XDECREF(self->cparams); |
|
185 | Py_XDECREF(self->cparams); | |
186 | Py_XDECREF(self->dict); |
|
186 | Py_XDECREF(self->dict); | |
187 |
|
187 | |||
188 | if (self->cdict) { |
|
188 | if (self->cdict) { | |
189 | ZSTD_freeCDict(self->cdict); |
|
189 | ZSTD_freeCDict(self->cdict); | |
190 | self->cdict = NULL; |
|
190 | self->cdict = NULL; | |
191 | } |
|
191 | } | |
192 |
|
192 | |||
193 | if (self->cctx) { |
|
193 | if (self->cctx) { | |
194 | ZSTD_freeCCtx(self->cctx); |
|
194 | ZSTD_freeCCtx(self->cctx); | |
195 | self->cctx = NULL; |
|
195 | self->cctx = NULL; | |
196 | } |
|
196 | } | |
197 |
|
197 | |||
198 | PyObject_Del(self); |
|
198 | PyObject_Del(self); | |
199 | } |
|
199 | } | |
200 |
|
200 | |||
201 | PyDoc_STRVAR(ZstdCompressor_copy_stream__doc__, |
|
201 | PyDoc_STRVAR(ZstdCompressor_copy_stream__doc__, | |
202 | "copy_stream(ifh, ofh[, size=0, read_size=default, write_size=default])\n" |
|
202 | "copy_stream(ifh, ofh[, size=0, read_size=default, write_size=default])\n" | |
203 | "compress data between streams\n" |
|
203 | "compress data between streams\n" | |
204 | "\n" |
|
204 | "\n" | |
205 | "Data will be read from ``ifh``, compressed, and written to ``ofh``.\n" |
|
205 | "Data will be read from ``ifh``, compressed, and written to ``ofh``.\n" | |
206 | "``ifh`` must have a ``read(size)`` method. ``ofh`` must have a ``write(data)``\n" |
|
206 | "``ifh`` must have a ``read(size)`` method. ``ofh`` must have a ``write(data)``\n" | |
207 | "method.\n" |
|
207 | "method.\n" | |
208 | "\n" |
|
208 | "\n" | |
209 | "An optional ``size`` argument specifies the size of the source stream.\n" |
|
209 | "An optional ``size`` argument specifies the size of the source stream.\n" | |
210 | "If defined, compression parameters will be tuned based on the size.\n" |
|
210 | "If defined, compression parameters will be tuned based on the size.\n" | |
211 | "\n" |
|
211 | "\n" | |
212 | "Optional arguments ``read_size`` and ``write_size`` define the chunk sizes\n" |
|
212 | "Optional arguments ``read_size`` and ``write_size`` define the chunk sizes\n" | |
213 | "of ``read()`` and ``write()`` operations, respectively. By default, they use\n" |
|
213 | "of ``read()`` and ``write()`` operations, respectively. By default, they use\n" | |
214 | "the default compression stream input and output sizes, respectively.\n" |
|
214 | "the default compression stream input and output sizes, respectively.\n" | |
215 | ); |
|
215 | ); | |
216 |
|
216 | |||
217 | static PyObject* ZstdCompressor_copy_stream(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { |
|
217 | static PyObject* ZstdCompressor_copy_stream(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { | |
218 | static char* kwlist[] = { |
|
218 | static char* kwlist[] = { | |
219 | "ifh", |
|
219 | "ifh", | |
220 | "ofh", |
|
220 | "ofh", | |
221 | "size", |
|
221 | "size", | |
222 | "read_size", |
|
222 | "read_size", | |
223 | "write_size", |
|
223 | "write_size", | |
224 | NULL |
|
224 | NULL | |
225 | }; |
|
225 | }; | |
226 |
|
226 | |||
227 | PyObject* source; |
|
227 | PyObject* source; | |
228 | PyObject* dest; |
|
228 | PyObject* dest; | |
229 | Py_ssize_t sourceSize = 0; |
|
229 | Py_ssize_t sourceSize = 0; | |
230 | size_t inSize = ZSTD_CStreamInSize(); |
|
230 | size_t inSize = ZSTD_CStreamInSize(); | |
231 | size_t outSize = ZSTD_CStreamOutSize(); |
|
231 | size_t outSize = ZSTD_CStreamOutSize(); | |
232 | ZSTD_CStream* cstream; |
|
232 | ZSTD_CStream* cstream; | |
233 | ZSTD_inBuffer input; |
|
233 | ZSTD_inBuffer input; | |
234 | ZSTD_outBuffer output; |
|
234 | ZSTD_outBuffer output; | |
235 | Py_ssize_t totalRead = 0; |
|
235 | Py_ssize_t totalRead = 0; | |
236 | Py_ssize_t totalWrite = 0; |
|
236 | Py_ssize_t totalWrite = 0; | |
237 | char* readBuffer; |
|
237 | char* readBuffer; | |
238 | Py_ssize_t readSize; |
|
238 | Py_ssize_t readSize; | |
239 | PyObject* readResult; |
|
239 | PyObject* readResult; | |
240 | PyObject* res = NULL; |
|
240 | PyObject* res = NULL; | |
241 | size_t zresult; |
|
241 | size_t zresult; | |
242 | PyObject* writeResult; |
|
242 | PyObject* writeResult; | |
243 | PyObject* totalReadPy; |
|
243 | PyObject* totalReadPy; | |
244 | PyObject* totalWritePy; |
|
244 | PyObject* totalWritePy; | |
245 |
|
245 | |||
246 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|nkk", kwlist, &source, &dest, &sourceSize, |
|
246 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|nkk", kwlist, &source, &dest, &sourceSize, | |
247 | &inSize, &outSize)) { |
|
247 | &inSize, &outSize)) { | |
248 | return NULL; |
|
248 | return NULL; | |
249 | } |
|
249 | } | |
250 |
|
250 | |||
251 | if (!PyObject_HasAttrString(source, "read")) { |
|
251 | if (!PyObject_HasAttrString(source, "read")) { | |
252 | PyErr_SetString(PyExc_ValueError, "first argument must have a read() method"); |
|
252 | PyErr_SetString(PyExc_ValueError, "first argument must have a read() method"); | |
253 | return NULL; |
|
253 | return NULL; | |
254 | } |
|
254 | } | |
255 |
|
255 | |||
256 | if (!PyObject_HasAttrString(dest, "write")) { |
|
256 | if (!PyObject_HasAttrString(dest, "write")) { | |
257 | PyErr_SetString(PyExc_ValueError, "second argument must have a write() method"); |
|
257 | PyErr_SetString(PyExc_ValueError, "second argument must have a write() method"); | |
258 | return NULL; |
|
258 | return NULL; | |
259 | } |
|
259 | } | |
260 |
|
260 | |||
|
261 | /* Prevent free on uninitialized memory in finally. */ | |||
|
262 | output.dst = NULL; | |||
|
263 | ||||
261 | cstream = CStream_from_ZstdCompressor(self, sourceSize); |
|
264 | cstream = CStream_from_ZstdCompressor(self, sourceSize); | |
262 | if (!cstream) { |
|
265 | if (!cstream) { | |
263 | res = NULL; |
|
266 | res = NULL; | |
264 | goto finally; |
|
267 | goto finally; | |
265 | } |
|
268 | } | |
266 |
|
269 | |||
267 | output.dst = PyMem_Malloc(outSize); |
|
270 | output.dst = PyMem_Malloc(outSize); | |
268 | if (!output.dst) { |
|
271 | if (!output.dst) { | |
269 | PyErr_NoMemory(); |
|
272 | PyErr_NoMemory(); | |
270 | res = NULL; |
|
273 | res = NULL; | |
271 | goto finally; |
|
274 | goto finally; | |
272 | } |
|
275 | } | |
273 | output.size = outSize; |
|
276 | output.size = outSize; | |
274 | output.pos = 0; |
|
277 | output.pos = 0; | |
275 |
|
278 | |||
276 | while (1) { |
|
279 | while (1) { | |
277 | /* Try to read from source stream. */ |
|
280 | /* Try to read from source stream. */ | |
278 | readResult = PyObject_CallMethod(source, "read", "n", inSize); |
|
281 | readResult = PyObject_CallMethod(source, "read", "n", inSize); | |
279 | if (!readResult) { |
|
282 | if (!readResult) { | |
280 | PyErr_SetString(ZstdError, "could not read() from source"); |
|
283 | PyErr_SetString(ZstdError, "could not read() from source"); | |
281 | goto finally; |
|
284 | goto finally; | |
282 | } |
|
285 | } | |
283 |
|
286 | |||
284 | PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); |
|
287 | PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); | |
285 |
|
288 | |||
286 | /* If no data was read, we're at EOF. */ |
|
289 | /* If no data was read, we're at EOF. */ | |
287 | if (0 == readSize) { |
|
290 | if (0 == readSize) { | |
288 | break; |
|
291 | break; | |
289 | } |
|
292 | } | |
290 |
|
293 | |||
291 | totalRead += readSize; |
|
294 | totalRead += readSize; | |
292 |
|
295 | |||
293 | /* Send data to compressor */ |
|
296 | /* Send data to compressor */ | |
294 | input.src = readBuffer; |
|
297 | input.src = readBuffer; | |
295 | input.size = readSize; |
|
298 | input.size = readSize; | |
296 | input.pos = 0; |
|
299 | input.pos = 0; | |
297 |
|
300 | |||
298 | while (input.pos < input.size) { |
|
301 | while (input.pos < input.size) { | |
299 | Py_BEGIN_ALLOW_THREADS |
|
302 | Py_BEGIN_ALLOW_THREADS | |
300 | zresult = ZSTD_compressStream(cstream, &output, &input); |
|
303 | zresult = ZSTD_compressStream(cstream, &output, &input); | |
301 | Py_END_ALLOW_THREADS |
|
304 | Py_END_ALLOW_THREADS | |
302 |
|
305 | |||
303 | if (ZSTD_isError(zresult)) { |
|
306 | if (ZSTD_isError(zresult)) { | |
304 | res = NULL; |
|
307 | res = NULL; | |
305 | PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); |
|
308 | PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); | |
306 | goto finally; |
|
309 | goto finally; | |
307 | } |
|
310 | } | |
308 |
|
311 | |||
309 | if (output.pos) { |
|
312 | if (output.pos) { | |
310 | #if PY_MAJOR_VERSION >= 3 |
|
313 | #if PY_MAJOR_VERSION >= 3 | |
311 | writeResult = PyObject_CallMethod(dest, "write", "y#", |
|
314 | writeResult = PyObject_CallMethod(dest, "write", "y#", | |
312 | #else |
|
315 | #else | |
313 | writeResult = PyObject_CallMethod(dest, "write", "s#", |
|
316 | writeResult = PyObject_CallMethod(dest, "write", "s#", | |
314 | #endif |
|
317 | #endif | |
315 | output.dst, output.pos); |
|
318 | output.dst, output.pos); | |
316 | Py_XDECREF(writeResult); |
|
319 | Py_XDECREF(writeResult); | |
317 | totalWrite += output.pos; |
|
320 | totalWrite += output.pos; | |
318 | output.pos = 0; |
|
321 | output.pos = 0; | |
319 | } |
|
322 | } | |
320 | } |
|
323 | } | |
321 | } |
|
324 | } | |
322 |
|
325 | |||
323 | /* We've finished reading. Now flush the compressor stream. */ |
|
326 | /* We've finished reading. Now flush the compressor stream. */ | |
324 | while (1) { |
|
327 | while (1) { | |
325 | zresult = ZSTD_endStream(cstream, &output); |
|
328 | zresult = ZSTD_endStream(cstream, &output); | |
326 | if (ZSTD_isError(zresult)) { |
|
329 | if (ZSTD_isError(zresult)) { | |
327 | PyErr_Format(ZstdError, "error ending compression stream: %s", |
|
330 | PyErr_Format(ZstdError, "error ending compression stream: %s", | |
328 | ZSTD_getErrorName(zresult)); |
|
331 | ZSTD_getErrorName(zresult)); | |
329 | res = NULL; |
|
332 | res = NULL; | |
330 | goto finally; |
|
333 | goto finally; | |
331 | } |
|
334 | } | |
332 |
|
335 | |||
333 | if (output.pos) { |
|
336 | if (output.pos) { | |
334 | #if PY_MAJOR_VERSION >= 3 |
|
337 | #if PY_MAJOR_VERSION >= 3 | |
335 | writeResult = PyObject_CallMethod(dest, "write", "y#", |
|
338 | writeResult = PyObject_CallMethod(dest, "write", "y#", | |
336 | #else |
|
339 | #else | |
337 | writeResult = PyObject_CallMethod(dest, "write", "s#", |
|
340 | writeResult = PyObject_CallMethod(dest, "write", "s#", | |
338 | #endif |
|
341 | #endif | |
339 | output.dst, output.pos); |
|
342 | output.dst, output.pos); | |
340 | totalWrite += output.pos; |
|
343 | totalWrite += output.pos; | |
341 | Py_XDECREF(writeResult); |
|
344 | Py_XDECREF(writeResult); | |
342 | output.pos = 0; |
|
345 | output.pos = 0; | |
343 | } |
|
346 | } | |
344 |
|
347 | |||
345 | if (!zresult) { |
|
348 | if (!zresult) { | |
346 | break; |
|
349 | break; | |
347 | } |
|
350 | } | |
348 | } |
|
351 | } | |
349 |
|
352 | |||
350 | ZSTD_freeCStream(cstream); |
|
353 | ZSTD_freeCStream(cstream); | |
351 | cstream = NULL; |
|
354 | cstream = NULL; | |
352 |
|
355 | |||
353 | totalReadPy = PyLong_FromSsize_t(totalRead); |
|
356 | totalReadPy = PyLong_FromSsize_t(totalRead); | |
354 | totalWritePy = PyLong_FromSsize_t(totalWrite); |
|
357 | totalWritePy = PyLong_FromSsize_t(totalWrite); | |
355 | res = PyTuple_Pack(2, totalReadPy, totalWritePy); |
|
358 | res = PyTuple_Pack(2, totalReadPy, totalWritePy); | |
356 | Py_DecRef(totalReadPy); |
|
359 | Py_DecRef(totalReadPy); | |
357 | Py_DecRef(totalWritePy); |
|
360 | Py_DecRef(totalWritePy); | |
358 |
|
361 | |||
359 | finally: |
|
362 | finally: | |
360 | if (output.dst) { |
|
363 | if (output.dst) { | |
361 | PyMem_Free(output.dst); |
|
364 | PyMem_Free(output.dst); | |
362 | } |
|
365 | } | |
363 |
|
366 | |||
364 | if (cstream) { |
|
367 | if (cstream) { | |
365 | ZSTD_freeCStream(cstream); |
|
368 | ZSTD_freeCStream(cstream); | |
366 | } |
|
369 | } | |
367 |
|
370 | |||
368 | return res; |
|
371 | return res; | |
369 | } |
|
372 | } | |
370 |
|
373 | |||
371 | PyDoc_STRVAR(ZstdCompressor_compress__doc__, |
|
374 | PyDoc_STRVAR(ZstdCompressor_compress__doc__, | |
372 | "compress(data, allow_empty=False)\n" |
|
375 | "compress(data, allow_empty=False)\n" | |
373 | "\n" |
|
376 | "\n" | |
374 | "Compress data in a single operation.\n" |
|
377 | "Compress data in a single operation.\n" | |
375 | "\n" |
|
378 | "\n" | |
376 | "This is the simplest mechanism to perform compression: simply pass in a\n" |
|
379 | "This is the simplest mechanism to perform compression: simply pass in a\n" | |
377 | "value and get a compressed value back. It is almost the most prone to abuse.\n" |
|
380 | "value and get a compressed value back. It is almost the most prone to abuse.\n" | |
378 | "The input and output values must fit in memory, so passing in very large\n" |
|
381 | "The input and output values must fit in memory, so passing in very large\n" | |
379 | "values can result in excessive memory usage. For this reason, one of the\n" |
|
382 | "values can result in excessive memory usage. For this reason, one of the\n" | |
380 | "streaming based APIs is preferred for larger values.\n" |
|
383 | "streaming based APIs is preferred for larger values.\n" | |
381 | ); |
|
384 | ); | |
382 |
|
385 | |||
383 | static PyObject* ZstdCompressor_compress(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { |
|
386 | static PyObject* ZstdCompressor_compress(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { | |
384 | static char* kwlist[] = { |
|
387 | static char* kwlist[] = { | |
385 | "data", |
|
388 | "data", | |
386 | "allow_empty", |
|
389 | "allow_empty", | |
387 | NULL |
|
390 | NULL | |
388 | }; |
|
391 | }; | |
389 |
|
392 | |||
390 | const char* source; |
|
393 | const char* source; | |
391 | Py_ssize_t sourceSize; |
|
394 | Py_ssize_t sourceSize; | |
392 | PyObject* allowEmpty = NULL; |
|
395 | PyObject* allowEmpty = NULL; | |
393 | size_t destSize; |
|
396 | size_t destSize; | |
394 | PyObject* output; |
|
397 | PyObject* output; | |
395 | char* dest; |
|
398 | char* dest; | |
396 | void* dictData = NULL; |
|
399 | void* dictData = NULL; | |
397 | size_t dictSize = 0; |
|
400 | size_t dictSize = 0; | |
398 | size_t zresult; |
|
401 | size_t zresult; | |
399 | ZSTD_parameters zparams; |
|
402 | ZSTD_parameters zparams; | |
400 |
|
403 | |||
401 | #if PY_MAJOR_VERSION >= 3 |
|
404 | #if PY_MAJOR_VERSION >= 3 | |
402 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|O", |
|
405 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|O", | |
403 | #else |
|
406 | #else | |
404 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|O", |
|
407 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|O", | |
405 | #endif |
|
408 | #endif | |
406 | kwlist, &source, &sourceSize, &allowEmpty)) { |
|
409 | kwlist, &source, &sourceSize, &allowEmpty)) { | |
407 | return NULL; |
|
410 | return NULL; | |
408 | } |
|
411 | } | |
409 |
|
412 | |||
410 | /* Limitation in zstd C API doesn't let decompression side distinguish |
|
413 | /* Limitation in zstd C API doesn't let decompression side distinguish | |
411 | between content size of 0 and unknown content size. This can make round |
|
414 | between content size of 0 and unknown content size. This can make round | |
412 | tripping via Python difficult. Until this is fixed, require a flag |
|
415 | tripping via Python difficult. Until this is fixed, require a flag | |
413 | to fire the footgun. |
|
416 | to fire the footgun. | |
414 | https://github.com/indygreg/python-zstandard/issues/11 */ |
|
417 | https://github.com/indygreg/python-zstandard/issues/11 */ | |
415 | if (0 == sourceSize && self->fparams.contentSizeFlag |
|
418 | if (0 == sourceSize && self->fparams.contentSizeFlag | |
416 | && (!allowEmpty || PyObject_Not(allowEmpty))) { |
|
419 | && (!allowEmpty || PyObject_Not(allowEmpty))) { | |
417 | PyErr_SetString(PyExc_ValueError, "cannot write empty inputs when writing content sizes"); |
|
420 | PyErr_SetString(PyExc_ValueError, "cannot write empty inputs when writing content sizes"); | |
418 | return NULL; |
|
421 | return NULL; | |
419 | } |
|
422 | } | |
420 |
|
423 | |||
421 | destSize = ZSTD_compressBound(sourceSize); |
|
424 | destSize = ZSTD_compressBound(sourceSize); | |
422 | output = PyBytes_FromStringAndSize(NULL, destSize); |
|
425 | output = PyBytes_FromStringAndSize(NULL, destSize); | |
423 | if (!output) { |
|
426 | if (!output) { | |
424 | return NULL; |
|
427 | return NULL; | |
425 | } |
|
428 | } | |
426 |
|
429 | |||
427 | dest = PyBytes_AsString(output); |
|
430 | dest = PyBytes_AsString(output); | |
428 |
|
431 | |||
429 | if (self->dict) { |
|
432 | if (self->dict) { | |
430 | dictData = self->dict->dictData; |
|
433 | dictData = self->dict->dictData; | |
431 | dictSize = self->dict->dictSize; |
|
434 | dictSize = self->dict->dictSize; | |
432 | } |
|
435 | } | |
433 |
|
436 | |||
434 | memset(&zparams, 0, sizeof(zparams)); |
|
437 | memset(&zparams, 0, sizeof(zparams)); | |
435 | if (!self->cparams) { |
|
438 | if (!self->cparams) { | |
436 | zparams.cParams = ZSTD_getCParams(self->compressionLevel, sourceSize, dictSize); |
|
439 | zparams.cParams = ZSTD_getCParams(self->compressionLevel, sourceSize, dictSize); | |
437 | } |
|
440 | } | |
438 | else { |
|
441 | else { | |
439 | ztopy_compression_parameters(self->cparams, &zparams.cParams); |
|
442 | ztopy_compression_parameters(self->cparams, &zparams.cParams); | |
440 | /* Do NOT call ZSTD_adjustCParams() here because the compression params |
|
443 | /* Do NOT call ZSTD_adjustCParams() here because the compression params | |
441 | come from the user. */ |
|
444 | come from the user. */ | |
442 | } |
|
445 | } | |
443 |
|
446 | |||
444 | zparams.fParams = self->fparams; |
|
447 | zparams.fParams = self->fparams; | |
445 |
|
448 | |||
446 | /* The raw dict data has to be processed before it can be used. Since this |
|
449 | /* The raw dict data has to be processed before it can be used. Since this | |
447 | adds overhead - especially if multiple dictionary compression operations |
|
450 | adds overhead - especially if multiple dictionary compression operations | |
448 | are performed on the same ZstdCompressor instance - we create a |
|
451 | are performed on the same ZstdCompressor instance - we create a | |
449 | ZSTD_CDict once and reuse it for all operations. |
|
452 | ZSTD_CDict once and reuse it for all operations. | |
450 |
|
453 | |||
451 | Note: the compression parameters used for the first invocation (possibly |
|
454 | Note: the compression parameters used for the first invocation (possibly | |
452 | derived from the source size) will be reused on all subsequent invocations. |
|
455 | derived from the source size) will be reused on all subsequent invocations. | |
453 | https://github.com/facebook/zstd/issues/358 contains more info. We could |
|
456 | https://github.com/facebook/zstd/issues/358 contains more info. We could | |
454 | potentially add an argument somewhere to control this behavior. |
|
457 | potentially add an argument somewhere to control this behavior. | |
455 | */ |
|
458 | */ | |
456 | if (dictData && !self->cdict) { |
|
459 | if (dictData && !self->cdict) { | |
457 | if (populate_cdict(self, dictData, dictSize, &zparams)) { |
|
460 | if (populate_cdict(self, dictData, dictSize, &zparams)) { | |
458 | Py_DECREF(output); |
|
461 | Py_DECREF(output); | |
459 | return NULL; |
|
462 | return NULL; | |
460 | } |
|
463 | } | |
461 | } |
|
464 | } | |
462 |
|
465 | |||
463 | Py_BEGIN_ALLOW_THREADS |
|
466 | Py_BEGIN_ALLOW_THREADS | |
464 | /* By avoiding ZSTD_compress(), we don't necessarily write out content |
|
467 | /* By avoiding ZSTD_compress(), we don't necessarily write out content | |
465 | size. This means the argument to ZstdCompressor to control frame |
|
468 | size. This means the argument to ZstdCompressor to control frame | |
466 | parameters is honored. */ |
|
469 | parameters is honored. */ | |
467 | if (self->cdict) { |
|
470 | if (self->cdict) { | |
468 | zresult = ZSTD_compress_usingCDict(self->cctx, dest, destSize, |
|
471 | zresult = ZSTD_compress_usingCDict(self->cctx, dest, destSize, | |
469 | source, sourceSize, self->cdict); |
|
472 | source, sourceSize, self->cdict); | |
470 | } |
|
473 | } | |
471 | else { |
|
474 | else { | |
472 | zresult = ZSTD_compress_advanced(self->cctx, dest, destSize, |
|
475 | zresult = ZSTD_compress_advanced(self->cctx, dest, destSize, | |
473 | source, sourceSize, dictData, dictSize, zparams); |
|
476 | source, sourceSize, dictData, dictSize, zparams); | |
474 | } |
|
477 | } | |
475 | Py_END_ALLOW_THREADS |
|
478 | Py_END_ALLOW_THREADS | |
476 |
|
479 | |||
477 | if (ZSTD_isError(zresult)) { |
|
480 | if (ZSTD_isError(zresult)) { | |
478 | PyErr_Format(ZstdError, "cannot compress: %s", ZSTD_getErrorName(zresult)); |
|
481 | PyErr_Format(ZstdError, "cannot compress: %s", ZSTD_getErrorName(zresult)); | |
479 | Py_CLEAR(output); |
|
482 | Py_CLEAR(output); | |
480 | return NULL; |
|
483 | return NULL; | |
481 | } |
|
484 | } | |
482 | else { |
|
485 | else { | |
483 | Py_SIZE(output) = zresult; |
|
486 | Py_SIZE(output) = zresult; | |
484 | } |
|
487 | } | |
485 |
|
488 | |||
486 | return output; |
|
489 | return output; | |
487 | } |
|
490 | } | |
488 |
|
491 | |||
489 | PyDoc_STRVAR(ZstdCompressionObj__doc__, |
|
492 | PyDoc_STRVAR(ZstdCompressionObj__doc__, | |
490 | "compressobj()\n" |
|
493 | "compressobj()\n" | |
491 | "\n" |
|
494 | "\n" | |
492 | "Return an object exposing ``compress(data)`` and ``flush()`` methods.\n" |
|
495 | "Return an object exposing ``compress(data)`` and ``flush()`` methods.\n" | |
493 | "\n" |
|
496 | "\n" | |
494 | "The returned object exposes an API similar to ``zlib.compressobj`` and\n" |
|
497 | "The returned object exposes an API similar to ``zlib.compressobj`` and\n" | |
495 | "``bz2.BZ2Compressor`` so that callers can swap in the zstd compressor\n" |
|
498 | "``bz2.BZ2Compressor`` so that callers can swap in the zstd compressor\n" | |
496 | "without changing how compression is performed.\n" |
|
499 | "without changing how compression is performed.\n" | |
497 | ); |
|
500 | ); | |
498 |
|
501 | |||
499 | static ZstdCompressionObj* ZstdCompressor_compressobj(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { |
|
502 | static ZstdCompressionObj* ZstdCompressor_compressobj(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { | |
500 | static char* kwlist[] = { |
|
503 | static char* kwlist[] = { | |
501 | "size", |
|
504 | "size", | |
502 | NULL |
|
505 | NULL | |
503 | }; |
|
506 | }; | |
504 |
|
507 | |||
505 | Py_ssize_t inSize = 0; |
|
508 | Py_ssize_t inSize = 0; | |
506 | size_t outSize = ZSTD_CStreamOutSize(); |
|
509 | size_t outSize = ZSTD_CStreamOutSize(); | |
507 | ZstdCompressionObj* result = PyObject_New(ZstdCompressionObj, &ZstdCompressionObjType); |
|
510 | ZstdCompressionObj* result = PyObject_New(ZstdCompressionObj, &ZstdCompressionObjType); | |
508 | if (!result) { |
|
511 | if (!result) { | |
509 | return NULL; |
|
512 | return NULL; | |
510 | } |
|
513 | } | |
511 |
|
514 | |||
512 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &inSize)) { |
|
515 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &inSize)) { | |
513 | return NULL; |
|
516 | return NULL; | |
514 | } |
|
517 | } | |
515 |
|
518 | |||
516 | result->cstream = CStream_from_ZstdCompressor(self, inSize); |
|
519 | result->cstream = CStream_from_ZstdCompressor(self, inSize); | |
517 | if (!result->cstream) { |
|
520 | if (!result->cstream) { | |
518 | Py_DECREF(result); |
|
521 | Py_DECREF(result); | |
519 | return NULL; |
|
522 | return NULL; | |
520 | } |
|
523 | } | |
521 |
|
524 | |||
522 | result->output.dst = PyMem_Malloc(outSize); |
|
525 | result->output.dst = PyMem_Malloc(outSize); | |
523 | if (!result->output.dst) { |
|
526 | if (!result->output.dst) { | |
524 | PyErr_NoMemory(); |
|
527 | PyErr_NoMemory(); | |
525 | Py_DECREF(result); |
|
528 | Py_DECREF(result); | |
526 | return NULL; |
|
529 | return NULL; | |
527 | } |
|
530 | } | |
528 | result->output.size = outSize; |
|
531 | result->output.size = outSize; | |
529 | result->output.pos = 0; |
|
532 | result->output.pos = 0; | |
530 |
|
533 | |||
531 | result->compressor = self; |
|
534 | result->compressor = self; | |
532 | Py_INCREF(result->compressor); |
|
535 | Py_INCREF(result->compressor); | |
533 |
|
536 | |||
534 | result->finished = 0; |
|
537 | result->finished = 0; | |
535 |
|
538 | |||
536 | return result; |
|
539 | return result; | |
537 | } |
|
540 | } | |
538 |
|
541 | |||
539 | PyDoc_STRVAR(ZstdCompressor_read_from__doc__, |
|
542 | PyDoc_STRVAR(ZstdCompressor_read_from__doc__, | |
540 | "read_from(reader, [size=0, read_size=default, write_size=default])\n" |
|
543 | "read_from(reader, [size=0, read_size=default, write_size=default])\n" | |
541 | "Read uncompress data from a reader and return an iterator\n" |
|
544 | "Read uncompress data from a reader and return an iterator\n" | |
542 | "\n" |
|
545 | "\n" | |
543 | "Returns an iterator of compressed data produced from reading from ``reader``.\n" |
|
546 | "Returns an iterator of compressed data produced from reading from ``reader``.\n" | |
544 | "\n" |
|
547 | "\n" | |
545 | "Uncompressed data will be obtained from ``reader`` by calling the\n" |
|
548 | "Uncompressed data will be obtained from ``reader`` by calling the\n" | |
546 | "``read(size)`` method of it. The source data will be streamed into a\n" |
|
549 | "``read(size)`` method of it. The source data will be streamed into a\n" | |
547 | "compressor. As compressed data is available, it will be exposed to the\n" |
|
550 | "compressor. As compressed data is available, it will be exposed to the\n" | |
548 | "iterator.\n" |
|
551 | "iterator.\n" | |
549 | "\n" |
|
552 | "\n" | |
550 | "Data is read from the source in chunks of ``read_size``. Compressed chunks\n" |
|
553 | "Data is read from the source in chunks of ``read_size``. Compressed chunks\n" | |
551 | "are at most ``write_size`` bytes. Both values default to the zstd input and\n" |
|
554 | "are at most ``write_size`` bytes. Both values default to the zstd input and\n" | |
552 | "and output defaults, respectively.\n" |
|
555 | "and output defaults, respectively.\n" | |
553 | "\n" |
|
556 | "\n" | |
554 | "The caller is partially in control of how fast data is fed into the\n" |
|
557 | "The caller is partially in control of how fast data is fed into the\n" | |
555 | "compressor by how it consumes the returned iterator. The compressor will\n" |
|
558 | "compressor by how it consumes the returned iterator. The compressor will\n" | |
556 | "not consume from the reader unless the caller consumes from the iterator.\n" |
|
559 | "not consume from the reader unless the caller consumes from the iterator.\n" | |
557 | ); |
|
560 | ); | |
558 |
|
561 | |||
559 | static ZstdCompressorIterator* ZstdCompressor_read_from(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { |
|
562 | static ZstdCompressorIterator* ZstdCompressor_read_from(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { | |
560 | static char* kwlist[] = { |
|
563 | static char* kwlist[] = { | |
561 | "reader", |
|
564 | "reader", | |
562 | "size", |
|
565 | "size", | |
563 | "read_size", |
|
566 | "read_size", | |
564 | "write_size", |
|
567 | "write_size", | |
565 | NULL |
|
568 | NULL | |
566 | }; |
|
569 | }; | |
567 |
|
570 | |||
568 | PyObject* reader; |
|
571 | PyObject* reader; | |
569 | Py_ssize_t sourceSize = 0; |
|
572 | Py_ssize_t sourceSize = 0; | |
570 | size_t inSize = ZSTD_CStreamInSize(); |
|
573 | size_t inSize = ZSTD_CStreamInSize(); | |
571 | size_t outSize = ZSTD_CStreamOutSize(); |
|
574 | size_t outSize = ZSTD_CStreamOutSize(); | |
572 | ZstdCompressorIterator* result; |
|
575 | ZstdCompressorIterator* result; | |
573 |
|
576 | |||
574 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nkk", kwlist, &reader, &sourceSize, |
|
577 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nkk", kwlist, &reader, &sourceSize, | |
575 | &inSize, &outSize)) { |
|
578 | &inSize, &outSize)) { | |
576 | return NULL; |
|
579 | return NULL; | |
577 | } |
|
580 | } | |
578 |
|
581 | |||
579 | result = PyObject_New(ZstdCompressorIterator, &ZstdCompressorIteratorType); |
|
582 | result = PyObject_New(ZstdCompressorIterator, &ZstdCompressorIteratorType); | |
580 | if (!result) { |
|
583 | if (!result) { | |
581 | return NULL; |
|
584 | return NULL; | |
582 | } |
|
585 | } | |
583 |
|
586 | |||
584 | result->compressor = NULL; |
|
587 | result->compressor = NULL; | |
585 | result->reader = NULL; |
|
588 | result->reader = NULL; | |
586 | result->buffer = NULL; |
|
589 | result->buffer = NULL; | |
587 | result->cstream = NULL; |
|
590 | result->cstream = NULL; | |
588 | result->input.src = NULL; |
|
591 | result->input.src = NULL; | |
589 | result->output.dst = NULL; |
|
592 | result->output.dst = NULL; | |
590 | result->readResult = NULL; |
|
593 | result->readResult = NULL; | |
591 |
|
594 | |||
592 | if (PyObject_HasAttrString(reader, "read")) { |
|
595 | if (PyObject_HasAttrString(reader, "read")) { | |
593 | result->reader = reader; |
|
596 | result->reader = reader; | |
594 | Py_INCREF(result->reader); |
|
597 | Py_INCREF(result->reader); | |
595 | } |
|
598 | } | |
596 | else if (1 == PyObject_CheckBuffer(reader)) { |
|
599 | else if (1 == PyObject_CheckBuffer(reader)) { | |
597 | result->buffer = PyMem_Malloc(sizeof(Py_buffer)); |
|
600 | result->buffer = PyMem_Malloc(sizeof(Py_buffer)); | |
598 | if (!result->buffer) { |
|
601 | if (!result->buffer) { | |
599 | goto except; |
|
602 | goto except; | |
600 | } |
|
603 | } | |
601 |
|
604 | |||
602 | memset(result->buffer, 0, sizeof(Py_buffer)); |
|
605 | memset(result->buffer, 0, sizeof(Py_buffer)); | |
603 |
|
606 | |||
604 | if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) { |
|
607 | if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) { | |
605 | goto except; |
|
608 | goto except; | |
606 | } |
|
609 | } | |
607 |
|
610 | |||
608 | result->bufferOffset = 0; |
|
611 | result->bufferOffset = 0; | |
609 | sourceSize = result->buffer->len; |
|
612 | sourceSize = result->buffer->len; | |
610 | } |
|
613 | } | |
611 | else { |
|
614 | else { | |
612 | PyErr_SetString(PyExc_ValueError, |
|
615 | PyErr_SetString(PyExc_ValueError, | |
613 | "must pass an object with a read() method or conforms to buffer protocol"); |
|
616 | "must pass an object with a read() method or conforms to buffer protocol"); | |
614 | goto except; |
|
617 | goto except; | |
615 | } |
|
618 | } | |
616 |
|
619 | |||
617 | result->compressor = self; |
|
620 | result->compressor = self; | |
618 | Py_INCREF(result->compressor); |
|
621 | Py_INCREF(result->compressor); | |
619 |
|
622 | |||
620 | result->sourceSize = sourceSize; |
|
623 | result->sourceSize = sourceSize; | |
621 | result->cstream = CStream_from_ZstdCompressor(self, sourceSize); |
|
624 | result->cstream = CStream_from_ZstdCompressor(self, sourceSize); | |
622 | if (!result->cstream) { |
|
625 | if (!result->cstream) { | |
623 | goto except; |
|
626 | goto except; | |
624 | } |
|
627 | } | |
625 |
|
628 | |||
626 | result->inSize = inSize; |
|
629 | result->inSize = inSize; | |
627 | result->outSize = outSize; |
|
630 | result->outSize = outSize; | |
628 |
|
631 | |||
629 | result->output.dst = PyMem_Malloc(outSize); |
|
632 | result->output.dst = PyMem_Malloc(outSize); | |
630 | if (!result->output.dst) { |
|
633 | if (!result->output.dst) { | |
631 | PyErr_NoMemory(); |
|
634 | PyErr_NoMemory(); | |
632 | goto except; |
|
635 | goto except; | |
633 | } |
|
636 | } | |
634 | result->output.size = outSize; |
|
637 | result->output.size = outSize; | |
635 | result->output.pos = 0; |
|
638 | result->output.pos = 0; | |
636 |
|
639 | |||
637 | result->input.src = NULL; |
|
640 | result->input.src = NULL; | |
638 | result->input.size = 0; |
|
641 | result->input.size = 0; | |
639 | result->input.pos = 0; |
|
642 | result->input.pos = 0; | |
640 |
|
643 | |||
641 | result->finishedInput = 0; |
|
644 | result->finishedInput = 0; | |
642 | result->finishedOutput = 0; |
|
645 | result->finishedOutput = 0; | |
643 |
|
646 | |||
644 | goto finally; |
|
647 | goto finally; | |
645 |
|
648 | |||
646 | except: |
|
649 | except: | |
647 | if (result->cstream) { |
|
650 | if (result->cstream) { | |
648 | ZSTD_freeCStream(result->cstream); |
|
651 | ZSTD_freeCStream(result->cstream); | |
649 | result->cstream = NULL; |
|
652 | result->cstream = NULL; | |
650 | } |
|
653 | } | |
651 |
|
654 | |||
652 | Py_DecRef((PyObject*)result->compressor); |
|
655 | Py_DecRef((PyObject*)result->compressor); | |
653 | Py_DecRef(result->reader); |
|
656 | Py_DecRef(result->reader); | |
654 |
|
657 | |||
655 | Py_DECREF(result); |
|
658 | Py_DECREF(result); | |
656 | result = NULL; |
|
659 | result = NULL; | |
657 |
|
660 | |||
658 | finally: |
|
661 | finally: | |
659 | return result; |
|
662 | return result; | |
660 | } |
|
663 | } | |
661 |
|
664 | |||
662 | PyDoc_STRVAR(ZstdCompressor_write_to___doc__, |
|
665 | PyDoc_STRVAR(ZstdCompressor_write_to___doc__, | |
663 | "Create a context manager to write compressed data to an object.\n" |
|
666 | "Create a context manager to write compressed data to an object.\n" | |
664 | "\n" |
|
667 | "\n" | |
665 | "The passed object must have a ``write()`` method.\n" |
|
668 | "The passed object must have a ``write()`` method.\n" | |
666 | "\n" |
|
669 | "\n" | |
667 | "The caller feeds input data to the object by calling ``compress(data)``.\n" |
|
670 | "The caller feeds input data to the object by calling ``compress(data)``.\n" | |
668 | "Compressed data is written to the argument given to this function.\n" |
|
671 | "Compressed data is written to the argument given to this function.\n" | |
669 | "\n" |
|
672 | "\n" | |
670 | "The function takes an optional ``size`` argument indicating the total size\n" |
|
673 | "The function takes an optional ``size`` argument indicating the total size\n" | |
671 | "of the eventual input. If specified, the size will influence compression\n" |
|
674 | "of the eventual input. If specified, the size will influence compression\n" | |
672 | "parameter tuning and could result in the size being written into the\n" |
|
675 | "parameter tuning and could result in the size being written into the\n" | |
673 | "header of the compressed data.\n" |
|
676 | "header of the compressed data.\n" | |
674 | "\n" |
|
677 | "\n" | |
675 | "An optional ``write_size`` argument is also accepted. It defines the maximum\n" |
|
678 | "An optional ``write_size`` argument is also accepted. It defines the maximum\n" | |
676 | "byte size of chunks fed to ``write()``. By default, it uses the zstd default\n" |
|
679 | "byte size of chunks fed to ``write()``. By default, it uses the zstd default\n" | |
677 | "for a compressor output stream.\n" |
|
680 | "for a compressor output stream.\n" | |
678 | ); |
|
681 | ); | |
679 |
|
682 | |||
680 | static ZstdCompressionWriter* ZstdCompressor_write_to(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { |
|
683 | static ZstdCompressionWriter* ZstdCompressor_write_to(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { | |
681 | static char* kwlist[] = { |
|
684 | static char* kwlist[] = { | |
682 | "writer", |
|
685 | "writer", | |
683 | "size", |
|
686 | "size", | |
684 | "write_size", |
|
687 | "write_size", | |
685 | NULL |
|
688 | NULL | |
686 | }; |
|
689 | }; | |
687 |
|
690 | |||
688 | PyObject* writer; |
|
691 | PyObject* writer; | |
689 | ZstdCompressionWriter* result; |
|
692 | ZstdCompressionWriter* result; | |
690 | Py_ssize_t sourceSize = 0; |
|
693 | Py_ssize_t sourceSize = 0; | |
691 | size_t outSize = ZSTD_CStreamOutSize(); |
|
694 | size_t outSize = ZSTD_CStreamOutSize(); | |
692 |
|
695 | |||
693 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nk", kwlist, &writer, &sourceSize, |
|
696 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nk", kwlist, &writer, &sourceSize, | |
694 | &outSize)) { |
|
697 | &outSize)) { | |
695 | return NULL; |
|
698 | return NULL; | |
696 | } |
|
699 | } | |
697 |
|
700 | |||
698 | if (!PyObject_HasAttrString(writer, "write")) { |
|
701 | if (!PyObject_HasAttrString(writer, "write")) { | |
699 | PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method"); |
|
702 | PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method"); | |
700 | return NULL; |
|
703 | return NULL; | |
701 | } |
|
704 | } | |
702 |
|
705 | |||
703 | result = PyObject_New(ZstdCompressionWriter, &ZstdCompressionWriterType); |
|
706 | result = PyObject_New(ZstdCompressionWriter, &ZstdCompressionWriterType); | |
704 | if (!result) { |
|
707 | if (!result) { | |
705 | return NULL; |
|
708 | return NULL; | |
706 | } |
|
709 | } | |
707 |
|
710 | |||
708 | result->compressor = self; |
|
711 | result->compressor = self; | |
709 | Py_INCREF(result->compressor); |
|
712 | Py_INCREF(result->compressor); | |
710 |
|
713 | |||
711 | result->writer = writer; |
|
714 | result->writer = writer; | |
712 | Py_INCREF(result->writer); |
|
715 | Py_INCREF(result->writer); | |
713 |
|
716 | |||
714 | result->sourceSize = sourceSize; |
|
717 | result->sourceSize = sourceSize; | |
715 |
|
718 | |||
716 | result->outSize = outSize; |
|
719 | result->outSize = outSize; | |
717 |
|
720 | |||
718 | result->entered = 0; |
|
721 | result->entered = 0; | |
719 | result->cstream = NULL; |
|
722 | result->cstream = NULL; | |
720 |
|
723 | |||
721 | return result; |
|
724 | return result; | |
722 | } |
|
725 | } | |
723 |
|
726 | |||
724 | static PyMethodDef ZstdCompressor_methods[] = { |
|
727 | static PyMethodDef ZstdCompressor_methods[] = { | |
725 | { "compress", (PyCFunction)ZstdCompressor_compress, |
|
728 | { "compress", (PyCFunction)ZstdCompressor_compress, | |
726 | METH_VARARGS | METH_KEYWORDS, ZstdCompressor_compress__doc__ }, |
|
729 | METH_VARARGS | METH_KEYWORDS, ZstdCompressor_compress__doc__ }, | |
727 | { "compressobj", (PyCFunction)ZstdCompressor_compressobj, |
|
730 | { "compressobj", (PyCFunction)ZstdCompressor_compressobj, | |
728 | METH_VARARGS | METH_KEYWORDS, ZstdCompressionObj__doc__ }, |
|
731 | METH_VARARGS | METH_KEYWORDS, ZstdCompressionObj__doc__ }, | |
729 | { "copy_stream", (PyCFunction)ZstdCompressor_copy_stream, |
|
732 | { "copy_stream", (PyCFunction)ZstdCompressor_copy_stream, | |
730 | METH_VARARGS | METH_KEYWORDS, ZstdCompressor_copy_stream__doc__ }, |
|
733 | METH_VARARGS | METH_KEYWORDS, ZstdCompressor_copy_stream__doc__ }, | |
731 | { "read_from", (PyCFunction)ZstdCompressor_read_from, |
|
734 | { "read_from", (PyCFunction)ZstdCompressor_read_from, | |
732 | METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_from__doc__ }, |
|
735 | METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_from__doc__ }, | |
733 | { "write_to", (PyCFunction)ZstdCompressor_write_to, |
|
736 | { "write_to", (PyCFunction)ZstdCompressor_write_to, | |
734 | METH_VARARGS | METH_KEYWORDS, ZstdCompressor_write_to___doc__ }, |
|
737 | METH_VARARGS | METH_KEYWORDS, ZstdCompressor_write_to___doc__ }, | |
735 | { NULL, NULL } |
|
738 | { NULL, NULL } | |
736 | }; |
|
739 | }; | |
737 |
|
740 | |||
738 | PyTypeObject ZstdCompressorType = { |
|
741 | PyTypeObject ZstdCompressorType = { | |
739 | PyVarObject_HEAD_INIT(NULL, 0) |
|
742 | PyVarObject_HEAD_INIT(NULL, 0) | |
740 | "zstd.ZstdCompressor", /* tp_name */ |
|
743 | "zstd.ZstdCompressor", /* tp_name */ | |
741 | sizeof(ZstdCompressor), /* tp_basicsize */ |
|
744 | sizeof(ZstdCompressor), /* tp_basicsize */ | |
742 | 0, /* tp_itemsize */ |
|
745 | 0, /* tp_itemsize */ | |
743 | (destructor)ZstdCompressor_dealloc, /* tp_dealloc */ |
|
746 | (destructor)ZstdCompressor_dealloc, /* tp_dealloc */ | |
744 | 0, /* tp_print */ |
|
747 | 0, /* tp_print */ | |
745 | 0, /* tp_getattr */ |
|
748 | 0, /* tp_getattr */ | |
746 | 0, /* tp_setattr */ |
|
749 | 0, /* tp_setattr */ | |
747 | 0, /* tp_compare */ |
|
750 | 0, /* tp_compare */ | |
748 | 0, /* tp_repr */ |
|
751 | 0, /* tp_repr */ | |
749 | 0, /* tp_as_number */ |
|
752 | 0, /* tp_as_number */ | |
750 | 0, /* tp_as_sequence */ |
|
753 | 0, /* tp_as_sequence */ | |
751 | 0, /* tp_as_mapping */ |
|
754 | 0, /* tp_as_mapping */ | |
752 | 0, /* tp_hash */ |
|
755 | 0, /* tp_hash */ | |
753 | 0, /* tp_call */ |
|
756 | 0, /* tp_call */ | |
754 | 0, /* tp_str */ |
|
757 | 0, /* tp_str */ | |
755 | 0, /* tp_getattro */ |
|
758 | 0, /* tp_getattro */ | |
756 | 0, /* tp_setattro */ |
|
759 | 0, /* tp_setattro */ | |
757 | 0, /* tp_as_buffer */ |
|
760 | 0, /* tp_as_buffer */ | |
758 | Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ |
|
761 | Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ | |
759 | ZstdCompressor__doc__, /* tp_doc */ |
|
762 | ZstdCompressor__doc__, /* tp_doc */ | |
760 | 0, /* tp_traverse */ |
|
763 | 0, /* tp_traverse */ | |
761 | 0, /* tp_clear */ |
|
764 | 0, /* tp_clear */ | |
762 | 0, /* tp_richcompare */ |
|
765 | 0, /* tp_richcompare */ | |
763 | 0, /* tp_weaklistoffset */ |
|
766 | 0, /* tp_weaklistoffset */ | |
764 | 0, /* tp_iter */ |
|
767 | 0, /* tp_iter */ | |
765 | 0, /* tp_iternext */ |
|
768 | 0, /* tp_iternext */ | |
766 | ZstdCompressor_methods, /* tp_methods */ |
|
769 | ZstdCompressor_methods, /* tp_methods */ | |
767 | 0, /* tp_members */ |
|
770 | 0, /* tp_members */ | |
768 | 0, /* tp_getset */ |
|
771 | 0, /* tp_getset */ | |
769 | 0, /* tp_base */ |
|
772 | 0, /* tp_base */ | |
770 | 0, /* tp_dict */ |
|
773 | 0, /* tp_dict */ | |
771 | 0, /* tp_descr_get */ |
|
774 | 0, /* tp_descr_get */ | |
772 | 0, /* tp_descr_set */ |
|
775 | 0, /* tp_descr_set */ | |
773 | 0, /* tp_dictoffset */ |
|
776 | 0, /* tp_dictoffset */ | |
774 | (initproc)ZstdCompressor_init, /* tp_init */ |
|
777 | (initproc)ZstdCompressor_init, /* tp_init */ | |
775 | 0, /* tp_alloc */ |
|
778 | 0, /* tp_alloc */ | |
776 | PyType_GenericNew, /* tp_new */ |
|
779 | PyType_GenericNew, /* tp_new */ | |
777 | }; |
|
780 | }; | |
778 |
|
781 | |||
779 | void compressor_module_init(PyObject* mod) { |
|
782 | void compressor_module_init(PyObject* mod) { | |
780 | Py_TYPE(&ZstdCompressorType) = &PyType_Type; |
|
783 | Py_TYPE(&ZstdCompressorType) = &PyType_Type; | |
781 | if (PyType_Ready(&ZstdCompressorType) < 0) { |
|
784 | if (PyType_Ready(&ZstdCompressorType) < 0) { | |
782 | return; |
|
785 | return; | |
783 | } |
|
786 | } | |
784 |
|
787 | |||
785 | Py_INCREF((PyObject*)&ZstdCompressorType); |
|
788 | Py_INCREF((PyObject*)&ZstdCompressorType); | |
786 | PyModule_AddObject(mod, "ZstdCompressor", |
|
789 | PyModule_AddObject(mod, "ZstdCompressor", | |
787 | (PyObject*)&ZstdCompressorType); |
|
790 | (PyObject*)&ZstdCompressorType); | |
788 | } |
|
791 | } |
@@ -1,669 +1,672 b'' | |||||
1 | /** |
|
1 | /** | |
2 | * Copyright (c) 2016-present, Gregory Szorc |
|
2 | * Copyright (c) 2016-present, Gregory Szorc | |
3 | * All rights reserved. |
|
3 | * All rights reserved. | |
4 | * |
|
4 | * | |
5 | * This software may be modified and distributed under the terms |
|
5 | * This software may be modified and distributed under the terms | |
6 | * of the BSD license. See the LICENSE file for details. |
|
6 | * of the BSD license. See the LICENSE file for details. | |
7 | */ |
|
7 | */ | |
8 |
|
8 | |||
9 | #include "python-zstandard.h" |
|
9 | #include "python-zstandard.h" | |
10 |
|
10 | |||
11 | extern PyObject* ZstdError; |
|
11 | extern PyObject* ZstdError; | |
12 |
|
12 | |||
13 | ZSTD_DStream* DStream_from_ZstdDecompressor(ZstdDecompressor* decompressor) { |
|
13 | ZSTD_DStream* DStream_from_ZstdDecompressor(ZstdDecompressor* decompressor) { | |
14 | ZSTD_DStream* dstream; |
|
14 | ZSTD_DStream* dstream; | |
15 | void* dictData = NULL; |
|
15 | void* dictData = NULL; | |
16 | size_t dictSize = 0; |
|
16 | size_t dictSize = 0; | |
17 | size_t zresult; |
|
17 | size_t zresult; | |
18 |
|
18 | |||
19 | dstream = ZSTD_createDStream(); |
|
19 | dstream = ZSTD_createDStream(); | |
20 | if (!dstream) { |
|
20 | if (!dstream) { | |
21 | PyErr_SetString(ZstdError, "could not create DStream"); |
|
21 | PyErr_SetString(ZstdError, "could not create DStream"); | |
22 | return NULL; |
|
22 | return NULL; | |
23 | } |
|
23 | } | |
24 |
|
24 | |||
25 | if (decompressor->dict) { |
|
25 | if (decompressor->dict) { | |
26 | dictData = decompressor->dict->dictData; |
|
26 | dictData = decompressor->dict->dictData; | |
27 | dictSize = decompressor->dict->dictSize; |
|
27 | dictSize = decompressor->dict->dictSize; | |
28 | } |
|
28 | } | |
29 |
|
29 | |||
30 | if (dictData) { |
|
30 | if (dictData) { | |
31 | zresult = ZSTD_initDStream_usingDict(dstream, dictData, dictSize); |
|
31 | zresult = ZSTD_initDStream_usingDict(dstream, dictData, dictSize); | |
32 | } |
|
32 | } | |
33 | else { |
|
33 | else { | |
34 | zresult = ZSTD_initDStream(dstream); |
|
34 | zresult = ZSTD_initDStream(dstream); | |
35 | } |
|
35 | } | |
36 |
|
36 | |||
37 | if (ZSTD_isError(zresult)) { |
|
37 | if (ZSTD_isError(zresult)) { | |
38 | PyErr_Format(ZstdError, "could not initialize DStream: %s", |
|
38 | PyErr_Format(ZstdError, "could not initialize DStream: %s", | |
39 | ZSTD_getErrorName(zresult)); |
|
39 | ZSTD_getErrorName(zresult)); | |
40 | return NULL; |
|
40 | return NULL; | |
41 | } |
|
41 | } | |
42 |
|
42 | |||
43 | return dstream; |
|
43 | return dstream; | |
44 | } |
|
44 | } | |
45 |
|
45 | |||
46 | PyDoc_STRVAR(Decompressor__doc__, |
|
46 | PyDoc_STRVAR(Decompressor__doc__, | |
47 | "ZstdDecompressor(dict_data=None)\n" |
|
47 | "ZstdDecompressor(dict_data=None)\n" | |
48 | "\n" |
|
48 | "\n" | |
49 | "Create an object used to perform Zstandard decompression.\n" |
|
49 | "Create an object used to perform Zstandard decompression.\n" | |
50 | "\n" |
|
50 | "\n" | |
51 | "An instance can perform multiple decompression operations." |
|
51 | "An instance can perform multiple decompression operations." | |
52 | ); |
|
52 | ); | |
53 |
|
53 | |||
54 | static int Decompressor_init(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { |
|
54 | static int Decompressor_init(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { | |
55 | static char* kwlist[] = { |
|
55 | static char* kwlist[] = { | |
56 | "dict_data", |
|
56 | "dict_data", | |
57 | NULL |
|
57 | NULL | |
58 | }; |
|
58 | }; | |
59 |
|
59 | |||
60 | ZstdCompressionDict* dict = NULL; |
|
60 | ZstdCompressionDict* dict = NULL; | |
61 |
|
61 | |||
62 | self->refdctx = NULL; |
|
62 | self->refdctx = NULL; | |
63 | self->dict = NULL; |
|
63 | self->dict = NULL; | |
64 | self->ddict = NULL; |
|
64 | self->ddict = NULL; | |
65 |
|
65 | |||
66 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!", kwlist, |
|
66 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!", kwlist, | |
67 | &ZstdCompressionDictType, &dict)) { |
|
67 | &ZstdCompressionDictType, &dict)) { | |
68 | return -1; |
|
68 | return -1; | |
69 | } |
|
69 | } | |
70 |
|
70 | |||
71 | /* Instead of creating a ZSTD_DCtx for every decompression operation, |
|
71 | /* Instead of creating a ZSTD_DCtx for every decompression operation, | |
72 | we create an instance at object creation time and recycle it via |
|
72 | we create an instance at object creation time and recycle it via | |
73 | ZSTD_copyDCTx() on each use. This means each use is a malloc+memcpy |
|
73 | ZSTD_copyDCTx() on each use. This means each use is a malloc+memcpy | |
74 | instead of a malloc+init. */ |
|
74 | instead of a malloc+init. */ | |
75 | /* TODO lazily initialize the reference ZSTD_DCtx on first use since |
|
75 | /* TODO lazily initialize the reference ZSTD_DCtx on first use since | |
76 | not instances of ZstdDecompressor will use a ZSTD_DCtx. */ |
|
76 | not instances of ZstdDecompressor will use a ZSTD_DCtx. */ | |
77 | self->refdctx = ZSTD_createDCtx(); |
|
77 | self->refdctx = ZSTD_createDCtx(); | |
78 | if (!self->refdctx) { |
|
78 | if (!self->refdctx) { | |
79 | PyErr_NoMemory(); |
|
79 | PyErr_NoMemory(); | |
80 | goto except; |
|
80 | goto except; | |
81 | } |
|
81 | } | |
82 |
|
82 | |||
83 | if (dict) { |
|
83 | if (dict) { | |
84 | self->dict = dict; |
|
84 | self->dict = dict; | |
85 | Py_INCREF(dict); |
|
85 | Py_INCREF(dict); | |
86 | } |
|
86 | } | |
87 |
|
87 | |||
88 | return 0; |
|
88 | return 0; | |
89 |
|
89 | |||
90 | except: |
|
90 | except: | |
91 | if (self->refdctx) { |
|
91 | if (self->refdctx) { | |
92 | ZSTD_freeDCtx(self->refdctx); |
|
92 | ZSTD_freeDCtx(self->refdctx); | |
93 | self->refdctx = NULL; |
|
93 | self->refdctx = NULL; | |
94 | } |
|
94 | } | |
95 |
|
95 | |||
96 | return -1; |
|
96 | return -1; | |
97 | } |
|
97 | } | |
98 |
|
98 | |||
99 | static void Decompressor_dealloc(ZstdDecompressor* self) { |
|
99 | static void Decompressor_dealloc(ZstdDecompressor* self) { | |
100 | if (self->refdctx) { |
|
100 | if (self->refdctx) { | |
101 | ZSTD_freeDCtx(self->refdctx); |
|
101 | ZSTD_freeDCtx(self->refdctx); | |
102 | } |
|
102 | } | |
103 |
|
103 | |||
104 | Py_XDECREF(self->dict); |
|
104 | Py_XDECREF(self->dict); | |
105 |
|
105 | |||
106 | if (self->ddict) { |
|
106 | if (self->ddict) { | |
107 | ZSTD_freeDDict(self->ddict); |
|
107 | ZSTD_freeDDict(self->ddict); | |
108 | self->ddict = NULL; |
|
108 | self->ddict = NULL; | |
109 | } |
|
109 | } | |
110 |
|
110 | |||
111 | PyObject_Del(self); |
|
111 | PyObject_Del(self); | |
112 | } |
|
112 | } | |
113 |
|
113 | |||
114 | PyDoc_STRVAR(Decompressor_copy_stream__doc__, |
|
114 | PyDoc_STRVAR(Decompressor_copy_stream__doc__, | |
115 | "copy_stream(ifh, ofh[, read_size=default, write_size=default]) -- decompress data between streams\n" |
|
115 | "copy_stream(ifh, ofh[, read_size=default, write_size=default]) -- decompress data between streams\n" | |
116 | "\n" |
|
116 | "\n" | |
117 | "Compressed data will be read from ``ifh``, decompressed, and written to\n" |
|
117 | "Compressed data will be read from ``ifh``, decompressed, and written to\n" | |
118 | "``ofh``. ``ifh`` must have a ``read(size)`` method. ``ofh`` must have a\n" |
|
118 | "``ofh``. ``ifh`` must have a ``read(size)`` method. ``ofh`` must have a\n" | |
119 | "``write(data)`` method.\n" |
|
119 | "``write(data)`` method.\n" | |
120 | "\n" |
|
120 | "\n" | |
121 | "The optional ``read_size`` and ``write_size`` arguments control the chunk\n" |
|
121 | "The optional ``read_size`` and ``write_size`` arguments control the chunk\n" | |
122 | "size of data that is ``read()`` and ``write()`` between streams. They default\n" |
|
122 | "size of data that is ``read()`` and ``write()`` between streams. They default\n" | |
123 | "to the default input and output sizes of zstd decompressor streams.\n" |
|
123 | "to the default input and output sizes of zstd decompressor streams.\n" | |
124 | ); |
|
124 | ); | |
125 |
|
125 | |||
126 | static PyObject* Decompressor_copy_stream(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { |
|
126 | static PyObject* Decompressor_copy_stream(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { | |
127 | static char* kwlist[] = { |
|
127 | static char* kwlist[] = { | |
128 | "ifh", |
|
128 | "ifh", | |
129 | "ofh", |
|
129 | "ofh", | |
130 | "read_size", |
|
130 | "read_size", | |
131 | "write_size", |
|
131 | "write_size", | |
132 | NULL |
|
132 | NULL | |
133 | }; |
|
133 | }; | |
134 |
|
134 | |||
135 | PyObject* source; |
|
135 | PyObject* source; | |
136 | PyObject* dest; |
|
136 | PyObject* dest; | |
137 | size_t inSize = ZSTD_DStreamInSize(); |
|
137 | size_t inSize = ZSTD_DStreamInSize(); | |
138 | size_t outSize = ZSTD_DStreamOutSize(); |
|
138 | size_t outSize = ZSTD_DStreamOutSize(); | |
139 | ZSTD_DStream* dstream; |
|
139 | ZSTD_DStream* dstream; | |
140 | ZSTD_inBuffer input; |
|
140 | ZSTD_inBuffer input; | |
141 | ZSTD_outBuffer output; |
|
141 | ZSTD_outBuffer output; | |
142 | Py_ssize_t totalRead = 0; |
|
142 | Py_ssize_t totalRead = 0; | |
143 | Py_ssize_t totalWrite = 0; |
|
143 | Py_ssize_t totalWrite = 0; | |
144 | char* readBuffer; |
|
144 | char* readBuffer; | |
145 | Py_ssize_t readSize; |
|
145 | Py_ssize_t readSize; | |
146 | PyObject* readResult; |
|
146 | PyObject* readResult; | |
147 | PyObject* res = NULL; |
|
147 | PyObject* res = NULL; | |
148 | size_t zresult = 0; |
|
148 | size_t zresult = 0; | |
149 | PyObject* writeResult; |
|
149 | PyObject* writeResult; | |
150 | PyObject* totalReadPy; |
|
150 | PyObject* totalReadPy; | |
151 | PyObject* totalWritePy; |
|
151 | PyObject* totalWritePy; | |
152 |
|
152 | |||
153 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|kk", kwlist, &source, |
|
153 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|kk", kwlist, &source, | |
154 | &dest, &inSize, &outSize)) { |
|
154 | &dest, &inSize, &outSize)) { | |
155 | return NULL; |
|
155 | return NULL; | |
156 | } |
|
156 | } | |
157 |
|
157 | |||
158 | if (!PyObject_HasAttrString(source, "read")) { |
|
158 | if (!PyObject_HasAttrString(source, "read")) { | |
159 | PyErr_SetString(PyExc_ValueError, "first argument must have a read() method"); |
|
159 | PyErr_SetString(PyExc_ValueError, "first argument must have a read() method"); | |
160 | return NULL; |
|
160 | return NULL; | |
161 | } |
|
161 | } | |
162 |
|
162 | |||
163 | if (!PyObject_HasAttrString(dest, "write")) { |
|
163 | if (!PyObject_HasAttrString(dest, "write")) { | |
164 | PyErr_SetString(PyExc_ValueError, "second argument must have a write() method"); |
|
164 | PyErr_SetString(PyExc_ValueError, "second argument must have a write() method"); | |
165 | return NULL; |
|
165 | return NULL; | |
166 | } |
|
166 | } | |
167 |
|
167 | |||
|
168 | /* Prevent free on uninitialized memory in finally. */ | |||
|
169 | output.dst = NULL; | |||
|
170 | ||||
168 | dstream = DStream_from_ZstdDecompressor(self); |
|
171 | dstream = DStream_from_ZstdDecompressor(self); | |
169 | if (!dstream) { |
|
172 | if (!dstream) { | |
170 | res = NULL; |
|
173 | res = NULL; | |
171 | goto finally; |
|
174 | goto finally; | |
172 | } |
|
175 | } | |
173 |
|
176 | |||
174 | output.dst = PyMem_Malloc(outSize); |
|
177 | output.dst = PyMem_Malloc(outSize); | |
175 | if (!output.dst) { |
|
178 | if (!output.dst) { | |
176 | PyErr_NoMemory(); |
|
179 | PyErr_NoMemory(); | |
177 | res = NULL; |
|
180 | res = NULL; | |
178 | goto finally; |
|
181 | goto finally; | |
179 | } |
|
182 | } | |
180 | output.size = outSize; |
|
183 | output.size = outSize; | |
181 | output.pos = 0; |
|
184 | output.pos = 0; | |
182 |
|
185 | |||
183 | /* Read source stream until EOF */ |
|
186 | /* Read source stream until EOF */ | |
184 | while (1) { |
|
187 | while (1) { | |
185 | readResult = PyObject_CallMethod(source, "read", "n", inSize); |
|
188 | readResult = PyObject_CallMethod(source, "read", "n", inSize); | |
186 | if (!readResult) { |
|
189 | if (!readResult) { | |
187 | PyErr_SetString(ZstdError, "could not read() from source"); |
|
190 | PyErr_SetString(ZstdError, "could not read() from source"); | |
188 | goto finally; |
|
191 | goto finally; | |
189 | } |
|
192 | } | |
190 |
|
193 | |||
191 | PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); |
|
194 | PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); | |
192 |
|
195 | |||
193 | /* If no data was read, we're at EOF. */ |
|
196 | /* If no data was read, we're at EOF. */ | |
194 | if (0 == readSize) { |
|
197 | if (0 == readSize) { | |
195 | break; |
|
198 | break; | |
196 | } |
|
199 | } | |
197 |
|
200 | |||
198 | totalRead += readSize; |
|
201 | totalRead += readSize; | |
199 |
|
202 | |||
200 | /* Send data to decompressor */ |
|
203 | /* Send data to decompressor */ | |
201 | input.src = readBuffer; |
|
204 | input.src = readBuffer; | |
202 | input.size = readSize; |
|
205 | input.size = readSize; | |
203 | input.pos = 0; |
|
206 | input.pos = 0; | |
204 |
|
207 | |||
205 | while (input.pos < input.size) { |
|
208 | while (input.pos < input.size) { | |
206 | Py_BEGIN_ALLOW_THREADS |
|
209 | Py_BEGIN_ALLOW_THREADS | |
207 | zresult = ZSTD_decompressStream(dstream, &output, &input); |
|
210 | zresult = ZSTD_decompressStream(dstream, &output, &input); | |
208 | Py_END_ALLOW_THREADS |
|
211 | Py_END_ALLOW_THREADS | |
209 |
|
212 | |||
210 | if (ZSTD_isError(zresult)) { |
|
213 | if (ZSTD_isError(zresult)) { | |
211 | PyErr_Format(ZstdError, "zstd decompressor error: %s", |
|
214 | PyErr_Format(ZstdError, "zstd decompressor error: %s", | |
212 | ZSTD_getErrorName(zresult)); |
|
215 | ZSTD_getErrorName(zresult)); | |
213 | res = NULL; |
|
216 | res = NULL; | |
214 | goto finally; |
|
217 | goto finally; | |
215 | } |
|
218 | } | |
216 |
|
219 | |||
217 | if (output.pos) { |
|
220 | if (output.pos) { | |
218 | #if PY_MAJOR_VERSION >= 3 |
|
221 | #if PY_MAJOR_VERSION >= 3 | |
219 | writeResult = PyObject_CallMethod(dest, "write", "y#", |
|
222 | writeResult = PyObject_CallMethod(dest, "write", "y#", | |
220 | #else |
|
223 | #else | |
221 | writeResult = PyObject_CallMethod(dest, "write", "s#", |
|
224 | writeResult = PyObject_CallMethod(dest, "write", "s#", | |
222 | #endif |
|
225 | #endif | |
223 | output.dst, output.pos); |
|
226 | output.dst, output.pos); | |
224 |
|
227 | |||
225 | Py_XDECREF(writeResult); |
|
228 | Py_XDECREF(writeResult); | |
226 | totalWrite += output.pos; |
|
229 | totalWrite += output.pos; | |
227 | output.pos = 0; |
|
230 | output.pos = 0; | |
228 | } |
|
231 | } | |
229 | } |
|
232 | } | |
230 | } |
|
233 | } | |
231 |
|
234 | |||
232 | /* Source stream is exhausted. Finish up. */ |
|
235 | /* Source stream is exhausted. Finish up. */ | |
233 |
|
236 | |||
234 | ZSTD_freeDStream(dstream); |
|
237 | ZSTD_freeDStream(dstream); | |
235 | dstream = NULL; |
|
238 | dstream = NULL; | |
236 |
|
239 | |||
237 | totalReadPy = PyLong_FromSsize_t(totalRead); |
|
240 | totalReadPy = PyLong_FromSsize_t(totalRead); | |
238 | totalWritePy = PyLong_FromSsize_t(totalWrite); |
|
241 | totalWritePy = PyLong_FromSsize_t(totalWrite); | |
239 | res = PyTuple_Pack(2, totalReadPy, totalWritePy); |
|
242 | res = PyTuple_Pack(2, totalReadPy, totalWritePy); | |
240 | Py_DecRef(totalReadPy); |
|
243 | Py_DecRef(totalReadPy); | |
241 | Py_DecRef(totalWritePy); |
|
244 | Py_DecRef(totalWritePy); | |
242 |
|
245 | |||
243 | finally: |
|
246 | finally: | |
244 | if (output.dst) { |
|
247 | if (output.dst) { | |
245 | PyMem_Free(output.dst); |
|
248 | PyMem_Free(output.dst); | |
246 | } |
|
249 | } | |
247 |
|
250 | |||
248 | if (dstream) { |
|
251 | if (dstream) { | |
249 | ZSTD_freeDStream(dstream); |
|
252 | ZSTD_freeDStream(dstream); | |
250 | } |
|
253 | } | |
251 |
|
254 | |||
252 | return res; |
|
255 | return res; | |
253 | } |
|
256 | } | |
254 |
|
257 | |||
255 | PyDoc_STRVAR(Decompressor_decompress__doc__, |
|
258 | PyDoc_STRVAR(Decompressor_decompress__doc__, | |
256 | "decompress(data[, max_output_size=None]) -- Decompress data in its entirety\n" |
|
259 | "decompress(data[, max_output_size=None]) -- Decompress data in its entirety\n" | |
257 | "\n" |
|
260 | "\n" | |
258 | "This method will decompress the entirety of the argument and return the\n" |
|
261 | "This method will decompress the entirety of the argument and return the\n" | |
259 | "result.\n" |
|
262 | "result.\n" | |
260 | "\n" |
|
263 | "\n" | |
261 | "The input bytes are expected to contain a full Zstandard frame (something\n" |
|
264 | "The input bytes are expected to contain a full Zstandard frame (something\n" | |
262 | "compressed with ``ZstdCompressor.compress()`` or similar). If the input does\n" |
|
265 | "compressed with ``ZstdCompressor.compress()`` or similar). If the input does\n" | |
263 | "not contain a full frame, an exception will be raised.\n" |
|
266 | "not contain a full frame, an exception will be raised.\n" | |
264 | "\n" |
|
267 | "\n" | |
265 | "If the frame header of the compressed data does not contain the content size\n" |
|
268 | "If the frame header of the compressed data does not contain the content size\n" | |
266 | "``max_output_size`` must be specified or ``ZstdError`` will be raised. An\n" |
|
269 | "``max_output_size`` must be specified or ``ZstdError`` will be raised. An\n" | |
267 | "allocation of size ``max_output_size`` will be performed and an attempt will\n" |
|
270 | "allocation of size ``max_output_size`` will be performed and an attempt will\n" | |
268 | "be made to perform decompression into that buffer. If the buffer is too\n" |
|
271 | "be made to perform decompression into that buffer. If the buffer is too\n" | |
269 | "small or cannot be allocated, ``ZstdError`` will be raised. The buffer will\n" |
|
272 | "small or cannot be allocated, ``ZstdError`` will be raised. The buffer will\n" | |
270 | "be resized if it is too large.\n" |
|
273 | "be resized if it is too large.\n" | |
271 | "\n" |
|
274 | "\n" | |
272 | "Uncompressed data could be much larger than compressed data. As a result,\n" |
|
275 | "Uncompressed data could be much larger than compressed data. As a result,\n" | |
273 | "calling this function could result in a very large memory allocation being\n" |
|
276 | "calling this function could result in a very large memory allocation being\n" | |
274 | "performed to hold the uncompressed data. Therefore it is **highly**\n" |
|
277 | "performed to hold the uncompressed data. Therefore it is **highly**\n" | |
275 | "recommended to use a streaming decompression method instead of this one.\n" |
|
278 | "recommended to use a streaming decompression method instead of this one.\n" | |
276 | ); |
|
279 | ); | |
277 |
|
280 | |||
278 | PyObject* Decompressor_decompress(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { |
|
281 | PyObject* Decompressor_decompress(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { | |
279 | static char* kwlist[] = { |
|
282 | static char* kwlist[] = { | |
280 | "data", |
|
283 | "data", | |
281 | "max_output_size", |
|
284 | "max_output_size", | |
282 | NULL |
|
285 | NULL | |
283 | }; |
|
286 | }; | |
284 |
|
287 | |||
285 | const char* source; |
|
288 | const char* source; | |
286 | Py_ssize_t sourceSize; |
|
289 | Py_ssize_t sourceSize; | |
287 | Py_ssize_t maxOutputSize = 0; |
|
290 | Py_ssize_t maxOutputSize = 0; | |
288 | unsigned long long decompressedSize; |
|
291 | unsigned long long decompressedSize; | |
289 | size_t destCapacity; |
|
292 | size_t destCapacity; | |
290 | PyObject* result = NULL; |
|
293 | PyObject* result = NULL; | |
291 | ZSTD_DCtx* dctx = NULL; |
|
294 | ZSTD_DCtx* dctx = NULL; | |
292 | void* dictData = NULL; |
|
295 | void* dictData = NULL; | |
293 | size_t dictSize = 0; |
|
296 | size_t dictSize = 0; | |
294 | size_t zresult; |
|
297 | size_t zresult; | |
295 |
|
298 | |||
296 | #if PY_MAJOR_VERSION >= 3 |
|
299 | #if PY_MAJOR_VERSION >= 3 | |
297 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|n", kwlist, |
|
300 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|n", kwlist, | |
298 | #else |
|
301 | #else | |
299 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|n", kwlist, |
|
302 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|n", kwlist, | |
300 | #endif |
|
303 | #endif | |
301 | &source, &sourceSize, &maxOutputSize)) { |
|
304 | &source, &sourceSize, &maxOutputSize)) { | |
302 | return NULL; |
|
305 | return NULL; | |
303 | } |
|
306 | } | |
304 |
|
307 | |||
305 | dctx = PyMem_Malloc(ZSTD_sizeof_DCtx(self->refdctx)); |
|
308 | dctx = PyMem_Malloc(ZSTD_sizeof_DCtx(self->refdctx)); | |
306 | if (!dctx) { |
|
309 | if (!dctx) { | |
307 | PyErr_NoMemory(); |
|
310 | PyErr_NoMemory(); | |
308 | return NULL; |
|
311 | return NULL; | |
309 | } |
|
312 | } | |
310 |
|
313 | |||
311 | ZSTD_copyDCtx(dctx, self->refdctx); |
|
314 | ZSTD_copyDCtx(dctx, self->refdctx); | |
312 |
|
315 | |||
313 | if (self->dict) { |
|
316 | if (self->dict) { | |
314 | dictData = self->dict->dictData; |
|
317 | dictData = self->dict->dictData; | |
315 | dictSize = self->dict->dictSize; |
|
318 | dictSize = self->dict->dictSize; | |
316 | } |
|
319 | } | |
317 |
|
320 | |||
318 | if (dictData && !self->ddict) { |
|
321 | if (dictData && !self->ddict) { | |
319 | Py_BEGIN_ALLOW_THREADS |
|
322 | Py_BEGIN_ALLOW_THREADS | |
320 | self->ddict = ZSTD_createDDict(dictData, dictSize); |
|
323 | self->ddict = ZSTD_createDDict(dictData, dictSize); | |
321 | Py_END_ALLOW_THREADS |
|
324 | Py_END_ALLOW_THREADS | |
322 |
|
325 | |||
323 | if (!self->ddict) { |
|
326 | if (!self->ddict) { | |
324 | PyErr_SetString(ZstdError, "could not create decompression dict"); |
|
327 | PyErr_SetString(ZstdError, "could not create decompression dict"); | |
325 | goto except; |
|
328 | goto except; | |
326 | } |
|
329 | } | |
327 | } |
|
330 | } | |
328 |
|
331 | |||
329 | decompressedSize = ZSTD_getDecompressedSize(source, sourceSize); |
|
332 | decompressedSize = ZSTD_getDecompressedSize(source, sourceSize); | |
330 | /* 0 returned if content size not in the zstd frame header */ |
|
333 | /* 0 returned if content size not in the zstd frame header */ | |
331 | if (0 == decompressedSize) { |
|
334 | if (0 == decompressedSize) { | |
332 | if (0 == maxOutputSize) { |
|
335 | if (0 == maxOutputSize) { | |
333 | PyErr_SetString(ZstdError, "input data invalid or missing content size " |
|
336 | PyErr_SetString(ZstdError, "input data invalid or missing content size " | |
334 | "in frame header"); |
|
337 | "in frame header"); | |
335 | goto except; |
|
338 | goto except; | |
336 | } |
|
339 | } | |
337 | else { |
|
340 | else { | |
338 | result = PyBytes_FromStringAndSize(NULL, maxOutputSize); |
|
341 | result = PyBytes_FromStringAndSize(NULL, maxOutputSize); | |
339 | destCapacity = maxOutputSize; |
|
342 | destCapacity = maxOutputSize; | |
340 | } |
|
343 | } | |
341 | } |
|
344 | } | |
342 | else { |
|
345 | else { | |
343 | result = PyBytes_FromStringAndSize(NULL, decompressedSize); |
|
346 | result = PyBytes_FromStringAndSize(NULL, decompressedSize); | |
344 | destCapacity = decompressedSize; |
|
347 | destCapacity = decompressedSize; | |
345 | } |
|
348 | } | |
346 |
|
349 | |||
347 | if (!result) { |
|
350 | if (!result) { | |
348 | goto except; |
|
351 | goto except; | |
349 | } |
|
352 | } | |
350 |
|
353 | |||
351 | Py_BEGIN_ALLOW_THREADS |
|
354 | Py_BEGIN_ALLOW_THREADS | |
352 | if (self->ddict) { |
|
355 | if (self->ddict) { | |
353 | zresult = ZSTD_decompress_usingDDict(dctx, PyBytes_AsString(result), destCapacity, |
|
356 | zresult = ZSTD_decompress_usingDDict(dctx, PyBytes_AsString(result), destCapacity, | |
354 | source, sourceSize, self->ddict); |
|
357 | source, sourceSize, self->ddict); | |
355 | } |
|
358 | } | |
356 | else { |
|
359 | else { | |
357 | zresult = ZSTD_decompressDCtx(dctx, PyBytes_AsString(result), destCapacity, source, sourceSize); |
|
360 | zresult = ZSTD_decompressDCtx(dctx, PyBytes_AsString(result), destCapacity, source, sourceSize); | |
358 | } |
|
361 | } | |
359 | Py_END_ALLOW_THREADS |
|
362 | Py_END_ALLOW_THREADS | |
360 |
|
363 | |||
361 | if (ZSTD_isError(zresult)) { |
|
364 | if (ZSTD_isError(zresult)) { | |
362 | PyErr_Format(ZstdError, "decompression error: %s", ZSTD_getErrorName(zresult)); |
|
365 | PyErr_Format(ZstdError, "decompression error: %s", ZSTD_getErrorName(zresult)); | |
363 | goto except; |
|
366 | goto except; | |
364 | } |
|
367 | } | |
365 | else if (decompressedSize && zresult != decompressedSize) { |
|
368 | else if (decompressedSize && zresult != decompressedSize) { | |
366 | PyErr_Format(ZstdError, "decompression error: decompressed %zu bytes; expected %llu", |
|
369 | PyErr_Format(ZstdError, "decompression error: decompressed %zu bytes; expected %llu", | |
367 | zresult, decompressedSize); |
|
370 | zresult, decompressedSize); | |
368 | goto except; |
|
371 | goto except; | |
369 | } |
|
372 | } | |
370 | else if (zresult < destCapacity) { |
|
373 | else if (zresult < destCapacity) { | |
371 | if (_PyBytes_Resize(&result, zresult)) { |
|
374 | if (_PyBytes_Resize(&result, zresult)) { | |
372 | goto except; |
|
375 | goto except; | |
373 | } |
|
376 | } | |
374 | } |
|
377 | } | |
375 |
|
378 | |||
376 | goto finally; |
|
379 | goto finally; | |
377 |
|
380 | |||
378 | except: |
|
381 | except: | |
379 | Py_DecRef(result); |
|
382 | Py_DecRef(result); | |
380 | result = NULL; |
|
383 | result = NULL; | |
381 |
|
384 | |||
382 | finally: |
|
385 | finally: | |
383 | if (dctx) { |
|
386 | if (dctx) { | |
384 | PyMem_FREE(dctx); |
|
387 | PyMem_FREE(dctx); | |
385 | } |
|
388 | } | |
386 |
|
389 | |||
387 | return result; |
|
390 | return result; | |
388 | } |
|
391 | } | |
389 |
|
392 | |||
390 | PyDoc_STRVAR(Decompressor_decompressobj__doc__, |
|
393 | PyDoc_STRVAR(Decompressor_decompressobj__doc__, | |
391 | "decompressobj()\n" |
|
394 | "decompressobj()\n" | |
392 | "\n" |
|
395 | "\n" | |
393 | "Incrementally feed data into a decompressor.\n" |
|
396 | "Incrementally feed data into a decompressor.\n" | |
394 | "\n" |
|
397 | "\n" | |
395 | "The returned object exposes a ``decompress(data)`` method. This makes it\n" |
|
398 | "The returned object exposes a ``decompress(data)`` method. This makes it\n" | |
396 | "compatible with ``zlib.decompressobj`` and ``bz2.BZ2Decompressor`` so that\n" |
|
399 | "compatible with ``zlib.decompressobj`` and ``bz2.BZ2Decompressor`` so that\n" | |
397 | "callers can swap in the zstd decompressor while using the same API.\n" |
|
400 | "callers can swap in the zstd decompressor while using the same API.\n" | |
398 | ); |
|
401 | ); | |
399 |
|
402 | |||
400 | static ZstdDecompressionObj* Decompressor_decompressobj(ZstdDecompressor* self) { |
|
403 | static ZstdDecompressionObj* Decompressor_decompressobj(ZstdDecompressor* self) { | |
401 | ZstdDecompressionObj* result = PyObject_New(ZstdDecompressionObj, &ZstdDecompressionObjType); |
|
404 | ZstdDecompressionObj* result = PyObject_New(ZstdDecompressionObj, &ZstdDecompressionObjType); | |
402 | if (!result) { |
|
405 | if (!result) { | |
403 | return NULL; |
|
406 | return NULL; | |
404 | } |
|
407 | } | |
405 |
|
408 | |||
406 | result->dstream = DStream_from_ZstdDecompressor(self); |
|
409 | result->dstream = DStream_from_ZstdDecompressor(self); | |
407 | if (!result->dstream) { |
|
410 | if (!result->dstream) { | |
408 | Py_DecRef((PyObject*)result); |
|
411 | Py_DecRef((PyObject*)result); | |
409 | return NULL; |
|
412 | return NULL; | |
410 | } |
|
413 | } | |
411 |
|
414 | |||
412 | result->decompressor = self; |
|
415 | result->decompressor = self; | |
413 | Py_INCREF(result->decompressor); |
|
416 | Py_INCREF(result->decompressor); | |
414 |
|
417 | |||
415 | result->finished = 0; |
|
418 | result->finished = 0; | |
416 |
|
419 | |||
417 | return result; |
|
420 | return result; | |
418 | } |
|
421 | } | |
419 |
|
422 | |||
420 | PyDoc_STRVAR(Decompressor_read_from__doc__, |
|
423 | PyDoc_STRVAR(Decompressor_read_from__doc__, | |
421 | "read_from(reader[, read_size=default, write_size=default, skip_bytes=0])\n" |
|
424 | "read_from(reader[, read_size=default, write_size=default, skip_bytes=0])\n" | |
422 | "Read compressed data and return an iterator\n" |
|
425 | "Read compressed data and return an iterator\n" | |
423 | "\n" |
|
426 | "\n" | |
424 | "Returns an iterator of decompressed data chunks produced from reading from\n" |
|
427 | "Returns an iterator of decompressed data chunks produced from reading from\n" | |
425 | "the ``reader``.\n" |
|
428 | "the ``reader``.\n" | |
426 | "\n" |
|
429 | "\n" | |
427 | "Compressed data will be obtained from ``reader`` by calling the\n" |
|
430 | "Compressed data will be obtained from ``reader`` by calling the\n" | |
428 | "``read(size)`` method of it. The source data will be streamed into a\n" |
|
431 | "``read(size)`` method of it. The source data will be streamed into a\n" | |
429 | "decompressor. As decompressed data is available, it will be exposed to the\n" |
|
432 | "decompressor. As decompressed data is available, it will be exposed to the\n" | |
430 | "returned iterator.\n" |
|
433 | "returned iterator.\n" | |
431 | "\n" |
|
434 | "\n" | |
432 | "Data is ``read()`` in chunks of size ``read_size`` and exposed to the\n" |
|
435 | "Data is ``read()`` in chunks of size ``read_size`` and exposed to the\n" | |
433 | "iterator in chunks of size ``write_size``. The default values are the input\n" |
|
436 | "iterator in chunks of size ``write_size``. The default values are the input\n" | |
434 | "and output sizes for a zstd streaming decompressor.\n" |
|
437 | "and output sizes for a zstd streaming decompressor.\n" | |
435 | "\n" |
|
438 | "\n" | |
436 | "There is also support for skipping the first ``skip_bytes`` of data from\n" |
|
439 | "There is also support for skipping the first ``skip_bytes`` of data from\n" | |
437 | "the source.\n" |
|
440 | "the source.\n" | |
438 | ); |
|
441 | ); | |
439 |
|
442 | |||
440 | static ZstdDecompressorIterator* Decompressor_read_from(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { |
|
443 | static ZstdDecompressorIterator* Decompressor_read_from(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { | |
441 | static char* kwlist[] = { |
|
444 | static char* kwlist[] = { | |
442 | "reader", |
|
445 | "reader", | |
443 | "read_size", |
|
446 | "read_size", | |
444 | "write_size", |
|
447 | "write_size", | |
445 | "skip_bytes", |
|
448 | "skip_bytes", | |
446 | NULL |
|
449 | NULL | |
447 | }; |
|
450 | }; | |
448 |
|
451 | |||
449 | PyObject* reader; |
|
452 | PyObject* reader; | |
450 | size_t inSize = ZSTD_DStreamInSize(); |
|
453 | size_t inSize = ZSTD_DStreamInSize(); | |
451 | size_t outSize = ZSTD_DStreamOutSize(); |
|
454 | size_t outSize = ZSTD_DStreamOutSize(); | |
452 | ZstdDecompressorIterator* result; |
|
455 | ZstdDecompressorIterator* result; | |
453 | size_t skipBytes = 0; |
|
456 | size_t skipBytes = 0; | |
454 |
|
457 | |||
455 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk", kwlist, &reader, |
|
458 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk", kwlist, &reader, | |
456 | &inSize, &outSize, &skipBytes)) { |
|
459 | &inSize, &outSize, &skipBytes)) { | |
457 | return NULL; |
|
460 | return NULL; | |
458 | } |
|
461 | } | |
459 |
|
462 | |||
460 | if (skipBytes >= inSize) { |
|
463 | if (skipBytes >= inSize) { | |
461 | PyErr_SetString(PyExc_ValueError, |
|
464 | PyErr_SetString(PyExc_ValueError, | |
462 | "skip_bytes must be smaller than read_size"); |
|
465 | "skip_bytes must be smaller than read_size"); | |
463 | return NULL; |
|
466 | return NULL; | |
464 | } |
|
467 | } | |
465 |
|
468 | |||
466 | result = PyObject_New(ZstdDecompressorIterator, &ZstdDecompressorIteratorType); |
|
469 | result = PyObject_New(ZstdDecompressorIterator, &ZstdDecompressorIteratorType); | |
467 | if (!result) { |
|
470 | if (!result) { | |
468 | return NULL; |
|
471 | return NULL; | |
469 | } |
|
472 | } | |
470 |
|
473 | |||
471 | result->decompressor = NULL; |
|
474 | result->decompressor = NULL; | |
472 | result->reader = NULL; |
|
475 | result->reader = NULL; | |
473 | result->buffer = NULL; |
|
476 | result->buffer = NULL; | |
474 | result->dstream = NULL; |
|
477 | result->dstream = NULL; | |
475 | result->input.src = NULL; |
|
478 | result->input.src = NULL; | |
476 | result->output.dst = NULL; |
|
479 | result->output.dst = NULL; | |
477 |
|
480 | |||
478 | if (PyObject_HasAttrString(reader, "read")) { |
|
481 | if (PyObject_HasAttrString(reader, "read")) { | |
479 | result->reader = reader; |
|
482 | result->reader = reader; | |
480 | Py_INCREF(result->reader); |
|
483 | Py_INCREF(result->reader); | |
481 | } |
|
484 | } | |
482 | else if (1 == PyObject_CheckBuffer(reader)) { |
|
485 | else if (1 == PyObject_CheckBuffer(reader)) { | |
483 | /* Object claims it is a buffer. Try to get a handle to it. */ |
|
486 | /* Object claims it is a buffer. Try to get a handle to it. */ | |
484 | result->buffer = PyMem_Malloc(sizeof(Py_buffer)); |
|
487 | result->buffer = PyMem_Malloc(sizeof(Py_buffer)); | |
485 | if (!result->buffer) { |
|
488 | if (!result->buffer) { | |
486 | goto except; |
|
489 | goto except; | |
487 | } |
|
490 | } | |
488 |
|
491 | |||
489 | memset(result->buffer, 0, sizeof(Py_buffer)); |
|
492 | memset(result->buffer, 0, sizeof(Py_buffer)); | |
490 |
|
493 | |||
491 | if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) { |
|
494 | if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) { | |
492 | goto except; |
|
495 | goto except; | |
493 | } |
|
496 | } | |
494 |
|
497 | |||
495 | result->bufferOffset = 0; |
|
498 | result->bufferOffset = 0; | |
496 | } |
|
499 | } | |
497 | else { |
|
500 | else { | |
498 | PyErr_SetString(PyExc_ValueError, |
|
501 | PyErr_SetString(PyExc_ValueError, | |
499 | "must pass an object with a read() method or conforms to buffer protocol"); |
|
502 | "must pass an object with a read() method or conforms to buffer protocol"); | |
500 | goto except; |
|
503 | goto except; | |
501 | } |
|
504 | } | |
502 |
|
505 | |||
503 | result->decompressor = self; |
|
506 | result->decompressor = self; | |
504 | Py_INCREF(result->decompressor); |
|
507 | Py_INCREF(result->decompressor); | |
505 |
|
508 | |||
506 | result->inSize = inSize; |
|
509 | result->inSize = inSize; | |
507 | result->outSize = outSize; |
|
510 | result->outSize = outSize; | |
508 | result->skipBytes = skipBytes; |
|
511 | result->skipBytes = skipBytes; | |
509 |
|
512 | |||
510 | result->dstream = DStream_from_ZstdDecompressor(self); |
|
513 | result->dstream = DStream_from_ZstdDecompressor(self); | |
511 | if (!result->dstream) { |
|
514 | if (!result->dstream) { | |
512 | goto except; |
|
515 | goto except; | |
513 | } |
|
516 | } | |
514 |
|
517 | |||
515 | result->input.src = PyMem_Malloc(inSize); |
|
518 | result->input.src = PyMem_Malloc(inSize); | |
516 | if (!result->input.src) { |
|
519 | if (!result->input.src) { | |
517 | PyErr_NoMemory(); |
|
520 | PyErr_NoMemory(); | |
518 | goto except; |
|
521 | goto except; | |
519 | } |
|
522 | } | |
520 | result->input.size = 0; |
|
523 | result->input.size = 0; | |
521 | result->input.pos = 0; |
|
524 | result->input.pos = 0; | |
522 |
|
525 | |||
523 | result->output.dst = NULL; |
|
526 | result->output.dst = NULL; | |
524 | result->output.size = 0; |
|
527 | result->output.size = 0; | |
525 | result->output.pos = 0; |
|
528 | result->output.pos = 0; | |
526 |
|
529 | |||
527 | result->readCount = 0; |
|
530 | result->readCount = 0; | |
528 | result->finishedInput = 0; |
|
531 | result->finishedInput = 0; | |
529 | result->finishedOutput = 0; |
|
532 | result->finishedOutput = 0; | |
530 |
|
533 | |||
531 | goto finally; |
|
534 | goto finally; | |
532 |
|
535 | |||
533 | except: |
|
536 | except: | |
534 | if (result->reader) { |
|
537 | if (result->reader) { | |
535 | Py_DECREF(result->reader); |
|
538 | Py_DECREF(result->reader); | |
536 | result->reader = NULL; |
|
539 | result->reader = NULL; | |
537 | } |
|
540 | } | |
538 |
|
541 | |||
539 | if (result->buffer) { |
|
542 | if (result->buffer) { | |
540 | PyBuffer_Release(result->buffer); |
|
543 | PyBuffer_Release(result->buffer); | |
541 | Py_DECREF(result->buffer); |
|
544 | Py_DECREF(result->buffer); | |
542 | result->buffer = NULL; |
|
545 | result->buffer = NULL; | |
543 | } |
|
546 | } | |
544 |
|
547 | |||
545 | Py_DECREF(result); |
|
548 | Py_DECREF(result); | |
546 | result = NULL; |
|
549 | result = NULL; | |
547 |
|
550 | |||
548 | finally: |
|
551 | finally: | |
549 |
|
552 | |||
550 | return result; |
|
553 | return result; | |
551 | } |
|
554 | } | |
552 |
|
555 | |||
553 | PyDoc_STRVAR(Decompressor_write_to__doc__, |
|
556 | PyDoc_STRVAR(Decompressor_write_to__doc__, | |
554 | "Create a context manager to write decompressed data to an object.\n" |
|
557 | "Create a context manager to write decompressed data to an object.\n" | |
555 | "\n" |
|
558 | "\n" | |
556 | "The passed object must have a ``write()`` method.\n" |
|
559 | "The passed object must have a ``write()`` method.\n" | |
557 | "\n" |
|
560 | "\n" | |
558 | "The caller feeds intput data to the object by calling ``write(data)``.\n" |
|
561 | "The caller feeds intput data to the object by calling ``write(data)``.\n" | |
559 | "Decompressed data is written to the argument given as it is decompressed.\n" |
|
562 | "Decompressed data is written to the argument given as it is decompressed.\n" | |
560 | "\n" |
|
563 | "\n" | |
561 | "An optional ``write_size`` argument defines the size of chunks to\n" |
|
564 | "An optional ``write_size`` argument defines the size of chunks to\n" | |
562 | "``write()`` to the writer. It defaults to the default output size for a zstd\n" |
|
565 | "``write()`` to the writer. It defaults to the default output size for a zstd\n" | |
563 | "streaming decompressor.\n" |
|
566 | "streaming decompressor.\n" | |
564 | ); |
|
567 | ); | |
565 |
|
568 | |||
566 | static ZstdDecompressionWriter* Decompressor_write_to(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { |
|
569 | static ZstdDecompressionWriter* Decompressor_write_to(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { | |
567 | static char* kwlist[] = { |
|
570 | static char* kwlist[] = { | |
568 | "writer", |
|
571 | "writer", | |
569 | "write_size", |
|
572 | "write_size", | |
570 | NULL |
|
573 | NULL | |
571 | }; |
|
574 | }; | |
572 |
|
575 | |||
573 | PyObject* writer; |
|
576 | PyObject* writer; | |
574 | size_t outSize = ZSTD_DStreamOutSize(); |
|
577 | size_t outSize = ZSTD_DStreamOutSize(); | |
575 | ZstdDecompressionWriter* result; |
|
578 | ZstdDecompressionWriter* result; | |
576 |
|
579 | |||
577 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k", kwlist, &writer, &outSize)) { |
|
580 | if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k", kwlist, &writer, &outSize)) { | |
578 | return NULL; |
|
581 | return NULL; | |
579 | } |
|
582 | } | |
580 |
|
583 | |||
581 | if (!PyObject_HasAttrString(writer, "write")) { |
|
584 | if (!PyObject_HasAttrString(writer, "write")) { | |
582 | PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method"); |
|
585 | PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method"); | |
583 | return NULL; |
|
586 | return NULL; | |
584 | } |
|
587 | } | |
585 |
|
588 | |||
586 | result = PyObject_New(ZstdDecompressionWriter, &ZstdDecompressionWriterType); |
|
589 | result = PyObject_New(ZstdDecompressionWriter, &ZstdDecompressionWriterType); | |
587 | if (!result) { |
|
590 | if (!result) { | |
588 | return NULL; |
|
591 | return NULL; | |
589 | } |
|
592 | } | |
590 |
|
593 | |||
591 | result->decompressor = self; |
|
594 | result->decompressor = self; | |
592 | Py_INCREF(result->decompressor); |
|
595 | Py_INCREF(result->decompressor); | |
593 |
|
596 | |||
594 | result->writer = writer; |
|
597 | result->writer = writer; | |
595 | Py_INCREF(result->writer); |
|
598 | Py_INCREF(result->writer); | |
596 |
|
599 | |||
597 | result->outSize = outSize; |
|
600 | result->outSize = outSize; | |
598 |
|
601 | |||
599 | result->entered = 0; |
|
602 | result->entered = 0; | |
600 | result->dstream = NULL; |
|
603 | result->dstream = NULL; | |
601 |
|
604 | |||
602 | return result; |
|
605 | return result; | |
603 | } |
|
606 | } | |
604 |
|
607 | |||
605 | static PyMethodDef Decompressor_methods[] = { |
|
608 | static PyMethodDef Decompressor_methods[] = { | |
606 | { "copy_stream", (PyCFunction)Decompressor_copy_stream, METH_VARARGS | METH_KEYWORDS, |
|
609 | { "copy_stream", (PyCFunction)Decompressor_copy_stream, METH_VARARGS | METH_KEYWORDS, | |
607 | Decompressor_copy_stream__doc__ }, |
|
610 | Decompressor_copy_stream__doc__ }, | |
608 | { "decompress", (PyCFunction)Decompressor_decompress, METH_VARARGS | METH_KEYWORDS, |
|
611 | { "decompress", (PyCFunction)Decompressor_decompress, METH_VARARGS | METH_KEYWORDS, | |
609 | Decompressor_decompress__doc__ }, |
|
612 | Decompressor_decompress__doc__ }, | |
610 | { "decompressobj", (PyCFunction)Decompressor_decompressobj, METH_NOARGS, |
|
613 | { "decompressobj", (PyCFunction)Decompressor_decompressobj, METH_NOARGS, | |
611 | Decompressor_decompressobj__doc__ }, |
|
614 | Decompressor_decompressobj__doc__ }, | |
612 | { "read_from", (PyCFunction)Decompressor_read_from, METH_VARARGS | METH_KEYWORDS, |
|
615 | { "read_from", (PyCFunction)Decompressor_read_from, METH_VARARGS | METH_KEYWORDS, | |
613 | Decompressor_read_from__doc__ }, |
|
616 | Decompressor_read_from__doc__ }, | |
614 | { "write_to", (PyCFunction)Decompressor_write_to, METH_VARARGS | METH_KEYWORDS, |
|
617 | { "write_to", (PyCFunction)Decompressor_write_to, METH_VARARGS | METH_KEYWORDS, | |
615 | Decompressor_write_to__doc__ }, |
|
618 | Decompressor_write_to__doc__ }, | |
616 | { NULL, NULL } |
|
619 | { NULL, NULL } | |
617 | }; |
|
620 | }; | |
618 |
|
621 | |||
619 | PyTypeObject ZstdDecompressorType = { |
|
622 | PyTypeObject ZstdDecompressorType = { | |
620 | PyVarObject_HEAD_INIT(NULL, 0) |
|
623 | PyVarObject_HEAD_INIT(NULL, 0) | |
621 | "zstd.ZstdDecompressor", /* tp_name */ |
|
624 | "zstd.ZstdDecompressor", /* tp_name */ | |
622 | sizeof(ZstdDecompressor), /* tp_basicsize */ |
|
625 | sizeof(ZstdDecompressor), /* tp_basicsize */ | |
623 | 0, /* tp_itemsize */ |
|
626 | 0, /* tp_itemsize */ | |
624 | (destructor)Decompressor_dealloc, /* tp_dealloc */ |
|
627 | (destructor)Decompressor_dealloc, /* tp_dealloc */ | |
625 | 0, /* tp_print */ |
|
628 | 0, /* tp_print */ | |
626 | 0, /* tp_getattr */ |
|
629 | 0, /* tp_getattr */ | |
627 | 0, /* tp_setattr */ |
|
630 | 0, /* tp_setattr */ | |
628 | 0, /* tp_compare */ |
|
631 | 0, /* tp_compare */ | |
629 | 0, /* tp_repr */ |
|
632 | 0, /* tp_repr */ | |
630 | 0, /* tp_as_number */ |
|
633 | 0, /* tp_as_number */ | |
631 | 0, /* tp_as_sequence */ |
|
634 | 0, /* tp_as_sequence */ | |
632 | 0, /* tp_as_mapping */ |
|
635 | 0, /* tp_as_mapping */ | |
633 | 0, /* tp_hash */ |
|
636 | 0, /* tp_hash */ | |
634 | 0, /* tp_call */ |
|
637 | 0, /* tp_call */ | |
635 | 0, /* tp_str */ |
|
638 | 0, /* tp_str */ | |
636 | 0, /* tp_getattro */ |
|
639 | 0, /* tp_getattro */ | |
637 | 0, /* tp_setattro */ |
|
640 | 0, /* tp_setattro */ | |
638 | 0, /* tp_as_buffer */ |
|
641 | 0, /* tp_as_buffer */ | |
639 | Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ |
|
642 | Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ | |
640 | Decompressor__doc__, /* tp_doc */ |
|
643 | Decompressor__doc__, /* tp_doc */ | |
641 | 0, /* tp_traverse */ |
|
644 | 0, /* tp_traverse */ | |
642 | 0, /* tp_clear */ |
|
645 | 0, /* tp_clear */ | |
643 | 0, /* tp_richcompare */ |
|
646 | 0, /* tp_richcompare */ | |
644 | 0, /* tp_weaklistoffset */ |
|
647 | 0, /* tp_weaklistoffset */ | |
645 | 0, /* tp_iter */ |
|
648 | 0, /* tp_iter */ | |
646 | 0, /* tp_iternext */ |
|
649 | 0, /* tp_iternext */ | |
647 | Decompressor_methods, /* tp_methods */ |
|
650 | Decompressor_methods, /* tp_methods */ | |
648 | 0, /* tp_members */ |
|
651 | 0, /* tp_members */ | |
649 | 0, /* tp_getset */ |
|
652 | 0, /* tp_getset */ | |
650 | 0, /* tp_base */ |
|
653 | 0, /* tp_base */ | |
651 | 0, /* tp_dict */ |
|
654 | 0, /* tp_dict */ | |
652 | 0, /* tp_descr_get */ |
|
655 | 0, /* tp_descr_get */ | |
653 | 0, /* tp_descr_set */ |
|
656 | 0, /* tp_descr_set */ | |
654 | 0, /* tp_dictoffset */ |
|
657 | 0, /* tp_dictoffset */ | |
655 | (initproc)Decompressor_init, /* tp_init */ |
|
658 | (initproc)Decompressor_init, /* tp_init */ | |
656 | 0, /* tp_alloc */ |
|
659 | 0, /* tp_alloc */ | |
657 | PyType_GenericNew, /* tp_new */ |
|
660 | PyType_GenericNew, /* tp_new */ | |
658 | }; |
|
661 | }; | |
659 |
|
662 | |||
660 | void decompressor_module_init(PyObject* mod) { |
|
663 | void decompressor_module_init(PyObject* mod) { | |
661 | Py_TYPE(&ZstdDecompressorType) = &PyType_Type; |
|
664 | Py_TYPE(&ZstdDecompressorType) = &PyType_Type; | |
662 | if (PyType_Ready(&ZstdDecompressorType) < 0) { |
|
665 | if (PyType_Ready(&ZstdDecompressorType) < 0) { | |
663 | return; |
|
666 | return; | |
664 | } |
|
667 | } | |
665 |
|
668 | |||
666 | Py_INCREF((PyObject*)&ZstdDecompressorType); |
|
669 | Py_INCREF((PyObject*)&ZstdDecompressorType); | |
667 | PyModule_AddObject(mod, "ZstdDecompressor", |
|
670 | PyModule_AddObject(mod, "ZstdDecompressor", | |
668 | (PyObject*)&ZstdDecompressorType); |
|
671 | (PyObject*)&ZstdDecompressorType); | |
669 | } |
|
672 | } |
General Comments 0
You need to be logged in to leave comments.
Login now