aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Modules/_zstd
diff options
context:
space:
mode:
Diffstat (limited to 'Modules/_zstd')
-rw-r--r--Modules/_zstd/_zstdmodule.c855
-rw-r--r--Modules/_zstd/_zstdmodule.h183
-rw-r--r--Modules/_zstd/buffer.h22
-rw-r--r--Modules/_zstd/clinic/_zstdmodule.c.h173
-rw-r--r--Modules/_zstd/clinic/compressor.c.h67
-rw-r--r--Modules/_zstd/clinic/decompressor.c.h31
-rw-r--r--Modules/_zstd/clinic/zstddict.c.h (renamed from Modules/_zstd/clinic/zdict.c.h)122
-rw-r--r--Modules/_zstd/compressor.c654
-rw-r--r--Modules/_zstd/decompressor.c566
-rw-r--r--Modules/_zstd/zdict.c286
-rw-r--r--Modules/_zstd/zstddict.c273
-rw-r--r--Modules/_zstd/zstddict.h29
12 files changed, 1477 insertions, 1784 deletions
diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c
index 18dc13b3fd1..d75c0779474 100644
--- a/Modules/_zstd/_zstdmodule.c
+++ b/Modules/_zstd/_zstdmodule.c
@@ -1,14 +1,16 @@
-/*
-Low level interface to Meta's zstd library for use in the compression.zstd
-Python module.
-*/
+/* Low level interface to the Zstandard algorthm & the zstd library. */
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif
+#include "Python.h"
+
#include "_zstdmodule.h"
+#include <zstd.h> // ZSTD_*()
+#include <zdict.h> // ZDICT_*()
+
/*[clinic input]
module _zstd
@@ -17,52 +19,91 @@ module _zstd
#include "clinic/_zstdmodule.c.h"
+ZstdDict *
+_Py_parse_zstd_dict(const _zstd_state *state, PyObject *dict, int *ptype)
+{
+ if (state == NULL) {
+ return NULL;
+ }
+
+ /* Check ZstdDict */
+ if (PyObject_TypeCheck(dict, state->ZstdDict_type)) {
+ return (ZstdDict*)dict;
+ }
+
+ /* Check (ZstdDict, type) */
+ if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2
+ && PyObject_TypeCheck(PyTuple_GET_ITEM(dict, 0), state->ZstdDict_type)
+ && PyLong_Check(PyTuple_GET_ITEM(dict, 1)))
+ {
+ int type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1));
+ if (type == -1 && PyErr_Occurred()) {
+ return NULL;
+ }
+ if (type == DICT_TYPE_DIGESTED
+ || type == DICT_TYPE_UNDIGESTED
+ || type == DICT_TYPE_PREFIX)
+ {
+ *ptype = type;
+ return (ZstdDict*)PyTuple_GET_ITEM(dict, 0);
+ }
+ }
+
+ /* Wrong type */
+ PyErr_SetString(PyExc_TypeError,
+ "zstd_dict argument should be a ZstdDict object.");
+ return NULL;
+}
+
/* Format error message and set ZstdError. */
void
-set_zstd_error(const _zstd_state* const state,
- error_type type, size_t zstd_ret)
+set_zstd_error(const _zstd_state *state, error_type type, size_t zstd_ret)
{
- char *msg;
+ const char *msg;
assert(ZSTD_isError(zstd_ret));
- switch (type)
- {
- case ERR_DECOMPRESS:
- msg = "Unable to decompress zstd data: %s";
- break;
- case ERR_COMPRESS:
- msg = "Unable to compress zstd data: %s";
- break;
- case ERR_SET_PLEDGED_INPUT_SIZE:
- msg = "Unable to set pledged uncompressed content size: %s";
- break;
-
- case ERR_LOAD_D_DICT:
- msg = "Unable to load zstd dictionary or prefix for decompression: %s";
- break;
- case ERR_LOAD_C_DICT:
- msg = "Unable to load zstd dictionary or prefix for compression: %s";
- break;
-
- case ERR_GET_C_BOUNDS:
- msg = "Unable to get zstd compression parameter bounds: %s";
- break;
- case ERR_GET_D_BOUNDS:
- msg = "Unable to get zstd decompression parameter bounds: %s";
- break;
- case ERR_SET_C_LEVEL:
- msg = "Unable to set zstd compression level: %s";
- break;
-
- case ERR_TRAIN_DICT:
- msg = "Unable to train zstd dictionary: %s";
- break;
- case ERR_FINALIZE_DICT:
- msg = "Unable to finalize zstd dictionary: %s";
- break;
-
- default:
- Py_UNREACHABLE();
+ if (state == NULL) {
+ return;
+ }
+ switch (type) {
+ case ERR_DECOMPRESS:
+ msg = "Unable to decompress Zstandard data: %s";
+ break;
+ case ERR_COMPRESS:
+ msg = "Unable to compress Zstandard data: %s";
+ break;
+ case ERR_SET_PLEDGED_INPUT_SIZE:
+ msg = "Unable to set pledged uncompressed content size: %s";
+ break;
+
+ case ERR_LOAD_D_DICT:
+ msg = "Unable to load Zstandard dictionary or prefix for "
+ "decompression: %s";
+ break;
+ case ERR_LOAD_C_DICT:
+ msg = "Unable to load Zstandard dictionary or prefix for "
+ "compression: %s";
+ break;
+
+ case ERR_GET_C_BOUNDS:
+ msg = "Unable to get zstd compression parameter bounds: %s";
+ break;
+ case ERR_GET_D_BOUNDS:
+ msg = "Unable to get zstd decompression parameter bounds: %s";
+ break;
+ case ERR_SET_C_LEVEL:
+ msg = "Unable to set zstd compression level: %s";
+ break;
+
+ case ERR_TRAIN_DICT:
+ msg = "Unable to train the Zstandard dictionary: %s";
+ break;
+ case ERR_FINALIZE_DICT:
+ msg = "Unable to finalize the Zstandard dictionary: %s";
+ break;
+
+ default:
+ Py_UNREACHABLE();
}
PyErr_Format(state->ZstdError, msg, ZSTD_getErrorName(zstd_ret));
}
@@ -72,48 +113,43 @@ typedef struct {
char parameter_name[32];
} ParameterInfo;
-static const ParameterInfo cp_list[] =
-{
- {ZSTD_c_compressionLevel, "compressionLevel"},
- {ZSTD_c_windowLog, "windowLog"},
- {ZSTD_c_hashLog, "hashLog"},
- {ZSTD_c_chainLog, "chainLog"},
- {ZSTD_c_searchLog, "searchLog"},
- {ZSTD_c_minMatch, "minMatch"},
- {ZSTD_c_targetLength, "targetLength"},
+static const ParameterInfo cp_list[] = {
+ {ZSTD_c_compressionLevel, "compression_level"},
+ {ZSTD_c_windowLog, "window_log"},
+ {ZSTD_c_hashLog, "hash_log"},
+ {ZSTD_c_chainLog, "chain_log"},
+ {ZSTD_c_searchLog, "search_log"},
+ {ZSTD_c_minMatch, "min_match"},
+ {ZSTD_c_targetLength, "target_length"},
{ZSTD_c_strategy, "strategy"},
- {ZSTD_c_enableLongDistanceMatching, "enableLongDistanceMatching"},
- {ZSTD_c_ldmHashLog, "ldmHashLog"},
- {ZSTD_c_ldmMinMatch, "ldmMinMatch"},
- {ZSTD_c_ldmBucketSizeLog, "ldmBucketSizeLog"},
- {ZSTD_c_ldmHashRateLog, "ldmHashRateLog"},
+ {ZSTD_c_enableLongDistanceMatching, "enable_long_distance_matching"},
+ {ZSTD_c_ldmHashLog, "ldm_hash_log"},
+ {ZSTD_c_ldmMinMatch, "ldm_min_match"},
+ {ZSTD_c_ldmBucketSizeLog, "ldm_bucket_size_log"},
+ {ZSTD_c_ldmHashRateLog, "ldm_hash_rate_log"},
- {ZSTD_c_contentSizeFlag, "contentSizeFlag"},
- {ZSTD_c_checksumFlag, "checksumFlag"},
- {ZSTD_c_dictIDFlag, "dictIDFlag"},
+ {ZSTD_c_contentSizeFlag, "content_size_flag"},
+ {ZSTD_c_checksumFlag, "checksum_flag"},
+ {ZSTD_c_dictIDFlag, "dict_id_flag"},
- {ZSTD_c_nbWorkers, "nbWorkers"},
- {ZSTD_c_jobSize, "jobSize"},
- {ZSTD_c_overlapLog, "overlapLog"}
+ {ZSTD_c_nbWorkers, "nb_workers"},
+ {ZSTD_c_jobSize, "job_size"},
+ {ZSTD_c_overlapLog, "overlap_log"}
};
-static const ParameterInfo dp_list[] =
-{
- {ZSTD_d_windowLogMax, "windowLogMax"}
+static const ParameterInfo dp_list[] = {
+ {ZSTD_d_windowLogMax, "window_log_max"}
};
void
-set_parameter_error(const _zstd_state* const state, int is_compress,
- int key_v, int value_v)
+set_parameter_error(int is_compress, int key_v, int value_v)
{
ParameterInfo const *list;
int list_size;
- char const *name;
char *type;
ZSTD_bounds bounds;
- int i;
- char pos_msg[128];
+ char pos_msg[64];
if (is_compress) {
list = cp_list;
@@ -127,8 +163,8 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
}
/* Find parameter's name */
- name = NULL;
- for (i = 0; i < list_size; i++) {
+ char const *name = NULL;
+ for (int i = 0; i < list_size; i++) {
if (key_v == (list+i)->parameter) {
name = (list+i)->parameter_name;
break;
@@ -150,20 +186,16 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
bounds = ZSTD_dParam_getBounds(key_v);
}
if (ZSTD_isError(bounds.error)) {
- PyErr_Format(state->ZstdError,
- "Zstd %s parameter \"%s\" is invalid. (zstd v%s)",
- type, name, ZSTD_versionString());
+ PyErr_Format(PyExc_ValueError, "invalid %s parameter '%s'",
+ type, name);
return;
}
/* Error message */
- PyErr_Format(state->ZstdError,
- "Error when setting zstd %s parameter \"%s\", it "
- "should %d <= value <= %d, provided value is %d. "
- "(zstd v%s, %d-bit build)",
- type, name,
- bounds.lowerBound, bounds.upperBound, value_v,
- ZSTD_versionString(), 8*(int)sizeof(Py_ssize_t));
+ PyErr_Format(PyExc_ValueError,
+ "%s parameter '%s' received an illegal value %d; "
+ "the valid range is [%d, %d]",
+ type, name, value_v, bounds.lowerBound, bounds.upperBound);
}
static inline _zstd_state*
@@ -174,72 +206,90 @@ get_zstd_state(PyObject *module)
return (_zstd_state *)state;
}
+static Py_ssize_t
+calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes,
+ size_t **chunk_sizes)
+{
+ Py_ssize_t chunks_number;
+ Py_ssize_t sizes_sum;
+ Py_ssize_t i;
+
+ chunks_number = PyTuple_GET_SIZE(samples_sizes);
+ if ((size_t) chunks_number > UINT32_MAX) {
+ PyErr_Format(PyExc_ValueError,
+ "The number of samples should be <= %u.", UINT32_MAX);
+ return -1;
+ }
+
+ /* Prepare chunk_sizes */
+ *chunk_sizes = PyMem_New(size_t, chunks_number);
+ if (*chunk_sizes == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+
+ sizes_sum = PyBytes_GET_SIZE(samples_bytes);
+ for (i = 0; i < chunks_number; i++) {
+ size_t size = PyLong_AsSize_t(PyTuple_GET_ITEM(samples_sizes, i));
+ (*chunk_sizes)[i] = size;
+ if (size == (size_t)-1 && PyErr_Occurred()) {
+ if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
+ goto sum_error;
+ }
+ return -1;
+ }
+ if ((size_t)sizes_sum < size) {
+ goto sum_error;
+ }
+ sizes_sum -= size;
+ }
+
+ if (sizes_sum != 0) {
+sum_error:
+ PyErr_SetString(PyExc_ValueError,
+ "The samples size tuple doesn't match the "
+ "concatenation's size.");
+ return -1;
+ }
+ return chunks_number;
+}
+
/*[clinic input]
-_zstd._train_dict
+_zstd.train_dict
samples_bytes: PyBytesObject
Concatenation of samples.
- samples_size_list: object(subclass_of='&PyList_Type')
- List of samples' sizes.
+ samples_sizes: object(subclass_of='&PyTuple_Type')
+ Tuple of samples' sizes.
dict_size: Py_ssize_t
The size of the dictionary.
/
-Internal function, train a zstd dictionary on sample data.
+Train a Zstandard dictionary on sample data.
[clinic start generated code]*/
static PyObject *
-_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
- PyObject *samples_size_list, Py_ssize_t dict_size)
-/*[clinic end generated code: output=ee53c34c8f77886b input=b21d092c695a3a81]*/
+_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
+ PyObject *samples_sizes, Py_ssize_t dict_size)
+/*[clinic end generated code: output=8e87fe43935e8f77 input=d20dedb21c72cb62]*/
{
- // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict
- // are pretty similar. We should see if we can refactor them to share that code.
- Py_ssize_t chunks_number;
- size_t *chunk_sizes = NULL;
PyObject *dst_dict_bytes = NULL;
+ size_t *chunk_sizes = NULL;
+ Py_ssize_t chunks_number;
size_t zstd_ret;
- Py_ssize_t sizes_sum;
- Py_ssize_t i;
/* Check arguments */
if (dict_size <= 0) {
- PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number.");
- return NULL;
- }
-
- chunks_number = Py_SIZE(samples_size_list);
- if ((size_t) chunks_number > UINT32_MAX) {
- PyErr_Format(PyExc_ValueError,
- "The number of samples should be <= %u.", UINT32_MAX);
+ PyErr_SetString(PyExc_ValueError,
+ "dict_size argument should be positive number.");
return NULL;
}
- /* Prepare chunk_sizes */
- chunk_sizes = PyMem_New(size_t, chunks_number);
- if (chunk_sizes == NULL) {
- PyErr_NoMemory();
- goto error;
- }
-
- sizes_sum = 0;
- for (i = 0; i < chunks_number; i++) {
- PyObject *size = PyList_GetItemRef(samples_size_list, i);
- chunk_sizes[i] = PyLong_AsSize_t(size);
- Py_DECREF(size);
- if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) {
- PyErr_Format(PyExc_ValueError,
- "Items in samples_size_list should be an int "
- "object, with a value between 0 and %u.", SIZE_MAX);
- goto error;
- }
- sizes_sum += chunk_sizes[i];
- }
-
- if (sizes_sum != Py_SIZE(samples_bytes)) {
- PyErr_SetString(PyExc_ValueError,
- "The samples size list doesn't match the concatenation's size.");
+ /* Check that the samples are valid and get their sizes */
+ chunks_number = calculate_samples_stats(samples_bytes, samples_sizes,
+ &chunk_sizes);
+ if (chunks_number < 0) {
goto error;
}
@@ -251,16 +301,16 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
/* Train the dictionary */
char *dst_dict_buffer = PyBytes_AS_STRING(dst_dict_bytes);
- char *samples_buffer = PyBytes_AS_STRING(samples_bytes);
+ const char *samples_buffer = PyBytes_AS_STRING(samples_bytes);
Py_BEGIN_ALLOW_THREADS
zstd_ret = ZDICT_trainFromBuffer(dst_dict_buffer, dict_size,
samples_buffer,
chunk_sizes, (uint32_t)chunks_number);
Py_END_ALLOW_THREADS
- /* Check zstd dict error */
+ /* Check Zstandard dict error */
if (ZDICT_isError(zstd_ret)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret);
goto error;
}
@@ -281,74 +331,47 @@ success:
}
/*[clinic input]
-_zstd._finalize_dict
+_zstd.finalize_dict
custom_dict_bytes: PyBytesObject
Custom dictionary content.
samples_bytes: PyBytesObject
Concatenation of samples.
- samples_size_list: object(subclass_of='&PyList_Type')
- List of samples' sizes.
+ samples_sizes: object(subclass_of='&PyTuple_Type')
+ Tuple of samples' sizes.
dict_size: Py_ssize_t
The size of the dictionary.
compression_level: int
- Optimize for a specific zstd compression level, 0 means default.
+ Optimize for a specific Zstandard compression level, 0 means default.
/
-Internal function, finalize a zstd dictionary.
+Finalize a Zstandard dictionary.
[clinic start generated code]*/
static PyObject *
-_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
- PyBytesObject *samples_bytes,
- PyObject *samples_size_list, Py_ssize_t dict_size,
- int compression_level)
-/*[clinic end generated code: output=9c2a7d8c845cee93 input=08531a803d87c56f]*/
+_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
+ PyBytesObject *samples_bytes,
+ PyObject *samples_sizes, Py_ssize_t dict_size,
+ int compression_level)
+/*[clinic end generated code: output=f91821ba5ae85bda input=3c7e2480aa08fb56]*/
{
Py_ssize_t chunks_number;
size_t *chunk_sizes = NULL;
PyObject *dst_dict_bytes = NULL;
size_t zstd_ret;
ZDICT_params_t params;
- Py_ssize_t sizes_sum;
- Py_ssize_t i;
/* Check arguments */
if (dict_size <= 0) {
- PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number.");
- return NULL;
- }
-
- chunks_number = Py_SIZE(samples_size_list);
- if ((size_t) chunks_number > UINT32_MAX) {
- PyErr_Format(PyExc_ValueError,
- "The number of samples should be <= %u.", UINT32_MAX);
+ PyErr_SetString(PyExc_ValueError,
+ "dict_size argument should be positive number.");
return NULL;
}
- /* Prepare chunk_sizes */
- chunk_sizes = PyMem_New(size_t, chunks_number);
- if (chunk_sizes == NULL) {
- PyErr_NoMemory();
- goto error;
- }
-
- sizes_sum = 0;
- for (i = 0; i < chunks_number; i++) {
- PyObject *size = PyList_GET_ITEM(samples_size_list, i);
- chunk_sizes[i] = PyLong_AsSize_t(size);
- if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) {
- PyErr_Format(PyExc_ValueError,
- "Items in samples_size_list should be an int "
- "object, with a value between 0 and %u.", SIZE_MAX);
- goto error;
- }
- sizes_sum += chunk_sizes[i];
- }
-
- if (sizes_sum != Py_SIZE(samples_bytes)) {
- PyErr_SetString(PyExc_ValueError,
- "The samples size list doesn't match the concatenation's size.");
+ /* Check that the samples are valid and get their sizes */
+ chunks_number = calculate_samples_stats(samples_bytes, samples_sizes,
+ &chunk_sizes);
+ if (chunks_number < 0) {
goto error;
}
@@ -360,7 +383,7 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
/* Parameters */
- /* Optimize for a specific zstd compression level, 0 means default. */
+ /* Optimize for a specific Zstandard compression level, 0 means default. */
params.compressionLevel = compression_level;
/* Write log to stderr, 0 = none. */
params.notificationLevel = 0;
@@ -371,14 +394,15 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
Py_BEGIN_ALLOW_THREADS
zstd_ret = ZDICT_finalizeDictionary(
PyBytes_AS_STRING(dst_dict_bytes), dict_size,
- PyBytes_AS_STRING(custom_dict_bytes), Py_SIZE(custom_dict_bytes),
+ PyBytes_AS_STRING(custom_dict_bytes),
+ Py_SIZE(custom_dict_bytes),
PyBytes_AS_STRING(samples_bytes), chunk_sizes,
(uint32_t)chunks_number, params);
Py_END_ALLOW_THREADS
- /* Check zstd dict error */
+ /* Check Zstandard dict error */
if (ZDICT_isError(zstd_ret)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret);
goto error;
}
@@ -400,26 +424,25 @@ success:
/*[clinic input]
-_zstd._get_param_bounds
+_zstd.get_param_bounds
- is_compress: bool
- True for CParameter, False for DParameter.
parameter: int
The parameter to get bounds.
+ is_compress: bool
+ True for CompressionParameter, False for DecompressionParameter.
-Internal function, get CParameter/DParameter bounds.
+Get CompressionParameter/DecompressionParameter bounds.
[clinic start generated code]*/
static PyObject *
-_zstd__get_param_bounds_impl(PyObject *module, int is_compress,
- int parameter)
-/*[clinic end generated code: output=b751dc710f89ef55 input=fb21ff96aff65df1]*/
+_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress)
+/*[clinic end generated code: output=4acf5a876f0620ca input=45742ef0a3531b65]*/
{
ZSTD_bounds bound;
if (is_compress) {
bound = ZSTD_cParam_getBounds(parameter);
if (ZSTD_isError(bound.error)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_GET_C_BOUNDS, bound.error);
return NULL;
}
@@ -427,7 +450,7 @@ _zstd__get_param_bounds_impl(PyObject *module, int is_compress,
else {
bound = ZSTD_dParam_getBounds(parameter);
if (ZSTD_isError(bound.error)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_GET_D_BOUNDS, bound.error);
return NULL;
}
@@ -443,24 +466,23 @@ _zstd.get_frame_size
A bytes-like object, it should start from the beginning of a frame,
and contains at least one complete frame.
-Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.
-
-It will iterate all blocks' headers within a frame, to accumulate the frame size.
+Get the size of a Zstandard frame, including the header and optional checksum.
[clinic start generated code]*/
static PyObject *
_zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
-/*[clinic end generated code: output=a7384c2f8780f442 input=7d3ad24311893bf3]*/
+/*[clinic end generated code: output=a7384c2f8780f442 input=3b9f73f8c8129d38]*/
{
size_t frame_size;
- frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf, frame_buffer->len);
+ frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf,
+ frame_buffer->len);
if (ZSTD_isError(frame_size)) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
PyErr_Format(mod_state->ZstdError,
- "Error when finding the compressed size of a zstd frame. "
- "Make sure the frame_buffer argument starts from the "
- "beginning of a frame, and its length not less than this "
+ "Error when finding the compressed size of a Zstandard frame. "
+ "Ensure the frame_buffer argument starts from the "
+ "beginning of a frame, and its length is not less than this "
"complete frame. Zstd error message: %s.",
ZSTD_getErrorName(frame_size));
return NULL;
@@ -470,17 +492,17 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
}
/*[clinic input]
-_zstd._get_frame_info
+_zstd.get_frame_info
frame_buffer: Py_buffer
- A bytes-like object, containing the header of a zstd frame.
+ A bytes-like object, containing the header of a Zstandard frame.
-Internal function, get zstd frame infomation from a frame header.
+Get Zstandard frame infomation from a frame header.
[clinic start generated code]*/
static PyObject *
-_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
-/*[clinic end generated code: output=5462855464ecdf81 input=67f1f8e4b7b89c4d]*/
+_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
+/*[clinic end generated code: output=56e033cf48001929 input=94b240583ae22ca5]*/
{
uint64_t decompressed_size;
uint32_t dict_id;
@@ -492,12 +514,12 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
/* #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) */
if (decompressed_size == ZSTD_CONTENTSIZE_ERROR) {
- _zstd_state* const mod_state = get_zstd_state(module);
+ _zstd_state* mod_state = get_zstd_state(module);
PyErr_SetString(mod_state->ZstdError,
"Error when getting information from the header of "
- "a zstd frame. Make sure the frame_buffer argument "
+ "a Zstandard frame. Ensure the frame_buffer argument "
"starts from the beginning of a frame, and its length "
- "not less than the frame header (6~18 bytes).");
+ "is not less than the frame header (6~18 bytes).");
return NULL;
}
@@ -512,325 +534,169 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
}
/*[clinic input]
-_zstd._set_parameter_types
+_zstd.set_parameter_types
c_parameter_type: object(subclass_of='&PyType_Type')
- CParameter IntEnum type object
+ CompressionParameter IntEnum type object
d_parameter_type: object(subclass_of='&PyType_Type')
- DParameter IntEnum type object
+ DecompressionParameter IntEnum type object
-Internal function, set CParameter/DParameter types for validity check.
+Set CompressionParameter and DecompressionParameter types for validity check.
[clinic start generated code]*/
static PyObject *
-_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
- PyObject *d_parameter_type)
-/*[clinic end generated code: output=a13d4890ccbd2873 input=3e7d0d37c3a1045a]*/
+_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
+ PyObject *d_parameter_type)
+/*[clinic end generated code: output=f3313b1294f19502 input=75d7a953580fae5f]*/
{
- _zstd_state* const mod_state = get_zstd_state(module);
-
- if (!PyType_Check(c_parameter_type) || !PyType_Check(d_parameter_type)) {
- PyErr_SetString(PyExc_ValueError,
- "The two arguments should be CParameter and "
- "DParameter types.");
- return NULL;
- }
+ _zstd_state* mod_state = get_zstd_state(module);
- Py_XDECREF(mod_state->CParameter_type);
Py_INCREF(c_parameter_type);
- mod_state->CParameter_type = (PyTypeObject*) c_parameter_type;
-
- Py_XDECREF(mod_state->DParameter_type);
+ Py_XSETREF(mod_state->CParameter_type, (PyTypeObject*)c_parameter_type);
Py_INCREF(d_parameter_type);
- mod_state->DParameter_type = (PyTypeObject*)d_parameter_type;
+ Py_XSETREF(mod_state->DParameter_type, (PyTypeObject*)d_parameter_type);
Py_RETURN_NONE;
}
static PyMethodDef _zstd_methods[] = {
- _ZSTD__TRAIN_DICT_METHODDEF
- _ZSTD__FINALIZE_DICT_METHODDEF
- _ZSTD__GET_PARAM_BOUNDS_METHODDEF
+ _ZSTD_TRAIN_DICT_METHODDEF
+ _ZSTD_FINALIZE_DICT_METHODDEF
+ _ZSTD_GET_PARAM_BOUNDS_METHODDEF
_ZSTD_GET_FRAME_SIZE_METHODDEF
- _ZSTD__GET_FRAME_INFO_METHODDEF
- _ZSTD__SET_PARAMETER_TYPES_METHODDEF
-
- {0}
+ _ZSTD_GET_FRAME_INFO_METHODDEF
+ _ZSTD_SET_PARAMETER_TYPES_METHODDEF
+ {NULL, NULL}
};
-
-#define ADD_INT_PREFIX_MACRO(module, macro) \
- do { \
- if (PyModule_AddIntConstant(module, "_" #macro, macro) < 0) { \
- return -1; \
- } \
- } while(0)
-
static int
-add_parameters(PyObject *module)
-{
- /* If add new parameters, please also add to cp_list/dp_list above. */
-
- /* Compression parameters */
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_compressionLevel);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_windowLog);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_hashLog);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_chainLog);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_searchLog);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy);
-
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmMinMatch);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmBucketSizeLog);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashRateLog);
-
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_contentSizeFlag);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_checksumFlag);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_dictIDFlag);
-
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_nbWorkers);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_jobSize);
- ADD_INT_PREFIX_MACRO(module, ZSTD_c_overlapLog);
-
- /* Decompression parameters */
- ADD_INT_PREFIX_MACRO(module, ZSTD_d_windowLogMax);
-
- /* ZSTD_strategy enum */
- ADD_INT_PREFIX_MACRO(module, ZSTD_fast);
- ADD_INT_PREFIX_MACRO(module, ZSTD_dfast);
- ADD_INT_PREFIX_MACRO(module, ZSTD_greedy);
- ADD_INT_PREFIX_MACRO(module, ZSTD_lazy);
- ADD_INT_PREFIX_MACRO(module, ZSTD_lazy2);
- ADD_INT_PREFIX_MACRO(module, ZSTD_btlazy2);
- ADD_INT_PREFIX_MACRO(module, ZSTD_btopt);
- ADD_INT_PREFIX_MACRO(module, ZSTD_btultra);
- ADD_INT_PREFIX_MACRO(module, ZSTD_btultra2);
-
- return 0;
-}
-
-static inline PyObject *
-get_zstd_version_info(void)
-{
- uint32_t ver = ZSTD_versionNumber();
- uint32_t major, minor, release;
-
- major = ver / 10000;
- minor = (ver / 100) % 100;
- release = ver % 100;
-
- return Py_BuildValue("III", major, minor, release);
-}
-
-static inline int
-add_vars_to_module(PyObject *module)
-{
- PyObject *obj;
-
- /* zstd_version, a str. */
- if (PyModule_AddStringConstant(module, "zstd_version",
- ZSTD_versionString()) < 0) {
- return -1;
- }
-
- /* zstd_version_info, a tuple. */
- obj = get_zstd_version_info();
- if (PyModule_AddObjectRef(module, "zstd_version_info", obj) < 0) {
- Py_XDECREF(obj);
- return -1;
- }
- Py_DECREF(obj);
-
- /* Add zstd parameters */
- if (add_parameters(module) < 0) {
- return -1;
- }
-
- /* _compressionLevel_values: (default, min, max)
- ZSTD_defaultCLevel() was added in zstd v1.5.0 */
- obj = Py_BuildValue("iii",
-#if ZSTD_VERSION_NUMBER < 10500
- ZSTD_CLEVEL_DEFAULT,
-#else
- ZSTD_defaultCLevel(),
-#endif
- ZSTD_minCLevel(),
- ZSTD_maxCLevel());
- if (PyModule_AddObjectRef(module,
- "_compressionLevel_values",
- obj) < 0) {
- Py_XDECREF(obj);
- return -1;
- }
- Py_DECREF(obj);
-
- /* _ZSTD_CStreamSizes */
- obj = Py_BuildValue("II",
- (uint32_t)ZSTD_CStreamInSize(),
- (uint32_t)ZSTD_CStreamOutSize());
- if (PyModule_AddObjectRef(module, "_ZSTD_CStreamSizes", obj) < 0) {
- Py_XDECREF(obj);
- return -1;
- }
- Py_DECREF(obj);
-
- /* _ZSTD_DStreamSizes */
- obj = Py_BuildValue("II",
- (uint32_t)ZSTD_DStreamInSize(),
- (uint32_t)ZSTD_DStreamOutSize());
- if (PyModule_AddObjectRef(module, "_ZSTD_DStreamSizes", obj) < 0) {
- Py_XDECREF(obj);
- return -1;
- }
- Py_DECREF(obj);
-
- /* _ZSTD_CONFIG */
- obj = Py_BuildValue("isOOO", 8*(int)sizeof(Py_ssize_t), "c",
- Py_False,
- Py_True,
-/* User mremap output buffer */
-#if defined(HAVE_MREMAP)
- Py_True
-#else
- Py_False
-#endif
- );
- if (PyModule_AddObjectRef(module, "_ZSTD_CONFIG", obj) < 0) {
- Py_XDECREF(obj);
- return -1;
- }
- Py_DECREF(obj);
-
- return 0;
-}
-
-#define ADD_STR_TO_STATE_MACRO(STR) \
- do { \
- mod_state->str_##STR = PyUnicode_FromString(#STR); \
- if (mod_state->str_##STR == NULL) { \
- return -1; \
- } \
- } while(0)
-
-static inline int
-add_type_to_module(PyObject *module, const char *name,
- PyType_Spec *type_spec, PyTypeObject **dest)
-{
- PyObject *temp = PyType_FromModuleAndSpec(module, type_spec, NULL);
-
- if (PyModule_AddObjectRef(module, name, temp) < 0) {
- Py_XDECREF(temp);
- return -1;
- }
-
- *dest = (PyTypeObject*) temp;
-
- return 0;
-}
-
-static inline int
-add_constant_to_type(PyTypeObject *type, const char *name, long value)
+_zstd_exec(PyObject *m)
{
- PyObject *temp;
-
- temp = PyLong_FromLong(value);
- if (temp == NULL) {
- return -1;
- }
-
- int rc = PyObject_SetAttrString((PyObject*) type, name, temp);
- Py_DECREF(temp);
- return rc;
-}
-
-static int _zstd_exec(PyObject *module) {
- _zstd_state* const mod_state = get_zstd_state(module);
+#define ADD_TYPE(TYPE, SPEC) \
+do { \
+ TYPE = (PyTypeObject *)PyType_FromModuleAndSpec(m, &(SPEC), NULL); \
+ if (TYPE == NULL) { \
+ return -1; \
+ } \
+ if (PyModule_AddType(m, TYPE) < 0) { \
+ return -1; \
+ } \
+} while (0)
+
+#define ADD_INT_MACRO(MACRO) \
+ if (PyModule_AddIntConstant((m), #MACRO, (MACRO)) < 0) { \
+ return -1; \
+ }
+
+#define ADD_INT_CONST_TO_TYPE(TYPE, NAME, VALUE) \
+do { \
+ PyObject *v = PyLong_FromLong((VALUE)); \
+ if (v == NULL || PyObject_SetAttrString((PyObject *)(TYPE), \
+ (NAME), v) < 0) { \
+ Py_XDECREF(v); \
+ return -1; \
+ } \
+ Py_DECREF(v); \
+} while (0)
+
+ _zstd_state* mod_state = get_zstd_state(m);
/* Reusable objects & variables */
- mod_state->empty_bytes = PyBytes_FromStringAndSize(NULL, 0);
- if (mod_state->empty_bytes == NULL) {
- return -1;
- }
-
- mod_state->empty_readonly_memoryview =
- PyMemoryView_FromMemory((char*)mod_state, 0, PyBUF_READ);
- if (mod_state->empty_readonly_memoryview == NULL) {
- return -1;
- }
-
- /* Add str to module state */
- ADD_STR_TO_STATE_MACRO(read);
- ADD_STR_TO_STATE_MACRO(readinto);
- ADD_STR_TO_STATE_MACRO(write);
- ADD_STR_TO_STATE_MACRO(flush);
-
mod_state->CParameter_type = NULL;
mod_state->DParameter_type = NULL;
- /* Add variables to module */
- if (add_vars_to_module(module) < 0) {
- return -1;
- }
-
- /* ZstdError */
+ /* Create and add heap types */
+ ADD_TYPE(mod_state->ZstdDict_type, zstd_dict_type_spec);
+ ADD_TYPE(mod_state->ZstdCompressor_type, zstd_compressor_type_spec);
+ ADD_TYPE(mod_state->ZstdDecompressor_type, zstd_decompressor_type_spec);
mod_state->ZstdError = PyErr_NewExceptionWithDoc(
- "_zstd.ZstdError",
- "Call to the underlying zstd library failed.",
- NULL, NULL);
+ "compression.zstd.ZstdError",
+ "An error occurred in the zstd library.",
+ NULL, NULL);
if (mod_state->ZstdError == NULL) {
return -1;
}
-
- if (PyModule_AddObjectRef(module, "ZstdError", mod_state->ZstdError) < 0) {
- Py_DECREF(mod_state->ZstdError);
+ if (PyModule_AddType(m, (PyTypeObject *)mod_state->ZstdError) < 0) {
return -1;
}
- /* ZstdDict */
- if (add_type_to_module(module,
- "ZstdDict",
- &zstddict_type_spec,
- &mod_state->ZstdDict_type) < 0) {
+ /* Add constants */
+ if (PyModule_AddIntConstant(m, "zstd_version_number",
+ ZSTD_versionNumber()) < 0) {
return -1;
}
- // ZstdCompressor
- if (add_type_to_module(module,
- "ZstdCompressor",
- &zstdcompressor_type_spec,
- &mod_state->ZstdCompressor_type) < 0) {
+ if (PyModule_AddStringConstant(m, "zstd_version",
+ ZSTD_versionString()) < 0) {
return -1;
}
- // Add EndDirective enum to ZstdCompressor
- if (add_constant_to_type(mod_state->ZstdCompressor_type,
- "CONTINUE",
- ZSTD_e_continue) < 0) {
+#if ZSTD_VERSION_NUMBER >= 10500
+ if (PyModule_AddIntConstant(m, "ZSTD_CLEVEL_DEFAULT",
+ ZSTD_defaultCLevel()) < 0) {
return -1;
}
+#else
+ ADD_INT_MACRO(ZSTD_CLEVEL_DEFAULT);
+#endif
- if (add_constant_to_type(mod_state->ZstdCompressor_type,
- "FLUSH_BLOCK",
- ZSTD_e_flush) < 0) {
+ if (PyModule_Add(m, "ZSTD_DStreamOutSize",
+ PyLong_FromSize_t(ZSTD_DStreamOutSize())) < 0) {
return -1;
}
- if (add_constant_to_type(mod_state->ZstdCompressor_type,
- "FLUSH_FRAME",
- ZSTD_e_end) < 0) {
- return -1;
- }
-
- // ZstdDecompressor
- if (add_type_to_module(module,
- "ZstdDecompressor",
- &ZstdDecompressor_type_spec,
- &mod_state->ZstdDecompressor_type) < 0) {
- return -1;
- }
+ /* Add zstd compression parameters. All should also be in cp_list. */
+ ADD_INT_MACRO(ZSTD_c_compressionLevel);
+ ADD_INT_MACRO(ZSTD_c_windowLog);
+ ADD_INT_MACRO(ZSTD_c_hashLog);
+ ADD_INT_MACRO(ZSTD_c_chainLog);
+ ADD_INT_MACRO(ZSTD_c_searchLog);
+ ADD_INT_MACRO(ZSTD_c_minMatch);
+ ADD_INT_MACRO(ZSTD_c_targetLength);
+ ADD_INT_MACRO(ZSTD_c_strategy);
+
+ ADD_INT_MACRO(ZSTD_c_enableLongDistanceMatching);
+ ADD_INT_MACRO(ZSTD_c_ldmHashLog);
+ ADD_INT_MACRO(ZSTD_c_ldmMinMatch);
+ ADD_INT_MACRO(ZSTD_c_ldmBucketSizeLog);
+ ADD_INT_MACRO(ZSTD_c_ldmHashRateLog);
+
+ ADD_INT_MACRO(ZSTD_c_contentSizeFlag);
+ ADD_INT_MACRO(ZSTD_c_checksumFlag);
+ ADD_INT_MACRO(ZSTD_c_dictIDFlag);
+
+ ADD_INT_MACRO(ZSTD_c_nbWorkers);
+ ADD_INT_MACRO(ZSTD_c_jobSize);
+ ADD_INT_MACRO(ZSTD_c_overlapLog);
+
+ /* Add zstd decompression parameters. All should also be in dp_list. */
+ ADD_INT_MACRO(ZSTD_d_windowLogMax);
+
+ /* Add ZSTD_strategy enum members */
+ ADD_INT_MACRO(ZSTD_fast);
+ ADD_INT_MACRO(ZSTD_dfast);
+ ADD_INT_MACRO(ZSTD_greedy);
+ ADD_INT_MACRO(ZSTD_lazy);
+ ADD_INT_MACRO(ZSTD_lazy2);
+ ADD_INT_MACRO(ZSTD_btlazy2);
+ ADD_INT_MACRO(ZSTD_btopt);
+ ADD_INT_MACRO(ZSTD_btultra);
+ ADD_INT_MACRO(ZSTD_btultra2);
+
+ /* Add ZSTD_EndDirective enum members to ZstdCompressor */
+ ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type,
+ "CONTINUE", ZSTD_e_continue);
+ ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type,
+ "FLUSH_BLOCK", ZSTD_e_flush);
+ ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type,
+ "FLUSH_FRAME", ZSTD_e_end);
+
+ /* Make ZstdCompressor immutable (set Py_TPFLAGS_IMMUTABLETYPE) */
+ PyType_Freeze(mod_state->ZstdCompressor_type);
+
+#undef ADD_TYPE
+#undef ADD_INT_MACRO
+#undef ADD_ZSTD_COMPRESSOR_INT_CONST
return 0;
}
@@ -838,14 +704,7 @@ static int _zstd_exec(PyObject *module) {
static int
_zstd_traverse(PyObject *module, visitproc visit, void *arg)
{
- _zstd_state* const mod_state = get_zstd_state(module);
-
- Py_VISIT(mod_state->empty_bytes);
- Py_VISIT(mod_state->empty_readonly_memoryview);
- Py_VISIT(mod_state->str_read);
- Py_VISIT(mod_state->str_readinto);
- Py_VISIT(mod_state->str_write);
- Py_VISIT(mod_state->str_flush);
+ _zstd_state* mod_state = get_zstd_state(module);
Py_VISIT(mod_state->ZstdDict_type);
Py_VISIT(mod_state->ZstdCompressor_type);
@@ -862,14 +721,7 @@ _zstd_traverse(PyObject *module, visitproc visit, void *arg)
static int
_zstd_clear(PyObject *module)
{
- _zstd_state* const mod_state = get_zstd_state(module);
-
- Py_CLEAR(mod_state->empty_bytes);
- Py_CLEAR(mod_state->empty_readonly_memoryview);
- Py_CLEAR(mod_state->str_read);
- Py_CLEAR(mod_state->str_readinto);
- Py_CLEAR(mod_state->str_write);
- Py_CLEAR(mod_state->str_flush);
+ _zstd_state* mod_state = get_zstd_state(module);
Py_CLEAR(mod_state->ZstdDict_type);
Py_CLEAR(mod_state->ZstdCompressor_type);
@@ -891,20 +743,21 @@ _zstd_free(void *module)
static struct PyModuleDef_Slot _zstd_slots[] = {
{Py_mod_exec, _zstd_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
-
- {0}
+ {0, NULL},
};
-struct PyModuleDef _zstdmodule = {
- PyModuleDef_HEAD_INIT,
+static struct PyModuleDef _zstdmodule = {
+ .m_base = PyModuleDef_HEAD_INIT,
.m_name = "_zstd",
+ .m_doc = "Implementation module for Zstandard compression.",
.m_size = sizeof(_zstd_state),
.m_slots = _zstd_slots,
.m_methods = _zstd_methods,
.m_traverse = _zstd_traverse,
.m_clear = _zstd_clear,
- .m_free = _zstd_free
+ .m_free = _zstd_free,
};
PyMODINIT_FUNC
diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h
index d50f1489e6f..4e8f708f223 100644
--- a/Modules/_zstd/_zstdmodule.h
+++ b/Modules/_zstd/_zstdmodule.h
@@ -1,146 +1,28 @@
-#pragma once
-/*
-Low level interface to Meta's zstd library for use in the compression.zstd
-Python module.
-*/
+/* Low level interface to the Zstandard algorthm & the zstd library. */
/* Declarations shared between different parts of the _zstd module*/
-#include "Python.h"
+#ifndef ZSTD_MODULE_H
+#define ZSTD_MODULE_H
-#include "zstd.h"
-#include "zdict.h"
+#include "zstddict.h"
+/* Type specs */
+extern PyType_Spec zstd_dict_type_spec;
+extern PyType_Spec zstd_compressor_type_spec;
+extern PyType_Spec zstd_decompressor_type_spec;
-// if you update the minimum version, you should update the compile
-// check in configure.ac
-#define PYTHON_MINIMUM_SUPPORTED_ZSTD_VERSION 10405
-
-#if ZSTD_VERSION_NUMBER < PYTHON_MINIMUM_SUPPORTED_ZSTD_VERSION
- #error "_zstd module requires zstd v1.4.5+"
-#endif
-
-/* Forward declaration of module state */
-typedef struct _zstd_state _zstd_state;
-
-/* Forward reference of module def */
-extern PyModuleDef _zstdmodule;
-
-/* For clinic type calculations */
-static inline _zstd_state *
-get_zstd_state_from_type(PyTypeObject *type) {
- PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule);
- if (module == NULL) {
- return NULL;
- }
- void *state = PyModule_GetState(module);
- assert(state != NULL);
- return (_zstd_state *)state;
-}
-
-extern PyType_Spec zstddict_type_spec;
-extern PyType_Spec zstdcompressor_type_spec;
-extern PyType_Spec ZstdDecompressor_type_spec;
-
-struct _zstd_state {
- PyObject *empty_bytes;
- PyObject *empty_readonly_memoryview;
- PyObject *str_read;
- PyObject *str_readinto;
- PyObject *str_write;
- PyObject *str_flush;
-
+typedef struct {
+ /* Module heap types. */
PyTypeObject *ZstdDict_type;
PyTypeObject *ZstdCompressor_type;
PyTypeObject *ZstdDecompressor_type;
PyObject *ZstdError;
+ /* enum types set by set_parameter_types. */
PyTypeObject *CParameter_type;
PyTypeObject *DParameter_type;
-};
-
-typedef struct {
- PyObject_HEAD
-
- /* Reusable compress/decompress dictionary, they are created once and
- can be shared by multiple threads concurrently, since its usage is
- read-only.
- c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */
- ZSTD_DDict *d_dict;
- PyObject *c_dicts;
-
- /* Content of the dictionary, bytes object. */
- PyObject *dict_content;
- /* Dictionary id */
- uint32_t dict_id;
-
- /* __init__ has been called, 0 or 1. */
- int inited;
-} ZstdDict;
-
-typedef struct {
- PyObject_HEAD
-
- /* Compression context */
- ZSTD_CCtx *cctx;
-
- /* ZstdDict object in use */
- PyObject *dict;
-
- /* Last mode, initialized to ZSTD_e_end */
- int last_mode;
-
- /* (nbWorker >= 1) ? 1 : 0 */
- int use_multithread;
-
- /* Compression level */
- int compression_level;
-
- /* __init__ has been called, 0 or 1. */
- int inited;
-} ZstdCompressor;
-
-typedef struct {
- PyObject_HEAD
-
- /* Decompression context */
- ZSTD_DCtx *dctx;
-
- /* ZstdDict object in use */
- PyObject *dict;
-
- /* Unconsumed input data */
- char *input_buffer;
- size_t input_buffer_size;
- size_t in_begin, in_end;
-
- /* Unused data */
- PyObject *unused_data;
-
- /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
- char needs_input;
-
- /* For decompress(), 0 or 1.
- 1 when both input and output streams are at a frame edge, means a
- frame is completely decoded and fully flushed, or the decompressor
- just be initialized. */
- char at_frame_edge;
-
- /* For ZstdDecompressor, 0 or 1.
- 1 means the end of the first frame has been reached. */
- char eof;
-
- /* Used for fast reset above three variables */
- char _unused_char_for_align;
-
- /* __init__ has been called, 0 or 1. */
- int inited;
-} ZstdDecompressor;
-
-typedef enum {
- TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class
- TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function
-} decompress_type;
+} _zstd_state;
typedef enum {
ERR_DECOMPRESS,
@@ -155,7 +37,7 @@ typedef enum {
ERR_SET_C_LEVEL,
ERR_TRAIN_DICT,
- ERR_FINALIZE_DICT
+ ERR_FINALIZE_DICT,
} error_type;
typedef enum {
@@ -164,41 +46,16 @@ typedef enum {
DICT_TYPE_PREFIX = 2
} dictionary_type;
-static inline int
-mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) {
- return in->size == in->pos && out->size != out->pos;
-}
+extern ZstdDict *
+_Py_parse_zstd_dict(const _zstd_state *state,
+ PyObject *dict, int *type);
/* Format error message and set ZstdError. */
extern void
-set_zstd_error(const _zstd_state* const state,
- const error_type type, size_t zstd_ret);
+set_zstd_error(const _zstd_state *state,
+ error_type type, size_t zstd_ret);
extern void
-set_parameter_error(const _zstd_state* const state, int is_compress,
- int key_v, int value_v);
-
-static const char init_twice_msg[] = "__init__ method is called twice.";
-
-extern int
-_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict);
-
-extern int
-_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict);
-
-extern int
-_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
- const char *arg_name, const char *arg_type);
-
-extern int
-_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options);
-
-extern PyObject *
-decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
- Py_ssize_t max_length,
- Py_ssize_t initial_size,
- decompress_type type);
+set_parameter_error(int is_compress, int key_v, int value_v);
-extern PyObject *
-compress_impl(ZstdCompressor *self, Py_buffer *data,
- ZSTD_EndDirective end_directive);
+#endif // !ZSTD_MODULE_H
diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h
index 319b1214833..4c885fa0d72 100644
--- a/Modules/_zstd/buffer.h
+++ b/Modules/_zstd/buffer.h
@@ -1,11 +1,12 @@
-/*
-Low level interface to Meta's zstd library for use in the compression.zstd
-Python module.
-*/
+/* Low level interface to the Zstandard algorthm & the zstd library. */
+
+#ifndef ZSTD_BUFFER_H
+#define ZSTD_BUFFER_H
-#include "_zstdmodule.h"
#include "pycore_blocks_output_buffer.h"
+#include <zstd.h> // ZSTD_outBuffer
+
/* Blocks output buffer wrapper code */
/* Initialize the buffer, and grow the buffer.
@@ -18,7 +19,8 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob,
/* Ensure .list was set to NULL */
assert(buffer->list == NULL);
- Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length, &ob->dst);
+ Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length,
+ &ob->dst);
if (res < 0) {
return -1;
}
@@ -33,8 +35,7 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob,
Return -1 on failure */
static inline int
_OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob,
- Py_ssize_t max_length,
- Py_ssize_t init_size)
+ Py_ssize_t max_length, Py_ssize_t init_size)
{
Py_ssize_t block_size;
@@ -49,7 +50,8 @@ _OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob,
block_size = init_size;
}
- Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size, &ob->dst);
+ Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size,
+ &ob->dst);
if (res < 0) {
return -1;
}
@@ -102,3 +104,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob)
return buffer->allocated == buffer->max_length;
}
+
+#endif // !ZSTD_BUFFER_H
diff --git a/Modules/_zstd/clinic/_zstdmodule.c.h b/Modules/_zstd/clinic/_zstdmodule.c.h
index 4b78bded67b..766e1cfa776 100644
--- a/Modules/_zstd/clinic/_zstdmodule.c.h
+++ b/Modules/_zstd/clinic/_zstdmodule.c.h
@@ -9,47 +9,47 @@ preserve
#include "pycore_abstract.h" // _PyNumber_Index()
#include "pycore_modsupport.h" // _PyArg_CheckPositional()
-PyDoc_STRVAR(_zstd__train_dict__doc__,
-"_train_dict($module, samples_bytes, samples_size_list, dict_size, /)\n"
+PyDoc_STRVAR(_zstd_train_dict__doc__,
+"train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n"
"--\n"
"\n"
-"Internal function, train a zstd dictionary on sample data.\n"
+"Train a Zstandard dictionary on sample data.\n"
"\n"
" samples_bytes\n"
" Concatenation of samples.\n"
-" samples_size_list\n"
-" List of samples\' sizes.\n"
+" samples_sizes\n"
+" Tuple of samples\' sizes.\n"
" dict_size\n"
" The size of the dictionary.");
-#define _ZSTD__TRAIN_DICT_METHODDEF \
- {"_train_dict", _PyCFunction_CAST(_zstd__train_dict), METH_FASTCALL, _zstd__train_dict__doc__},
+#define _ZSTD_TRAIN_DICT_METHODDEF \
+ {"train_dict", _PyCFunction_CAST(_zstd_train_dict), METH_FASTCALL, _zstd_train_dict__doc__},
static PyObject *
-_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
- PyObject *samples_size_list, Py_ssize_t dict_size);
+_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
+ PyObject *samples_sizes, Py_ssize_t dict_size);
static PyObject *
-_zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+_zstd_train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
PyBytesObject *samples_bytes;
- PyObject *samples_size_list;
+ PyObject *samples_sizes;
Py_ssize_t dict_size;
- if (!_PyArg_CheckPositional("_train_dict", nargs, 3, 3)) {
+ if (!_PyArg_CheckPositional("train_dict", nargs, 3, 3)) {
goto exit;
}
if (!PyBytes_Check(args[0])) {
- _PyArg_BadArgument("_train_dict", "argument 1", "bytes", args[0]);
+ _PyArg_BadArgument("train_dict", "argument 1", "bytes", args[0]);
goto exit;
}
samples_bytes = (PyBytesObject *)args[0];
- if (!PyList_Check(args[1])) {
- _PyArg_BadArgument("_train_dict", "argument 2", "list", args[1]);
+ if (!PyTuple_Check(args[1])) {
+ _PyArg_BadArgument("train_dict", "argument 2", "tuple", args[1]);
goto exit;
}
- samples_size_list = args[1];
+ samples_sizes = args[1];
{
Py_ssize_t ival = -1;
PyObject *iobj = _PyNumber_Index(args[2]);
@@ -62,67 +62,67 @@ _zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
}
dict_size = ival;
}
- return_value = _zstd__train_dict_impl(module, samples_bytes, samples_size_list, dict_size);
+ return_value = _zstd_train_dict_impl(module, samples_bytes, samples_sizes, dict_size);
exit:
return return_value;
}
-PyDoc_STRVAR(_zstd__finalize_dict__doc__,
-"_finalize_dict($module, custom_dict_bytes, samples_bytes,\n"
-" samples_size_list, dict_size, compression_level, /)\n"
+PyDoc_STRVAR(_zstd_finalize_dict__doc__,
+"finalize_dict($module, custom_dict_bytes, samples_bytes, samples_sizes,\n"
+" dict_size, compression_level, /)\n"
"--\n"
"\n"
-"Internal function, finalize a zstd dictionary.\n"
+"Finalize a Zstandard dictionary.\n"
"\n"
" custom_dict_bytes\n"
" Custom dictionary content.\n"
" samples_bytes\n"
" Concatenation of samples.\n"
-" samples_size_list\n"
-" List of samples\' sizes.\n"
+" samples_sizes\n"
+" Tuple of samples\' sizes.\n"
" dict_size\n"
" The size of the dictionary.\n"
" compression_level\n"
-" Optimize for a specific zstd compression level, 0 means default.");
+" Optimize for a specific Zstandard compression level, 0 means default.");
-#define _ZSTD__FINALIZE_DICT_METHODDEF \
- {"_finalize_dict", _PyCFunction_CAST(_zstd__finalize_dict), METH_FASTCALL, _zstd__finalize_dict__doc__},
+#define _ZSTD_FINALIZE_DICT_METHODDEF \
+ {"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__},
static PyObject *
-_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
- PyBytesObject *samples_bytes,
- PyObject *samples_size_list, Py_ssize_t dict_size,
- int compression_level);
+_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
+ PyBytesObject *samples_bytes,
+ PyObject *samples_sizes, Py_ssize_t dict_size,
+ int compression_level);
static PyObject *
-_zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+_zstd_finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
PyBytesObject *custom_dict_bytes;
PyBytesObject *samples_bytes;
- PyObject *samples_size_list;
+ PyObject *samples_sizes;
Py_ssize_t dict_size;
int compression_level;
- if (!_PyArg_CheckPositional("_finalize_dict", nargs, 5, 5)) {
+ if (!_PyArg_CheckPositional("finalize_dict", nargs, 5, 5)) {
goto exit;
}
if (!PyBytes_Check(args[0])) {
- _PyArg_BadArgument("_finalize_dict", "argument 1", "bytes", args[0]);
+ _PyArg_BadArgument("finalize_dict", "argument 1", "bytes", args[0]);
goto exit;
}
custom_dict_bytes = (PyBytesObject *)args[0];
if (!PyBytes_Check(args[1])) {
- _PyArg_BadArgument("_finalize_dict", "argument 2", "bytes", args[1]);
+ _PyArg_BadArgument("finalize_dict", "argument 2", "bytes", args[1]);
goto exit;
}
samples_bytes = (PyBytesObject *)args[1];
- if (!PyList_Check(args[2])) {
- _PyArg_BadArgument("_finalize_dict", "argument 3", "list", args[2]);
+ if (!PyTuple_Check(args[2])) {
+ _PyArg_BadArgument("finalize_dict", "argument 3", "tuple", args[2]);
goto exit;
}
- samples_size_list = args[2];
+ samples_sizes = args[2];
{
Py_ssize_t ival = -1;
PyObject *iobj = _PyNumber_Index(args[3]);
@@ -139,32 +139,31 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
if (compression_level == -1 && PyErr_Occurred()) {
goto exit;
}
- return_value = _zstd__finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_size_list, dict_size, compression_level);
+ return_value = _zstd_finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_sizes, dict_size, compression_level);
exit:
return return_value;
}
-PyDoc_STRVAR(_zstd__get_param_bounds__doc__,
-"_get_param_bounds($module, /, is_compress, parameter)\n"
+PyDoc_STRVAR(_zstd_get_param_bounds__doc__,
+"get_param_bounds($module, /, parameter, is_compress)\n"
"--\n"
"\n"
-"Internal function, get CParameter/DParameter bounds.\n"
+"Get CompressionParameter/DecompressionParameter bounds.\n"
"\n"
-" is_compress\n"
-" True for CParameter, False for DParameter.\n"
" parameter\n"
-" The parameter to get bounds.");
+" The parameter to get bounds.\n"
+" is_compress\n"
+" True for CompressionParameter, False for DecompressionParameter.");
-#define _ZSTD__GET_PARAM_BOUNDS_METHODDEF \
- {"_get_param_bounds", _PyCFunction_CAST(_zstd__get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd__get_param_bounds__doc__},
+#define _ZSTD_GET_PARAM_BOUNDS_METHODDEF \
+ {"get_param_bounds", _PyCFunction_CAST(_zstd_get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd_get_param_bounds__doc__},
static PyObject *
-_zstd__get_param_bounds_impl(PyObject *module, int is_compress,
- int parameter);
+_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress);
static PyObject *
-_zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+_zstd_get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
@@ -178,7 +177,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_hash = -1,
- .ob_item = { &_Py_ID(is_compress), &_Py_ID(parameter), },
+ .ob_item = { &_Py_ID(parameter), &_Py_ID(is_compress), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@@ -187,31 +186,31 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
- static const char * const _keywords[] = {"is_compress", "parameter", NULL};
+ static const char * const _keywords[] = {"parameter", "is_compress", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
- .fname = "_get_param_bounds",
+ .fname = "get_param_bounds",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
PyObject *argsbuf[2];
- int is_compress;
int parameter;
+ int is_compress;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser,
/*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
if (!args) {
goto exit;
}
- is_compress = PyObject_IsTrue(args[0]);
- if (is_compress < 0) {
+ parameter = PyLong_AsInt(args[0]);
+ if (parameter == -1 && PyErr_Occurred()) {
goto exit;
}
- parameter = PyLong_AsInt(args[1]);
- if (parameter == -1 && PyErr_Occurred()) {
+ is_compress = PyObject_IsTrue(args[1]);
+ if (is_compress < 0) {
goto exit;
}
- return_value = _zstd__get_param_bounds_impl(module, is_compress, parameter);
+ return_value = _zstd_get_param_bounds_impl(module, parameter, is_compress);
exit:
return return_value;
@@ -221,13 +220,11 @@ PyDoc_STRVAR(_zstd_get_frame_size__doc__,
"get_frame_size($module, /, frame_buffer)\n"
"--\n"
"\n"
-"Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.\n"
+"Get the size of a Zstandard frame, including the header and optional checksum.\n"
"\n"
" frame_buffer\n"
" A bytes-like object, it should start from the beginning of a frame,\n"
-" and contains at least one complete frame.\n"
-"\n"
-"It will iterate all blocks\' headers within a frame, to accumulate the frame size.");
+" and contains at least one complete frame.");
#define _ZSTD_GET_FRAME_SIZE_METHODDEF \
{"get_frame_size", _PyCFunction_CAST(_zstd_get_frame_size), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_size__doc__},
@@ -288,23 +285,23 @@ exit:
return return_value;
}
-PyDoc_STRVAR(_zstd__get_frame_info__doc__,
-"_get_frame_info($module, /, frame_buffer)\n"
+PyDoc_STRVAR(_zstd_get_frame_info__doc__,
+"get_frame_info($module, /, frame_buffer)\n"
"--\n"
"\n"
-"Internal function, get zstd frame infomation from a frame header.\n"
+"Get Zstandard frame infomation from a frame header.\n"
"\n"
" frame_buffer\n"
-" A bytes-like object, containing the header of a zstd frame.");
+" A bytes-like object, containing the header of a Zstandard frame.");
-#define _ZSTD__GET_FRAME_INFO_METHODDEF \
- {"_get_frame_info", _PyCFunction_CAST(_zstd__get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd__get_frame_info__doc__},
+#define _ZSTD_GET_FRAME_INFO_METHODDEF \
+ {"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__},
static PyObject *
-_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer);
+_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer);
static PyObject *
-_zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+_zstd_get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
@@ -330,7 +327,7 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
static const char * const _keywords[] = {"frame_buffer", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
- .fname = "_get_frame_info",
+ .fname = "get_frame_info",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
@@ -345,7 +342,7 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
if (PyObject_GetBuffer(args[0], &frame_buffer, PyBUF_SIMPLE) != 0) {
goto exit;
}
- return_value = _zstd__get_frame_info_impl(module, &frame_buffer);
+ return_value = _zstd_get_frame_info_impl(module, &frame_buffer);
exit:
/* Cleanup for frame_buffer */
@@ -356,26 +353,26 @@ exit:
return return_value;
}
-PyDoc_STRVAR(_zstd__set_parameter_types__doc__,
-"_set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n"
+PyDoc_STRVAR(_zstd_set_parameter_types__doc__,
+"set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n"
"--\n"
"\n"
-"Internal function, set CParameter/DParameter types for validity check.\n"
+"Set CompressionParameter and DecompressionParameter types for validity check.\n"
"\n"
" c_parameter_type\n"
-" CParameter IntEnum type object\n"
+" CompressionParameter IntEnum type object\n"
" d_parameter_type\n"
-" DParameter IntEnum type object");
+" DecompressionParameter IntEnum type object");
-#define _ZSTD__SET_PARAMETER_TYPES_METHODDEF \
- {"_set_parameter_types", _PyCFunction_CAST(_zstd__set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd__set_parameter_types__doc__},
+#define _ZSTD_SET_PARAMETER_TYPES_METHODDEF \
+ {"set_parameter_types", _PyCFunction_CAST(_zstd_set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd_set_parameter_types__doc__},
static PyObject *
-_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
- PyObject *d_parameter_type);
+_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
+ PyObject *d_parameter_type);
static PyObject *
-_zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+_zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
@@ -401,7 +398,7 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n
static const char * const _keywords[] = {"c_parameter_type", "d_parameter_type", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
- .fname = "_set_parameter_types",
+ .fname = "set_parameter_types",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
@@ -415,18 +412,18 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n
goto exit;
}
if (!PyObject_TypeCheck(args[0], &PyType_Type)) {
- _PyArg_BadArgument("_set_parameter_types", "argument 'c_parameter_type'", (&PyType_Type)->tp_name, args[0]);
+ _PyArg_BadArgument("set_parameter_types", "argument 'c_parameter_type'", (&PyType_Type)->tp_name, args[0]);
goto exit;
}
c_parameter_type = args[0];
if (!PyObject_TypeCheck(args[1], &PyType_Type)) {
- _PyArg_BadArgument("_set_parameter_types", "argument 'd_parameter_type'", (&PyType_Type)->tp_name, args[1]);
+ _PyArg_BadArgument("set_parameter_types", "argument 'd_parameter_type'", (&PyType_Type)->tp_name, args[1]);
goto exit;
}
d_parameter_type = args[1];
- return_value = _zstd__set_parameter_types_impl(module, c_parameter_type, d_parameter_type);
+ return_value = _zstd_set_parameter_types_impl(module, c_parameter_type, d_parameter_type);
exit:
return return_value;
}
-/*[clinic end generated code: output=077c8ea2b11fb188 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=437b084f149e68e5 input=a9049054013a1b77]*/
diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h
index d7909cdf89f..4f8d93fd9e8 100644
--- a/Modules/_zstd/clinic/compressor.c.h
+++ b/Modules/_zstd/clinic/compressor.c.h
@@ -8,30 +8,30 @@ preserve
#endif
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
-PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__,
+PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__,
"ZstdCompressor(level=None, options=None, zstd_dict=None)\n"
"--\n"
"\n"
"Create a compressor object for compressing data incrementally.\n"
"\n"
" level\n"
-" The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.\n"
+" The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n"
" options\n"
" A dict object that contains advanced compression parameters.\n"
" zstd_dict\n"
-" A ZstdDict object, a pre-trained zstd dictionary.\n"
+" A ZstdDict object, a pre-trained Zstandard dictionary.\n"
"\n"
"Thread-safe at method level. For one-shot compression, use the compress()\n"
"function instead.");
-static int
-_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level,
- PyObject *options, PyObject *zstd_dict);
+static PyObject *
+_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level,
+ PyObject *options, PyObject *zstd_dict);
-static int
-_zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
+static PyObject *
+_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{
- int return_value = -1;
+ PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 3
@@ -89,7 +89,7 @@ _zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
}
zstd_dict = fastargs[2];
skip_optional_pos:
- return_value = _zstd_ZstdCompressor___init___impl((ZstdCompressor *)self, level, options, zstd_dict);
+ return_value = _zstd_ZstdCompressor_new_impl(type, level, options, zstd_dict);
exit:
return return_value;
@@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__,
" Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n"
" ZstdCompressor.FLUSH_BLOCK\n"
"\n"
-"Flush any remaining data left in internal buffers. Since zstd data consists\n"
-"of one or more independent frames, the compressor object can still be used\n"
-"after this method is called.");
+"Flush any remaining data left in internal buffers. Since Zstandard data\n"
+"consists of one or more independent frames, the compressor object can still\n"
+"be used after this method is called.");
#define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \
{"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__},
@@ -252,4 +252,43 @@ skip_optional_pos:
exit:
return return_value;
}
-/*[clinic end generated code: output=ef69eab155be39f6 input=a9049054013a1b77]*/
+
+PyDoc_STRVAR(_zstd_ZstdCompressor_set_pledged_input_size__doc__,
+"set_pledged_input_size($self, size, /)\n"
+"--\n"
+"\n"
+"Set the uncompressed content size to be written into the frame header.\n"
+"\n"
+" size\n"
+" The size of the uncompressed data to be provided to the compressor.\n"
+"\n"
+"This method can be used to ensure the header of the frame about to be written\n"
+"includes the size of the data, unless the CompressionParameter.content_size_flag\n"
+"is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n"
+"\n"
+"It is important to ensure that the pledged data size matches the actual data\n"
+"size. If they do not match the compressed output data may be corrupted and the\n"
+"final chunk written may be lost.");
+
+#define _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF \
+ {"set_pledged_input_size", (PyCFunction)_zstd_ZstdCompressor_set_pledged_input_size, METH_O, _zstd_ZstdCompressor_set_pledged_input_size__doc__},
+
+static PyObject *
+_zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self,
+ unsigned long long size);
+
+static PyObject *
+_zstd_ZstdCompressor_set_pledged_input_size(PyObject *self, PyObject *arg)
+{
+ PyObject *return_value = NULL;
+ unsigned long long size;
+
+ if (!zstd_contentsize_converter(arg, &size)) {
+ goto exit;
+ }
+ return_value = _zstd_ZstdCompressor_set_pledged_input_size_impl((ZstdCompressor *)self, size);
+
+exit:
+ return return_value;
+}
+/*[clinic end generated code: output=c1d5c2cf06a8becd input=a9049054013a1b77]*/
diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h
index 9359c637203..c6fdae74ab0 100644
--- a/Modules/_zstd/clinic/decompressor.c.h
+++ b/Modules/_zstd/clinic/decompressor.c.h
@@ -7,31 +7,30 @@ preserve
# include "pycore_runtime.h" // _Py_ID()
#endif
#include "pycore_abstract.h" // _PyNumber_Index()
-#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
-PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__,
+PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__,
"ZstdDecompressor(zstd_dict=None, options=None)\n"
"--\n"
"\n"
"Create a decompressor object for decompressing data incrementally.\n"
"\n"
" zstd_dict\n"
-" A ZstdDict object, a pre-trained zstd dictionary.\n"
+" A ZstdDict object, a pre-trained Zstandard dictionary.\n"
" options\n"
" A dict object that contains advanced decompression parameters.\n"
"\n"
"Thread-safe at method level. For one-shot decompression, use the decompress()\n"
"function instead.");
-static int
-_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self,
- PyObject *zstd_dict, PyObject *options);
+static PyObject *
+_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
+ PyObject *options);
-static int
-_zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
+static PyObject *
+_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{
- int return_value = -1;
+ PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 2
@@ -82,7 +81,7 @@ _zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs
}
options = fastargs[1];
skip_optional_pos:
- return_value = _zstd_ZstdDecompressor___init___impl((ZstdDecompressor *)self, zstd_dict, options);
+ return_value = _zstd_ZstdDecompressor_new_impl(type, zstd_dict, options);
exit:
return return_value;
@@ -114,13 +113,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self);
static PyObject *
_zstd_ZstdDecompressor_unused_data_get(PyObject *self, void *Py_UNUSED(context))
{
- PyObject *return_value = NULL;
-
- Py_BEGIN_CRITICAL_SECTION(self);
- return_value = _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self);
- Py_END_CRITICAL_SECTION();
-
- return return_value;
+ return _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self);
}
PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__,
@@ -130,7 +123,7 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__,
"Decompress *data*, returning uncompressed bytes if possible, or b\'\' otherwise.\n"
"\n"
" data\n"
-" A bytes-like object, zstd data to be decompressed.\n"
+" A bytes-like object, Zstandard data to be decompressed.\n"
" max_length\n"
" Maximum size of returned data. When it is negative, the size of\n"
" output buffer is unlimited. When it is nonnegative, returns at\n"
@@ -227,4 +220,4 @@ exit:
return return_value;
}
-/*[clinic end generated code: output=ae703f0465a2906d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=30c12ef047027ede input=a9049054013a1b77]*/
diff --git a/Modules/_zstd/clinic/zdict.c.h b/Modules/_zstd/clinic/zstddict.c.h
index 4e0f7b64172..79db85405d6 100644
--- a/Modules/_zstd/clinic/zdict.c.h
+++ b/Modules/_zstd/clinic/zstddict.c.h
@@ -6,38 +6,35 @@ preserve
# include "pycore_gc.h" // PyGC_Head
# include "pycore_runtime.h" // _Py_ID()
#endif
-#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
-PyDoc_STRVAR(_zstd_ZstdDict___init____doc__,
-"ZstdDict(dict_content, is_raw=False)\n"
+PyDoc_STRVAR(_zstd_ZstdDict_new__doc__,
+"ZstdDict(dict_content, /, *, is_raw=False)\n"
"--\n"
"\n"
-"Represents a zstd dictionary, which can be used for compression/decompression.\n"
+"Represents a Zstandard dictionary.\n"
"\n"
" dict_content\n"
-" A bytes-like object, dictionary\'s content.\n"
+" The content of a Zstandard dictionary as a bytes-like object.\n"
" is_raw\n"
-" This parameter is for advanced user. True means dict_content\n"
-" argument is a \"raw content\" dictionary, free of any format\n"
-" restriction. False means dict_content argument is an ordinary\n"
-" zstd dictionary, was created by zstd functions, follow a\n"
-" specified format.\n"
+" If true, perform no checks on *dict_content*, useful for some\n"
+" advanced cases. Otherwise, check that the content represents\n"
+" a Zstandard dictionary created by the zstd library or CLI.\n"
"\n"
-"It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n"
-"ZstdDecompressor objects.");
+"The dictionary can be used for compression or decompression, and can be shared\n"
+"by multiple ZstdCompressor or ZstdDecompressor objects.");
-static int
-_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content,
- int is_raw);
+static PyObject *
+_zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content,
+ int is_raw);
-static int
-_zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
+static PyObject *
+_zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{
- int return_value = -1;
+ PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
- #define NUM_KEYWORDS 2
+ #define NUM_KEYWORDS 1
static struct {
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
@@ -46,7 +43,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_hash = -1,
- .ob_item = { &_Py_ID(dict_content), &_Py_ID(is_raw), },
+ .ob_item = { &_Py_ID(is_raw), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@@ -55,7 +52,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
- static const char * const _keywords[] = {"dict_content", "is_raw", NULL};
+ static const char * const _keywords[] = {"", "is_raw", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "ZstdDict",
@@ -66,33 +63,68 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
PyObject * const *fastargs;
Py_ssize_t nargs = PyTuple_GET_SIZE(args);
Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1;
- PyObject *dict_content;
+ Py_buffer dict_content = {NULL, NULL};
int is_raw = 0;
fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser,
- /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
+ /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
if (!fastargs) {
goto exit;
}
- dict_content = fastargs[0];
+ if (PyObject_GetBuffer(fastargs[0], &dict_content, PyBUF_SIMPLE) != 0) {
+ goto exit;
+ }
if (!noptargs) {
- goto skip_optional_pos;
+ goto skip_optional_kwonly;
}
is_raw = PyObject_IsTrue(fastargs[1]);
if (is_raw < 0) {
goto exit;
}
-skip_optional_pos:
- return_value = _zstd_ZstdDict___init___impl((ZstdDict *)self, dict_content, is_raw);
+skip_optional_kwonly:
+ return_value = _zstd_ZstdDict_new_impl(type, &dict_content, is_raw);
exit:
+ /* Cleanup for dict_content */
+ if (dict_content.obj) {
+ PyBuffer_Release(&dict_content);
+ }
+
return return_value;
}
+PyDoc_STRVAR(_zstd_ZstdDict_dict_content__doc__,
+"The content of a Zstandard dictionary, as a bytes object.");
+#if defined(_zstd_ZstdDict_dict_content_DOCSTR)
+# undef _zstd_ZstdDict_dict_content_DOCSTR
+#endif
+#define _zstd_ZstdDict_dict_content_DOCSTR _zstd_ZstdDict_dict_content__doc__
+
+#if !defined(_zstd_ZstdDict_dict_content_DOCSTR)
+# define _zstd_ZstdDict_dict_content_DOCSTR NULL
+#endif
+#if defined(_ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF)
+# undef _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF
+# define _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF {"dict_content", (getter)_zstd_ZstdDict_dict_content_get, (setter)_zstd_ZstdDict_dict_content_set, _zstd_ZstdDict_dict_content_DOCSTR},
+#else
+# define _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF {"dict_content", (getter)_zstd_ZstdDict_dict_content_get, NULL, _zstd_ZstdDict_dict_content_DOCSTR},
+#endif
+
+static PyObject *
+_zstd_ZstdDict_dict_content_get_impl(ZstdDict *self);
+
+static PyObject *
+_zstd_ZstdDict_dict_content_get(PyObject *self, void *Py_UNUSED(context))
+{
+ return _zstd_ZstdDict_dict_content_get_impl((ZstdDict *)self);
+}
+
PyDoc_STRVAR(_zstd_ZstdDict_as_digested_dict__doc__,
"Load as a digested dictionary to compressor.\n"
"\n"
-"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict)\n"
+"Pass this attribute as zstd_dict argument:\n"
+"compress(dat, zstd_dict=zd.as_digested_dict)\n"
+"\n"
"1. Some advanced compression parameters of compressor may be overridden\n"
" by parameters of digested dictionary.\n"
"2. ZstdDict has a digested dictionaries cache for each compression level.\n"
@@ -120,19 +152,15 @@ _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self);
static PyObject *
_zstd_ZstdDict_as_digested_dict_get(PyObject *self, void *Py_UNUSED(context))
{
- PyObject *return_value = NULL;
-
- Py_BEGIN_CRITICAL_SECTION(self);
- return_value = _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self);
- Py_END_CRITICAL_SECTION();
-
- return return_value;
+ return _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self);
}
PyDoc_STRVAR(_zstd_ZstdDict_as_undigested_dict__doc__,
"Load as an undigested dictionary to compressor.\n"
"\n"
-"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict)\n"
+"Pass this attribute as zstd_dict argument:\n"
+"compress(dat, zstd_dict=zd.as_undigested_dict)\n"
+"\n"
"1. The advanced compression parameters of compressor will not be overridden.\n"
"2. Loading an undigested dictionary is costly. If load an undigested dictionary\n"
" multiple times, consider reusing a compressor object.\n"
@@ -158,19 +186,15 @@ _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self);
static PyObject *
_zstd_ZstdDict_as_undigested_dict_get(PyObject *self, void *Py_UNUSED(context))
{
- PyObject *return_value = NULL;
-
- Py_BEGIN_CRITICAL_SECTION(self);
- return_value = _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self);
- Py_END_CRITICAL_SECTION();
-
- return return_value;
+ return _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self);
}
PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__,
"Load as a prefix to compressor/decompressor.\n"
"\n"
-"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix)\n"
+"Pass this attribute as zstd_dict argument:\n"
+"compress(dat, zstd_dict=zd.as_prefix)\n"
+"\n"
"1. Prefix is compatible with long distance matching, while dictionary is not.\n"
"2. It only works for the first frame, then the compressor/decompressor will\n"
" return to no prefix state.\n"
@@ -196,12 +220,6 @@ _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self);
static PyObject *
_zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context))
{
- PyObject *return_value = NULL;
-
- Py_BEGIN_CRITICAL_SECTION(self);
- return_value = _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self);
- Py_END_CRITICAL_SECTION();
-
- return return_value;
+ return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self);
}
-/*[clinic end generated code: output=59257c053f74eda7 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=4696cbc722e5fdfc input=a9049054013a1b77]*/
diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c
index d0f677be821..bc9e6eff89a 100644
--- a/Modules/_zstd/compressor.c
+++ b/Modules/_zstd/compressor.c
@@ -1,123 +1,196 @@
-/*
-Low level interface to Meta's zstd library for use in the compression.zstd
-Python module.
-*/
+/* Low level interface to the Zstandard algorthm & the zstd library. */
/* ZstdCompressor class definitions */
/*[clinic input]
module _zstd
-class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type"
+class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec"
[clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/
-
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif
-#include "_zstdmodule.h"
+#include "Python.h"
+#include "_zstdmodule.h"
#include "buffer.h"
+#include "internal/pycore_lock.h" // PyMutex_IsLocked
#include <stddef.h> // offsetof()
+#include <zstd.h> // ZSTD_*()
+
+typedef struct {
+ PyObject_HEAD
+
+ /* Compression context */
+ ZSTD_CCtx *cctx;
+
+ /* ZstdDict object in use */
+ PyObject *dict;
+
+ /* Last mode, initialized to ZSTD_e_end */
+ int last_mode;
+ /* (nbWorker >= 1) ? 1 : 0 */
+ int use_multithread;
+
+ /* Compression level */
+ int compression_level;
+
+ /* Lock to protect the compression context */
+ PyMutex lock;
+} ZstdCompressor;
#define ZstdCompressor_CAST(op) ((ZstdCompressor *)op)
-int
-_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
- const char *arg_name, const char* arg_type)
+/*[python input]
+
+class zstd_contentsize_converter(CConverter):
+ type = 'unsigned long long'
+ converter = 'zstd_contentsize_converter'
+
+[python start generated code]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=0932c350d633c7de]*/
+
+
+static int
+zstd_contentsize_converter(PyObject *size, unsigned long long *p)
{
- size_t zstd_ret;
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
- if (mod_state == NULL) {
- return -1;
+ // None means the user indicates the size is unknown.
+ if (size == Py_None) {
+ *p = ZSTD_CONTENTSIZE_UNKNOWN;
}
-
- /* Integer compression level */
- if (PyLong_Check(level_or_options)) {
- int level = PyLong_AsInt(level_or_options);
- if (level == -1 && PyErr_Occurred()) {
+ else {
+ /* ZSTD_CONTENTSIZE_UNKNOWN is 0ULL - 1
+ ZSTD_CONTENTSIZE_ERROR is 0ULL - 2
+ Users should only pass values < ZSTD_CONTENTSIZE_ERROR */
+ unsigned long long pledged_size = PyLong_AsUnsignedLongLong(size);
+ /* Here we check for (unsigned long long)-1 as a sign of an error in
+ PyLong_AsUnsignedLongLong */
+ if (pledged_size == (unsigned long long)-1 && PyErr_Occurred()) {
+ *p = ZSTD_CONTENTSIZE_ERROR;
+ if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
+ PyErr_Format(PyExc_ValueError,
+ "size argument should be a positive int less "
+ "than %ull", ZSTD_CONTENTSIZE_ERROR);
+ return 0;
+ }
+ return 0;
+ }
+ if (pledged_size >= ZSTD_CONTENTSIZE_ERROR) {
+ *p = ZSTD_CONTENTSIZE_ERROR;
PyErr_Format(PyExc_ValueError,
- "Compression level should be an int value between %d and %d.",
- ZSTD_minCLevel(), ZSTD_maxCLevel());
- return -1;
+ "size argument should be a positive int less "
+ "than %ull", ZSTD_CONTENTSIZE_ERROR);
+ return 0;
}
+ *p = pledged_size;
+ }
+ return 1;
+}
- /* Save for generating ZSTD_CDICT */
- self->compression_level = level;
+#include "clinic/compressor.c.h"
- /* Set compressionLevel to compression context */
- zstd_ret = ZSTD_CCtx_setParameter(self->cctx,
- ZSTD_c_compressionLevel,
- level);
+static int
+_zstd_set_c_level(ZstdCompressor *self, int level)
+{
+ /* Set integer compression level */
+ int min_level = ZSTD_minCLevel();
+ int max_level = ZSTD_maxCLevel();
+ if (level < min_level || level > max_level) {
+ PyErr_Format(PyExc_ValueError,
+ "illegal compression level %d; the valid range is [%d, %d]",
+ level, min_level, max_level);
+ return -1;
+ }
- /* Check error */
- if (ZSTD_isError(zstd_ret)) {
- set_zstd_error(mod_state, ERR_SET_C_LEVEL, zstd_ret);
+ /* Save for generating ZSTD_CDICT */
+ self->compression_level = level;
+
+ /* Set compressionLevel to compression context */
+ size_t zstd_ret = ZSTD_CCtx_setParameter(
+ self->cctx, ZSTD_c_compressionLevel, level);
+
+ /* Check error */
+ if (ZSTD_isError(zstd_ret)) {
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
+ set_zstd_error(mod_state, ERR_SET_C_LEVEL, zstd_ret);
+ return -1;
+ }
+ return 0;
+}
+
+static int
+_zstd_set_c_parameters(ZstdCompressor *self, PyObject *options)
+{
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
+ if (mod_state == NULL) {
+ return -1;
+ }
+
+ if (!PyDict_Check(options)) {
+ PyErr_Format(PyExc_TypeError,
+ "ZstdCompressor() argument 'options' must be dict, not %T",
+ options);
+ return -1;
+ }
+
+ Py_ssize_t pos = 0;
+ PyObject *key, *value;
+ while (PyDict_Next(options, &pos, &key, &value)) {
+ /* Check key type */
+ if (Py_TYPE(key) == mod_state->DParameter_type) {
+ PyErr_SetString(PyExc_TypeError,
+ "compression options dictionary key must not be a "
+ "DecompressionParameter attribute");
return -1;
}
- return 0;
- }
- /* Options dict */
- if (PyDict_Check(level_or_options)) {
- PyObject *key, *value;
- Py_ssize_t pos = 0;
+ Py_INCREF(key);
+ Py_INCREF(value);
+ int key_v = PyLong_AsInt(key);
+ Py_DECREF(key);
+ if (key_v == -1 && PyErr_Occurred()) {
+ Py_DECREF(value);
+ return -1;
+ }
- while (PyDict_Next(level_or_options, &pos, &key, &value)) {
- /* Check key type */
- if (Py_TYPE(key) == mod_state->DParameter_type) {
- PyErr_SetString(PyExc_TypeError,
- "Key of compression option dict should "
- "NOT be DParameter.");
- return -1;
- }
+ int value_v = PyLong_AsInt(value);
+ Py_DECREF(value);
+ if (value_v == -1 && PyErr_Occurred()) {
+ return -1;
+ }
- int key_v = PyLong_AsInt(key);
- if (key_v == -1 && PyErr_Occurred()) {
- PyErr_SetString(PyExc_ValueError,
- "Key of options dict should be a CParameter attribute.");
+ if (key_v == ZSTD_c_compressionLevel) {
+ if (_zstd_set_c_level(self, value_v) < 0) {
return -1;
}
-
- // TODO(emmatyping): check bounds when there is a value error here for better
- // error message?
- int value_v = PyLong_AsInt(value);
- if (value_v == -1 && PyErr_Occurred()) {
- PyErr_SetString(PyExc_ValueError,
- "Value of option dict should be an int.");
- return -1;
+ continue;
+ }
+ if (key_v == ZSTD_c_nbWorkers) {
+ /* From the zstd library docs:
+ 1. When nbWorkers >= 1, triggers asynchronous mode when
+ used with ZSTD_compressStream2().
+ 2, Default value is `0`, aka "single-threaded mode" : no
+ worker is spawned, compression is performed inside
+ caller's thread, all invocations are blocking. */
+ if (value_v != 0) {
+ self->use_multithread = 1;
}
+ }
- if (key_v == ZSTD_c_compressionLevel) {
- /* Save for generating ZSTD_CDICT */
- self->compression_level = value_v;
- }
- else if (key_v == ZSTD_c_nbWorkers) {
- /* From zstd library doc:
- 1. When nbWorkers >= 1, triggers asynchronous mode when
- used with ZSTD_compressStream2().
- 2, Default value is `0`, aka "single-threaded mode" : no
- worker is spawned, compression is performed inside
- caller's thread, all invocations are blocking. */
- if (value_v != 0) {
- self->use_multithread = 1;
- }
- }
+ /* Set parameter to compression context */
+ size_t zstd_ret = ZSTD_CCtx_setParameter(self->cctx, key_v, value_v);
- /* Set parameter to compression context */
- zstd_ret = ZSTD_CCtx_setParameter(self->cctx, key_v, value_v);
- if (ZSTD_isError(zstd_ret)) {
- set_parameter_error(mod_state, 1, key_v, value_v);
- return -1;
- }
+ /* Check error */
+ if (ZSTD_isError(zstd_ret)) {
+ set_parameter_error(1, key_v, value_v);
+ return -1;
}
- return 0;
}
- PyErr_Format(PyExc_TypeError, "Invalid type for %s. Expected %s", arg_name, arg_type);
- return -1;
+ return 0;
}
static void
@@ -130,12 +203,12 @@ capsule_free_cdict(PyObject *capsule)
ZSTD_CDict *
_get_CDict(ZstdDict *self, int compressionLevel)
{
+ assert(PyMutex_IsLocked(&self->lock));
PyObject *level = NULL;
- PyObject *capsule;
+ PyObject *capsule = NULL;
ZSTD_CDict *cdict;
+ int ret;
- // TODO(emmatyping): refactor critical section code into a lock_held function
- Py_BEGIN_CRITICAL_SECTION(self);
/* int level object */
level = PyLong_FromLong(compressionLevel);
@@ -144,27 +217,23 @@ _get_CDict(ZstdDict *self, int compressionLevel)
}
/* Get PyCapsule object from self->c_dicts */
- capsule = PyDict_GetItemWithError(self->c_dicts, level);
+ ret = PyDict_GetItemRef(self->c_dicts, level, &capsule);
+ if (ret < 0) {
+ goto error;
+ }
if (capsule == NULL) {
- if (PyErr_Occurred()) {
- goto error;
- }
-
/* Create ZSTD_CDict instance */
- char *dict_buffer = PyBytes_AS_STRING(self->dict_content);
- Py_ssize_t dict_len = Py_SIZE(self->dict_content);
Py_BEGIN_ALLOW_THREADS
- cdict = ZSTD_createCDict(dict_buffer,
- dict_len,
+ cdict = ZSTD_createCDict(self->dict_buffer, self->dict_len,
compressionLevel);
Py_END_ALLOW_THREADS
if (cdict == NULL) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError,
- "Failed to create ZSTD_CDict instance from zstd "
- "dictionary content. Maybe the content is corrupted.");
+ "Failed to create a ZSTD_CDict instance from "
+ "Zstandard dictionary content.");
}
goto error;
}
@@ -177,11 +246,10 @@ _get_CDict(ZstdDict *self, int compressionLevel)
}
/* Add PyCapsule object to self->c_dicts */
- if (PyDict_SetItem(self->c_dicts, level, capsule) < 0) {
- Py_DECREF(capsule);
+ ret = PyDict_SetItem(self->c_dicts, level, capsule);
+ if (ret < 0) {
goto error;
}
- Py_DECREF(capsule);
}
else {
/* ZSTD_CDict instance already exists */
@@ -193,61 +261,15 @@ error:
cdict = NULL;
success:
Py_XDECREF(level);
- Py_END_CRITICAL_SECTION();
+ Py_XDECREF(capsule);
return cdict;
}
-int
-_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) {
-
+static int
+_zstd_load_impl(ZstdCompressor *self, ZstdDict *zd,
+ _zstd_state *mod_state, int type)
+{
size_t zstd_ret;
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
- if (mod_state == NULL) {
- return -1;
- }
- ZstdDict *zd;
- int type, ret;
-
- /* Check ZstdDict */
- ret = PyObject_IsInstance(dict, (PyObject*)mod_state->ZstdDict_type);
- if (ret < 0) {
- return -1;
- }
- else if (ret > 0) {
- /* When compressing, use undigested dictionary by default. */
- zd = (ZstdDict*)dict;
- type = DICT_TYPE_UNDIGESTED;
- goto load;
- }
-
- /* Check (ZstdDict, type) */
- if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2) {
- /* Check ZstdDict */
- ret = PyObject_IsInstance(PyTuple_GET_ITEM(dict, 0),
- (PyObject*)mod_state->ZstdDict_type);
- if (ret < 0) {
- return -1;
- }
- else if (ret > 0) {
- /* type == -1 may indicate an error. */
- type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1));
- if (type == DICT_TYPE_DIGESTED ||
- type == DICT_TYPE_UNDIGESTED ||
- type == DICT_TYPE_PREFIX)
- {
- assert(type >= 0);
- zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0);
- goto load;
- }
- }
- }
-
- /* Wrong type */
- PyErr_SetString(PyExc_TypeError,
- "zstd_dict argument should be ZstdDict object.");
- return -1;
-
-load:
if (type == DICT_TYPE_DIGESTED) {
/* Get ZSTD_CDict */
ZSTD_CDict *c_dict = _get_CDict(zd, self->compression_level);
@@ -256,28 +278,18 @@ load:
}
/* Reference a prepared dictionary.
It overrides some compression context's parameters. */
- Py_BEGIN_CRITICAL_SECTION(self);
zstd_ret = ZSTD_CCtx_refCDict(self->cctx, c_dict);
- Py_END_CRITICAL_SECTION();
}
else if (type == DICT_TYPE_UNDIGESTED) {
/* Load a dictionary.
It doesn't override compression context's parameters. */
- Py_BEGIN_CRITICAL_SECTION2(self, zd);
- zstd_ret = ZSTD_CCtx_loadDictionary(
- self->cctx,
- PyBytes_AS_STRING(zd->dict_content),
- Py_SIZE(zd->dict_content));
- Py_END_CRITICAL_SECTION2();
+ zstd_ret = ZSTD_CCtx_loadDictionary(self->cctx, zd->dict_buffer,
+ zd->dict_len);
}
else if (type == DICT_TYPE_PREFIX) {
/* Load a prefix */
- Py_BEGIN_CRITICAL_SECTION2(self, zd);
- zstd_ret = ZSTD_CCtx_refPrefix(
- self->cctx,
- PyBytes_AS_STRING(zd->dict_content),
- Py_SIZE(zd->dict_content));
- Py_END_CRITICAL_SECTION2();
+ zstd_ret = ZSTD_CCtx_refPrefix(self->cctx, zd->dict_buffer,
+ zd->dict_len);
}
else {
Py_UNREACHABLE();
@@ -291,28 +303,57 @@ load:
return 0;
}
-#define clinic_state() (get_zstd_state_from_type(type))
-#include "clinic/compressor.c.h"
-#undef clinic_state
+static int
+_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict)
+{
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
+ /* When compressing, use undigested dictionary by default. */
+ int type = DICT_TYPE_UNDIGESTED;
+ ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type);
+ if (zd == NULL) {
+ return -1;
+ }
+ int ret;
+ PyMutex_Lock(&zd->lock);
+ ret = _zstd_load_impl(self, zd, mod_state, type);
+ PyMutex_Unlock(&zd->lock);
+ return ret;
+}
+
+/*[clinic input]
+@classmethod
+_zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new
+ level: object = None
+ The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.
+ options: object = None
+ A dict object that contains advanced compression parameters.
+ zstd_dict: object = None
+ A ZstdDict object, a pre-trained Zstandard dictionary.
+
+Create a compressor object for compressing data incrementally.
+
+Thread-safe at method level. For one-shot compression, use the compress()
+function instead.
+[clinic start generated code]*/
static PyObject *
-_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs))
+_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level,
+ PyObject *options, PyObject *zstd_dict)
+/*[clinic end generated code: output=cdef61eafecac3d7 input=92de0211ae20ffdc]*/
{
- ZstdCompressor *self;
- self = PyObject_GC_New(ZstdCompressor, type);
+ ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type);
if (self == NULL) {
goto error;
}
- self->inited = 0;
- self->dict = NULL;
self->use_multithread = 0;
-
+ self->dict = NULL;
+ self->lock = (PyMutex){0};
/* Compression context */
self->cctx = ZSTD_createCCtx();
if (self->cctx == NULL) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError,
"Unable to create ZSTD_CCtx instance.");
@@ -323,12 +364,56 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject
/* Last mode */
self->last_mode = ZSTD_e_end;
+ if (level != Py_None && options != Py_None) {
+ PyErr_SetString(PyExc_TypeError,
+ "Only one of level or options should be used.");
+ goto error;
+ }
+
+ /* Set compression level */
+ if (level != Py_None) {
+ if (!PyLong_Check(level)) {
+ PyErr_SetString(PyExc_TypeError,
+ "invalid type for level, expected int");
+ goto error;
+ }
+ int level_v = PyLong_AsInt(level);
+ if (level_v == -1 && PyErr_Occurred()) {
+ if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
+ PyErr_Format(PyExc_ValueError,
+ "illegal compression level; the valid range is [%d, %d]",
+ ZSTD_minCLevel(), ZSTD_maxCLevel());
+ }
+ goto error;
+ }
+ if (_zstd_set_c_level(self, level_v) < 0) {
+ goto error;
+ }
+ }
+
+ /* Set options dictionary */
+ if (options != Py_None) {
+ if (_zstd_set_c_parameters(self, options) < 0) {
+ goto error;
+ }
+ }
+
+ /* Load Zstandard dictionary to compression context */
+ if (zstd_dict != Py_None) {
+ if (_zstd_load_c_dict(self, zstd_dict) < 0) {
+ goto error;
+ }
+ Py_INCREF(zstd_dict);
+ self->dict = zstd_dict;
+ }
+
+ // We can only start GC tracking once self->dict is set.
+ PyObject_GC_Track(self);
+
return (PyObject*)self;
error:
- if (self != NULL) {
- PyObject_GC_Del(self);
- }
+ Py_XDECREF(self);
return NULL;
}
@@ -340,7 +425,11 @@ ZstdCompressor_dealloc(PyObject *ob)
PyObject_GC_UnTrack(self);
/* Free compression context */
- ZSTD_freeCCtx(self->cctx);
+ if (self->cctx) {
+ ZSTD_freeCCtx(self->cctx);
+ }
+
+ assert(!PyMutex_IsLocked(&self->lock));
/* Py_XDECREF the dict after free the compression context */
Py_CLEAR(self->dict);
@@ -350,72 +439,11 @@ ZstdCompressor_dealloc(PyObject *ob)
Py_DECREF(tp);
}
-/*[clinic input]
-_zstd.ZstdCompressor.__init__
-
- level: object = None
- The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.
- options: object = None
- A dict object that contains advanced compression parameters.
- zstd_dict: object = None
- A ZstdDict object, a pre-trained zstd dictionary.
-
-Create a compressor object for compressing data incrementally.
-
-Thread-safe at method level. For one-shot compression, use the compress()
-function instead.
-[clinic start generated code]*/
-
-static int
-_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level,
- PyObject *options, PyObject *zstd_dict)
-/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/
-{
- /* Only called once */
- if (self->inited) {
- PyErr_SetString(PyExc_RuntimeError, init_twice_msg);
- return -1;
- }
- self->inited = 1;
-
- if (level != Py_None && options != Py_None) {
- PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
- return -1;
- }
-
- /* Set compressLevel/options to compression context */
- if (level != Py_None) {
- if (_PyZstd_set_c_parameters(self, level, "level", "int") < 0) {
- return -1;
- }
- }
-
- if (options != Py_None) {
- if (_PyZstd_set_c_parameters(self, options, "options", "dict") < 0) {
- return -1;
- }
- }
-
- /* Load dictionary to compression context */
- if (zstd_dict != Py_None) {
- if (_PyZstd_load_c_dict(self, zstd_dict) < 0) {
- return -1;
- }
-
- /* Py_INCREF the dict */
- Py_INCREF(zstd_dict);
- self->dict = zstd_dict;
- }
-
- // We can only start tracking self with the GC once self->dict is set.
- PyObject_GC_Track(self);
- return 0;
-}
-
-PyObject *
-compress_impl(ZstdCompressor *self, Py_buffer *data,
- ZSTD_EndDirective end_directive)
+static PyObject *
+compress_lock_held(ZstdCompressor *self, Py_buffer *data,
+ ZSTD_EndDirective end_directive)
{
+ assert(PyMutex_IsLocked(&self->lock));
ZSTD_inBuffer in;
ZSTD_outBuffer out;
_BlocksOutputBuffer buffer = {.list = NULL};
@@ -442,12 +470,12 @@ compress_impl(ZstdCompressor *self, Py_buffer *data,
}
if (_OutputBuffer_InitWithSize(&buffer, &out, -1,
- (Py_ssize_t) output_buffer_size) < 0) {
+ (Py_ssize_t) output_buffer_size) < 0) {
goto error;
}
- /* zstd stream compress */
+ /* Zstandard stream compress */
while (1) {
Py_BEGIN_ALLOW_THREADS
zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, end_directive);
@@ -455,10 +483,8 @@ compress_impl(ZstdCompressor *self, Py_buffer *data,
/* Check error */
if (ZSTD_isError(zstd_ret)) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
- if (mod_state != NULL) {
- set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret);
- }
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
+ set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret);
goto error;
}
@@ -487,9 +513,18 @@ error:
return NULL;
}
+#ifndef NDEBUG
+static inline int
+mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
+{
+ return in->size == in->pos && out->size != out->pos;
+}
+#endif
+
static PyObject *
-compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
+compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data)
{
+ assert(PyMutex_IsLocked(&self->lock));
ZSTD_inBuffer in;
ZSTD_outBuffer out;
_BlocksOutputBuffer buffer = {.list = NULL};
@@ -505,24 +540,25 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
goto error;
}
- /* zstd stream compress */
+ /* Zstandard stream compress */
while (1) {
Py_BEGIN_ALLOW_THREADS
do {
- zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, ZSTD_e_continue);
- } while (out.pos != out.size && in.pos != in.size && !ZSTD_isError(zstd_ret));
+ zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in,
+ ZSTD_e_continue);
+ } while (out.pos != out.size
+ && in.pos != in.size
+ && !ZSTD_isError(zstd_ret));
Py_END_ALLOW_THREADS
/* Check error */
if (ZSTD_isError(zstd_ret)) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
- if (mod_state != NULL) {
- set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret);
- }
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
+ set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret);
goto error;
}
- /* Like compress_impl(), output as much as possible. */
+ /* Like compress_lock_held(), output as much as possible. */
if (out.pos == out.size) {
if (_OutputBuffer_Grow(&buffer, &out) < 0) {
goto error;
@@ -581,14 +617,14 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data,
}
/* Thread-safe code */
- Py_BEGIN_CRITICAL_SECTION(self);
+ PyMutex_Lock(&self->lock);
/* Compress */
if (self->use_multithread && mode == ZSTD_e_continue) {
- ret = compress_mt_continue_impl(self, data);
+ ret = compress_mt_continue_lock_held(self, data);
}
else {
- ret = compress_impl(self, data, mode);
+ ret = compress_lock_held(self, data, mode);
}
if (ret) {
@@ -600,7 +636,7 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data,
/* Resetting cctx's session never fail */
ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
}
- Py_END_CRITICAL_SECTION();
+ PyMutex_Unlock(&self->lock);
return ret;
}
@@ -614,14 +650,14 @@ _zstd.ZstdCompressor.flush
Finish the compression process.
-Flush any remaining data left in internal buffers. Since zstd data consists
-of one or more independent frames, the compressor object can still be used
-after this method is called.
+Flush any remaining data left in internal buffers. Since Zstandard data
+consists of one or more independent frames, the compressor object can still
+be used after this method is called.
[clinic start generated code]*/
static PyObject *
_zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode)
-/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a766870301932b85]*/
+/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=0ab19627f323cdbc]*/
{
PyObject *ret;
@@ -635,8 +671,9 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode)
}
/* Thread-safe code */
- Py_BEGIN_CRITICAL_SECTION(self);
- ret = compress_impl(self, NULL, mode);
+ PyMutex_Lock(&self->lock);
+
+ ret = compress_lock_held(self, NULL, mode);
if (ret) {
self->last_mode = mode;
@@ -647,28 +684,79 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode)
/* Resetting cctx's session never fail */
ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
}
- Py_END_CRITICAL_SECTION();
+ PyMutex_Unlock(&self->lock);
return ret;
}
+
+/*[clinic input]
+_zstd.ZstdCompressor.set_pledged_input_size
+
+ size: zstd_contentsize
+ The size of the uncompressed data to be provided to the compressor.
+ /
+
+Set the uncompressed content size to be written into the frame header.
+
+This method can be used to ensure the header of the frame about to be written
+includes the size of the data, unless the CompressionParameter.content_size_flag
+is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised.
+
+It is important to ensure that the pledged data size matches the actual data
+size. If they do not match the compressed output data may be corrupted and the
+final chunk written may be lost.
+[clinic start generated code]*/
+
+static PyObject *
+_zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self,
+ unsigned long long size)
+/*[clinic end generated code: output=3a09e55cc0e3b4f9 input=afd8a7d78cff2eb5]*/
+{
+ // Error occured while converting argument, should be unreachable
+ assert(size != ZSTD_CONTENTSIZE_ERROR);
+
+ /* Thread-safe code */
+ PyMutex_Lock(&self->lock);
+
+ /* Check the current mode */
+ if (self->last_mode != ZSTD_e_end) {
+ PyErr_SetString(PyExc_ValueError,
+ "set_pledged_input_size() method must be called "
+ "when last_mode == FLUSH_FRAME");
+ PyMutex_Unlock(&self->lock);
+ return NULL;
+ }
+
+ /* Set pledged content size */
+ size_t zstd_ret = ZSTD_CCtx_setPledgedSrcSize(self->cctx, size);
+ PyMutex_Unlock(&self->lock);
+ if (ZSTD_isError(zstd_ret)) {
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
+ set_zstd_error(mod_state, ERR_SET_PLEDGED_INPUT_SIZE, zstd_ret);
+ return NULL;
+ }
+
+ Py_RETURN_NONE;
+}
+
static PyMethodDef ZstdCompressor_methods[] = {
_ZSTD_ZSTDCOMPRESSOR_COMPRESS_METHODDEF
_ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF
-
- {0}
+ _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF
+ {NULL, NULL}
};
PyDoc_STRVAR(ZstdCompressor_last_mode_doc,
"The last mode used to this compressor object, its value can be .CONTINUE,\n"
".FLUSH_BLOCK, .FLUSH_FRAME. Initialized to .FLUSH_FRAME.\n\n"
-"It can be used to get the current state of a compressor, such as, data flushed,\n"
-"a frame ended.");
+"It can be used to get the current state of a compressor, such as, data\n"
+"flushed, or a frame ended.");
static PyMemberDef ZstdCompressor_members[] = {
{"last_mode", Py_T_INT, offsetof(ZstdCompressor, last_mode),
- Py_READONLY, ZstdCompressor_last_mode_doc},
- {0}
+ Py_READONLY, ZstdCompressor_last_mode_doc},
+ {NULL}
};
static int
@@ -690,18 +778,20 @@ ZstdCompressor_clear(PyObject *ob)
static PyType_Slot zstdcompressor_slots[] = {
{Py_tp_new, _zstd_ZstdCompressor_new},
{Py_tp_dealloc, ZstdCompressor_dealloc},
- {Py_tp_init, _zstd_ZstdCompressor___init__},
{Py_tp_methods, ZstdCompressor_methods},
{Py_tp_members, ZstdCompressor_members},
- {Py_tp_doc, (char*)_zstd_ZstdCompressor___init____doc__},
+ {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__},
{Py_tp_traverse, ZstdCompressor_traverse},
{Py_tp_clear, ZstdCompressor_clear},
- {0}
+ {0, 0}
};
-PyType_Spec zstdcompressor_type_spec = {
- .name = "_zstd.ZstdCompressor",
+PyType_Spec zstd_compressor_type_spec = {
+ .name = "compression.zstd.ZstdCompressor",
.basicsize = sizeof(ZstdCompressor),
- .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+ // Py_TPFLAGS_IMMUTABLETYPE is not used here as several
+ // associated constants need to be added to the type.
+ // PyType_Freeze is called later to set the flag.
+ .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
.slots = zstdcompressor_slots,
};
diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c
index 4e3a28068be..c53d6e4cb05 100644
--- a/Modules/_zstd/decompressor.c
+++ b/Modules/_zstd/decompressor.c
@@ -1,176 +1,142 @@
-/*
-Low level interface to Meta's zstd library for use in the compression.zstd
-Python module.
-*/
+/* Low level interface to the Zstandard algorthm & the zstd library. */
/* ZstdDecompressor class definition */
/*[clinic input]
module _zstd
-class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompressor_type"
+class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec"
[clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4e6eae327c0c0c76]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/
#ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1
#endif
-#include "_zstdmodule.h"
+#include "Python.h"
+#include "_zstdmodule.h"
#include "buffer.h"
+#include "internal/pycore_lock.h" // PyMutex_IsLocked
+#include <stdbool.h> // bool
#include <stddef.h> // offsetof()
+#include <zstd.h> // ZSTD_*()
+
+typedef struct {
+ PyObject_HEAD
+
+ /* Decompression context */
+ ZSTD_DCtx *dctx;
+
+ /* ZstdDict object in use */
+ PyObject *dict;
+
+ /* Unconsumed input data */
+ char *input_buffer;
+ size_t input_buffer_size;
+ size_t in_begin, in_end;
+
+ /* Unused data */
+ PyObject *unused_data;
+
+ /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
+ bool needs_input;
+
+ /* For ZstdDecompressor, 0 or 1.
+ 1 means the end of the first frame has been reached. */
+ bool eof;
+
+ /* Lock to protect the decompression context */
+ PyMutex lock;
+} ZstdDecompressor;
#define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op)
+#include "clinic/decompressor.c.h"
+
static inline ZSTD_DDict *
_get_DDict(ZstdDict *self)
{
+ assert(PyMutex_IsLocked(&self->lock));
ZSTD_DDict *ret;
- /* Already created */
- if (self->d_dict != NULL) {
- return self->d_dict;
- }
-
- Py_BEGIN_CRITICAL_SECTION(self);
if (self->d_dict == NULL) {
/* Create ZSTD_DDict instance from dictionary content */
- char *dict_buffer = PyBytes_AS_STRING(self->dict_content);
- Py_ssize_t dict_len = Py_SIZE(self->dict_content);
Py_BEGIN_ALLOW_THREADS
- self->d_dict = ZSTD_createDDict(dict_buffer,
- dict_len);
+ ret = ZSTD_createDDict(self->dict_buffer, self->dict_len);
Py_END_ALLOW_THREADS
+ self->d_dict = ret;
if (self->d_dict == NULL) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError,
- "Failed to create ZSTD_DDict instance from zstd "
- "dictionary content. Maybe the content is corrupted.");
+ "Failed to create a ZSTD_DDict instance from "
+ "Zstandard dictionary content.");
}
}
}
- /* Don't lose any exception */
- ret = self->d_dict;
- Py_END_CRITICAL_SECTION();
-
- return ret;
+ return self->d_dict;
}
-/* Set decompression parameters to decompression context */
-int
-_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
+static int
+_zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
{
- size_t zstd_ret;
- PyObject *key, *value;
- Py_ssize_t pos;
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state == NULL) {
return -1;
}
if (!PyDict_Check(options)) {
- PyErr_SetString(PyExc_TypeError,
- "options argument should be dict object.");
+ PyErr_Format(PyExc_TypeError,
+ "ZstdDecompressor() argument 'options' must be dict, not %T",
+ options);
return -1;
}
- pos = 0;
+ Py_ssize_t pos = 0;
+ PyObject *key, *value;
while (PyDict_Next(options, &pos, &key, &value)) {
/* Check key type */
if (Py_TYPE(key) == mod_state->CParameter_type) {
PyErr_SetString(PyExc_TypeError,
- "Key of decompression options dict should "
- "NOT be CParameter.");
+ "compression options dictionary key must not be a "
+ "CompressionParameter attribute");
return -1;
}
- /* Both key & value should be 32-bit signed int */
+ Py_INCREF(key);
+ Py_INCREF(value);
int key_v = PyLong_AsInt(key);
+ Py_DECREF(key);
if (key_v == -1 && PyErr_Occurred()) {
- PyErr_SetString(PyExc_ValueError,
- "Key of options dict should be a DParameter attribute.");
return -1;
}
- // TODO(emmatyping): check bounds when there is a value error here for better
- // error message?
int value_v = PyLong_AsInt(value);
+ Py_DECREF(value);
if (value_v == -1 && PyErr_Occurred()) {
- PyErr_SetString(PyExc_ValueError,
- "Value of options dict should be an int.");
return -1;
}
/* Set parameter to compression context */
- Py_BEGIN_CRITICAL_SECTION(self);
- zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v);
- Py_END_CRITICAL_SECTION();
+ size_t zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v);
/* Check error */
if (ZSTD_isError(zstd_ret)) {
- set_parameter_error(mod_state, 0, key_v, value_v);
+ set_parameter_error(0, key_v, value_v);
return -1;
}
}
return 0;
}
-/* Load dictionary or prefix to decompression context */
-int
-_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
+static int
+_zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd,
+ _zstd_state *mod_state, int type)
{
size_t zstd_ret;
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
- if (mod_state == NULL) {
- return -1;
- }
- ZstdDict *zd;
- int type, ret;
-
- /* Check ZstdDict */
- ret = PyObject_IsInstance(dict, (PyObject*)mod_state->ZstdDict_type);
- if (ret < 0) {
- return -1;
- }
- else if (ret > 0) {
- /* When decompressing, use digested dictionary by default. */
- zd = (ZstdDict*)dict;
- type = DICT_TYPE_DIGESTED;
- goto load;
- }
-
- /* Check (ZstdDict, type) */
- if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2) {
- /* Check ZstdDict */
- ret = PyObject_IsInstance(PyTuple_GET_ITEM(dict, 0),
- (PyObject*)mod_state->ZstdDict_type);
- if (ret < 0) {
- return -1;
- }
- else if (ret > 0) {
- /* type == -1 may indicate an error. */
- type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1));
- if (type == DICT_TYPE_DIGESTED ||
- type == DICT_TYPE_UNDIGESTED ||
- type == DICT_TYPE_PREFIX)
- {
- assert(type >= 0);
- zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0);
- goto load;
- }
- }
- }
-
- /* Wrong type */
- PyErr_SetString(PyExc_TypeError,
- "zstd_dict argument should be ZstdDict object.");
- return -1;
-
-load:
if (type == DICT_TYPE_DIGESTED) {
/* Get ZSTD_DDict */
ZSTD_DDict *d_dict = _get_DDict(zd);
@@ -178,27 +144,17 @@ load:
return -1;
}
/* Reference a prepared dictionary */
- Py_BEGIN_CRITICAL_SECTION(self);
zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict);
- Py_END_CRITICAL_SECTION();
}
else if (type == DICT_TYPE_UNDIGESTED) {
/* Load a dictionary */
- Py_BEGIN_CRITICAL_SECTION2(self, zd);
- zstd_ret = ZSTD_DCtx_loadDictionary(
- self->dctx,
- PyBytes_AS_STRING(zd->dict_content),
- Py_SIZE(zd->dict_content));
- Py_END_CRITICAL_SECTION2();
+ zstd_ret = ZSTD_DCtx_loadDictionary(self->dctx, zd->dict_buffer,
+ zd->dict_len);
}
else if (type == DICT_TYPE_PREFIX) {
/* Load a prefix */
- Py_BEGIN_CRITICAL_SECTION2(self, zd);
- zstd_ret = ZSTD_DCtx_refPrefix(
- self->dctx,
- PyBytes_AS_STRING(zd->dict_content),
- Py_SIZE(zd->dict_content));
- Py_END_CRITICAL_SECTION2();
+ zstd_ret = ZSTD_DCtx_refPrefix(self->dctx, zd->dict_buffer,
+ zd->dict_len);
}
else {
/* Impossible code path */
@@ -215,22 +171,31 @@ load:
return 0;
}
-
+/* Load dictionary or prefix to decompression context */
+static int
+_zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
+{
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
+ /* When decompressing, use digested dictionary by default. */
+ int type = DICT_TYPE_DIGESTED;
+ ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type);
+ if (zd == NULL) {
+ return -1;
+ }
+ int ret;
+ PyMutex_Lock(&zd->lock);
+ ret = _zstd_load_impl(self, zd, mod_state, type);
+ PyMutex_Unlock(&zd->lock);
+ return ret;
+}
/*
- Given the two types of decompressors (defined in _zstdmodule.h):
-
- typedef enum {
- TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class
- TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function
- } decompress_type;
-
- Decompress implementation for <D>, <E>, pseudo code:
+ Decompress implementation in pseudo code:
initialize_output_buffer
while True:
decompress_data
- set_object_flag # .eof for <D>, .at_frame_edge for <E>.
+ set_object_flag # .eof
if output_buffer_exhausted:
if output_buffer_reached_max_length:
@@ -240,76 +205,26 @@ load:
finish
ZSTD_decompressStream()'s size_t return value:
- - 0 when a frame is completely decoded and fully flushed, zstd's internal
- buffer has no data.
+ - 0 when a frame is completely decoded and fully flushed,
+ zstd's internal buffer has no data.
- An error code, which can be tested using ZSTD_isError().
- Or any other value > 0, which means there is still some decoding or
flushing to do to complete current frame.
Note, decompressing "an empty input" in any case will make it > 0.
-
- <E> supports multiple frames, has an .at_frame_edge flag, it means both the
- input and output streams are at a frame edge. The flag can be set by this
- statement:
-
- .at_frame_edge = (zstd_ret == 0) ? 1 : 0
-
- But if decompressing "an empty input" at "a frame edge", zstd_ret will be
- non-zero, then .at_frame_edge will be wrongly set to false. To solve this
- problem, two AFE checks are needed to ensure that: when at "a frame edge",
- empty input will not be decompressed.
-
- // AFE check
- if (self->at_frame_edge && in->pos == in->size) {
- finish
- }
-
- In <E>, if .at_frame_edge is eventually set to true, but input stream has
- unconsumed data (in->pos < in->size), then the outer function
- stream_decompress() will set .at_frame_edge to false. In this case,
- although the output stream is at a frame edge, for the caller, the input
- stream is not at a frame edge, see below diagram. This behavior does not
- affect the next AFE check, since (in->pos < in->size).
-
- input stream: --------------|---
- ^
- output stream: ====================|
- ^
*/
-PyObject *
-decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
- Py_ssize_t max_length,
- Py_ssize_t initial_size,
- decompress_type type)
+static PyObject *
+decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in,
+ Py_ssize_t max_length)
{
size_t zstd_ret;
ZSTD_outBuffer out;
_BlocksOutputBuffer buffer = {.list = NULL};
PyObject *ret;
- /* The first AFE check for setting .at_frame_edge flag */
- if (type == TYPE_ENDLESS_DECOMPRESSOR) {
- if (self->at_frame_edge && in->pos == in->size) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
- if (mod_state == NULL) {
- return NULL;
- }
- ret = mod_state->empty_bytes;
- Py_INCREF(ret);
- return ret;
- }
- }
-
/* Initialize the output buffer */
- if (initial_size >= 0) {
- if (_OutputBuffer_InitWithSize(&buffer, &out, max_length, initial_size) < 0) {
- goto error;
- }
- }
- else {
- if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) {
- goto error;
- }
+ if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) {
+ goto error;
}
assert(out.pos == 0);
@@ -321,33 +236,20 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
/* Check error */
if (ZSTD_isError(zstd_ret)) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
- if (mod_state != NULL) {
- set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret);
- }
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
+ set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret);
goto error;
}
- /* Set .eof/.af_frame_edge flag */
- if (type == TYPE_DECOMPRESSOR) {
- /* ZstdDecompressor class stops when a frame is decompressed */
- if (zstd_ret == 0) {
- self->eof = 1;
- break;
- }
- }
- else if (type == TYPE_ENDLESS_DECOMPRESSOR) {
- /* decompress() function supports multiple frames */
- self->at_frame_edge = (zstd_ret == 0) ? 1 : 0;
-
- /* The second AFE check for setting .at_frame_edge flag */
- if (self->at_frame_edge && in->pos == in->size) {
- break;
- }
+ /* Set .eof flag */
+ if (zstd_ret == 0) {
+ /* Stop when a frame is decompressed */
+ self->eof = 1;
+ break;
}
/* Need to check out before in. Maybe zstd's internal buffer still has
- a few bytes can be output, grow the buffer and continue. */
+ a few bytes that can be output, grow the buffer and continue. */
if (out.pos == out.size) {
/* Output buffer exhausted */
@@ -380,67 +282,40 @@ error:
return NULL;
}
-void
-decompressor_reset_session(ZstdDecompressor *self,
- decompress_type type)
+static void
+decompressor_reset_session_lock_held(ZstdDecompressor *self)
{
- // TODO(emmatyping): use _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED here
- // and ensure lock is always held
+ assert(PyMutex_IsLocked(&self->lock));
/* Reset variables */
self->in_begin = 0;
self->in_end = 0;
- if (type == TYPE_DECOMPRESSOR) {
- Py_CLEAR(self->unused_data);
- }
+ Py_CLEAR(self->unused_data);
/* Reset variables in one operation */
self->needs_input = 1;
- self->at_frame_edge = 1;
self->eof = 0;
- self->_unused_char_for_align = 0;
- /* Resetting session never fail */
+ /* Resetting session is guaranteed to never fail */
ZSTD_DCtx_reset(self->dctx, ZSTD_reset_session_only);
}
-PyObject *
-stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length,
- decompress_type type)
+static PyObject *
+stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data,
+ Py_ssize_t max_length)
{
- Py_ssize_t initial_buffer_size = -1;
+ assert(PyMutex_IsLocked(&self->lock));
ZSTD_inBuffer in;
PyObject *ret = NULL;
int use_input_buffer;
- if (type == TYPE_DECOMPRESSOR) {
- /* Check .eof flag */
- if (self->eof) {
- PyErr_SetString(PyExc_EOFError, "Already at the end of a zstd frame.");
- assert(ret == NULL);
- goto success;
- }
- }
- else if (type == TYPE_ENDLESS_DECOMPRESSOR) {
- /* Fast path for the first frame */
- if (self->at_frame_edge && self->in_begin == self->in_end) {
- /* Read decompressed size */
- uint64_t decompressed_size = ZSTD_getFrameContentSize(data->buf, data->len);
-
- /* These two zstd constants always > PY_SSIZE_T_MAX:
- ZSTD_CONTENTSIZE_UNKNOWN is (0ULL - 1)
- ZSTD_CONTENTSIZE_ERROR is (0ULL - 2)
-
- Use ZSTD_findFrameCompressedSize() to check complete frame,
- prevent allocating too much memory for small input chunk. */
-
- if (decompressed_size <= (uint64_t) PY_SSIZE_T_MAX &&
- !ZSTD_isError(ZSTD_findFrameCompressedSize(data->buf, data->len)) )
- {
- initial_buffer_size = (Py_ssize_t) decompressed_size;
- }
- }
+ /* Check .eof flag */
+ if (self->eof) {
+ PyErr_SetString(PyExc_EOFError,
+ "Already at the end of a Zstandard frame.");
+ assert(ret == NULL);
+ return NULL;
}
/* Prepare input buffer w/wo unconsumed data */
@@ -527,30 +402,18 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length
assert(in.pos == 0);
/* Decompress */
- ret = decompress_impl(self, &in,
- max_length, initial_buffer_size,
- type);
+ ret = decompress_lock_held(self, &in, max_length);
if (ret == NULL) {
goto error;
}
/* Unconsumed input data */
if (in.pos == in.size) {
- if (type == TYPE_DECOMPRESSOR) {
- if (Py_SIZE(ret) == max_length || self->eof) {
- self->needs_input = 0;
- }
- else {
- self->needs_input = 1;
- }
+ if (Py_SIZE(ret) == max_length || self->eof) {
+ self->needs_input = 0;
}
- else if (type == TYPE_ENDLESS_DECOMPRESSOR) {
- if (Py_SIZE(ret) == max_length && !self->at_frame_edge) {
- self->needs_input = 0;
- }
- else {
- self->needs_input = 1;
- }
+ else {
+ self->needs_input = 1;
}
if (use_input_buffer) {
@@ -564,15 +427,11 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length
self->needs_input = 0;
- if (type == TYPE_ENDLESS_DECOMPRESSOR) {
- self->at_frame_edge = 0;
- }
-
if (!use_input_buffer) {
/* Discard buffer if it's too small
(resizing it may needlessly copy the current contents) */
- if (self->input_buffer != NULL &&
- self->input_buffer_size < data_size)
+ if (self->input_buffer != NULL
+ && self->input_buffer_size < data_size)
{
PyMem_Free(self->input_buffer);
self->input_buffer = NULL;
@@ -600,47 +459,57 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length
}
}
- goto success;
+ return ret;
error:
/* Reset decompressor's states/session */
- decompressor_reset_session(self, type);
+ decompressor_reset_session_lock_held(self);
Py_CLEAR(ret);
-success:
-
- return ret;
+ return NULL;
}
+/*[clinic input]
+@classmethod
+_zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new
+ zstd_dict: object = None
+ A ZstdDict object, a pre-trained Zstandard dictionary.
+ options: object = None
+ A dict object that contains advanced decompression parameters.
+
+Create a decompressor object for decompressing data incrementally.
+
+Thread-safe at method level. For one-shot decompression, use the decompress()
+function instead.
+[clinic start generated code]*/
+
static PyObject *
-_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
+ PyObject *options)
+/*[clinic end generated code: output=590ca65c1102ff4a input=213daa57e3ea4062]*/
{
- ZstdDecompressor *self;
- self = PyObject_GC_New(ZstdDecompressor, type);
+ ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type);
if (self == NULL) {
goto error;
}
- self->inited = 0;
- self->dict = NULL;
self->input_buffer = NULL;
self->input_buffer_size = 0;
self->in_begin = -1;
self->in_end = -1;
self->unused_data = NULL;
self->eof = 0;
+ self->dict = NULL;
+ self->lock = (PyMutex){0};
/* needs_input flag */
self->needs_input = 1;
- /* at_frame_edge flag */
- self->at_frame_edge = 1;
-
/* Decompression context */
self->dctx = ZSTD_createDCtx();
if (self->dctx == NULL) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
+ _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError,
"Unable to create ZSTD_DCtx instance.");
@@ -648,12 +517,29 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
goto error;
}
+ /* Load Zstandard dictionary to decompression context */
+ if (zstd_dict != Py_None) {
+ if (_zstd_load_d_dict(self, zstd_dict) < 0) {
+ goto error;
+ }
+ Py_INCREF(zstd_dict);
+ self->dict = zstd_dict;
+ }
+
+ /* Set options dictionary */
+ if (options != Py_None) {
+ if (_zstd_set_d_parameters(self, options) < 0) {
+ goto error;
+ }
+ }
+
+ // We can only start GC tracking once self->dict is set.
+ PyObject_GC_Track(self);
+
return (PyObject*)self;
error:
- if (self != NULL) {
- PyObject_GC_Del(self);
- }
+ Py_XDECREF(self);
return NULL;
}
@@ -665,7 +551,11 @@ ZstdDecompressor_dealloc(PyObject *ob)
PyObject_GC_UnTrack(self);
/* Free decompression context */
- ZSTD_freeDCtx(self->dctx);
+ if (self->dctx) {
+ ZSTD_freeDCtx(self->dctx);
+ }
+
+ assert(!PyMutex_IsLocked(&self->lock));
/* Py_CLEAR the dict after free decompression context */
Py_CLEAR(self->dict);
@@ -682,56 +572,6 @@ ZstdDecompressor_dealloc(PyObject *ob)
}
/*[clinic input]
-_zstd.ZstdDecompressor.__init__
-
- zstd_dict: object = None
- A ZstdDict object, a pre-trained zstd dictionary.
- options: object = None
- A dict object that contains advanced decompression parameters.
-
-Create a decompressor object for decompressing data incrementally.
-
-Thread-safe at method level. For one-shot decompression, use the decompress()
-function instead.
-[clinic start generated code]*/
-
-static int
-_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self,
- PyObject *zstd_dict, PyObject *options)
-/*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/
-{
- /* Only called once */
- if (self->inited) {
- PyErr_SetString(PyExc_RuntimeError, init_twice_msg);
- return -1;
- }
- self->inited = 1;
-
- /* Load dictionary to decompression context */
- if (zstd_dict != Py_None) {
- if (_PyZstd_load_d_dict(self, zstd_dict) < 0) {
- return -1;
- }
-
- /* Py_INCREF the dict */
- Py_INCREF(zstd_dict);
- self->dict = zstd_dict;
- }
-
- /* Set option to decompression context */
- if (options != Py_None) {
- if (_PyZstd_set_d_parameters(self, options) < 0) {
- return -1;
- }
- }
-
- // We can only start tracking self with the GC once self->dict is set.
- PyObject_GC_Track(self);
- return 0;
-}
-
-/*[clinic input]
-@critical_section
@getter
_zstd.ZstdDecompressor.unused_data
@@ -743,20 +583,15 @@ decompressed, unused input data after the frame. Otherwise this will be b''.
static PyObject *
_zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
-/*[clinic end generated code: output=f3a20940f11b6b09 input=5233800bef00df04]*/
+/*[clinic end generated code: output=f3a20940f11b6b09 input=54d41ecd681a3444]*/
{
PyObject *ret;
- /* Thread-safe code */
- Py_BEGIN_CRITICAL_SECTION(self);
+ PyMutex_Lock(&self->lock);
if (!self->eof) {
- _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
- if (mod_state == NULL) {
- return NULL;
- }
- ret = mod_state->empty_bytes;
- Py_INCREF(ret);
+ PyMutex_Unlock(&self->lock);
+ return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
}
else {
if (self->unused_data == NULL) {
@@ -772,8 +607,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
}
}
- Py_END_CRITICAL_SECTION();
-
+ PyMutex_Unlock(&self->lock);
return ret;
}
@@ -781,7 +615,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
_zstd.ZstdDecompressor.decompress
data: Py_buffer
- A bytes-like object, zstd data to be decompressed.
+ A bytes-like object, Zstandard data to be decompressed.
max_length: Py_ssize_t = -1
Maximum size of returned data. When it is negative, the size of
output buffer is unlimited. When it is nonnegative, returns at
@@ -807,25 +641,19 @@ static PyObject *
_zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self,
Py_buffer *data,
Py_ssize_t max_length)
-/*[clinic end generated code: output=a4302b3c940dbec6 input=830e455bc9a50b6e]*/
+/*[clinic end generated code: output=a4302b3c940dbec6 input=6463dfdf98091caa]*/
{
PyObject *ret;
/* Thread-safe code */
- Py_BEGIN_CRITICAL_SECTION(self);
-
- ret = stream_decompress(self, data, max_length, TYPE_DECOMPRESSOR);
- Py_END_CRITICAL_SECTION();
+ PyMutex_Lock(&self->lock);
+ ret = stream_decompress_lock_held(self, data, max_length);
+ PyMutex_Unlock(&self->lock);
return ret;
}
-#define clinic_state() (get_zstd_state_from_type(type))
-#include "clinic/decompressor.c.h"
-#undef clinic_state
-
static PyMethodDef ZstdDecompressor_methods[] = {
_ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF
-
- {0}
+ {NULL, NULL}
};
PyDoc_STRVAR(ZstdDecompressor_eof_doc,
@@ -833,24 +661,22 @@ PyDoc_STRVAR(ZstdDecompressor_eof_doc,
"after that, an EOFError exception will be raised.");
PyDoc_STRVAR(ZstdDecompressor_needs_input_doc,
-"If the max_length output limit in .decompress() method has been reached, and\n"
-"the decompressor has (or may has) unconsumed input data, it will be set to\n"
-"False. In this case, pass b'' to .decompress() method may output further data.");
+"If the max_length output limit in .decompress() method has been reached,\n"
+"and the decompressor has (or may has) unconsumed input data, it will be set\n"
+"to False. In this case, passing b'' to the .decompress() method may output\n"
+"further data.");
static PyMemberDef ZstdDecompressor_members[] = {
{"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof),
Py_READONLY, ZstdDecompressor_eof_doc},
-
{"needs_input", Py_T_BOOL, offsetof(ZstdDecompressor, needs_input),
Py_READONLY, ZstdDecompressor_needs_input_doc},
-
- {0}
+ {NULL}
};
static PyGetSetDef ZstdDecompressor_getset[] = {
_ZSTD_ZSTDDECOMPRESSOR_UNUSED_DATA_GETSETDEF
-
- {0}
+ {NULL}
};
static int
@@ -873,19 +699,19 @@ ZstdDecompressor_clear(PyObject *ob)
static PyType_Slot ZstdDecompressor_slots[] = {
{Py_tp_new, _zstd_ZstdDecompressor_new},
{Py_tp_dealloc, ZstdDecompressor_dealloc},
- {Py_tp_init, _zstd_ZstdDecompressor___init__},
{Py_tp_methods, ZstdDecompressor_methods},
{Py_tp_members, ZstdDecompressor_members},
{Py_tp_getset, ZstdDecompressor_getset},
- {Py_tp_doc, (char*)_zstd_ZstdDecompressor___init____doc__},
+ {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__},
{Py_tp_traverse, ZstdDecompressor_traverse},
{Py_tp_clear, ZstdDecompressor_clear},
- {0}
+ {0, 0}
};
-PyType_Spec ZstdDecompressor_type_spec = {
- .name = "_zstd.ZstdDecompressor",
+PyType_Spec zstd_decompressor_type_spec = {
+ .name = "compression.zstd.ZstdDecompressor",
.basicsize = sizeof(ZstdDecompressor),
- .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+ .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE
+ | Py_TPFLAGS_HAVE_GC,
.slots = ZstdDecompressor_slots,
};
diff --git a/Modules/_zstd/zdict.c b/Modules/_zstd/zdict.c
deleted file mode 100644
index 28ab964a6ca..00000000000
--- a/Modules/_zstd/zdict.c
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
-Low level interface to Meta's zstd library for use in the compression.zstd
-Python module.
-*/
-
-/* ZstdDict class definitions */
-
-/*[clinic input]
-module _zstd
-class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type"
-[clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5d1254c497e52ba]*/
-
-#ifndef Py_BUILD_CORE_BUILTIN
-# define Py_BUILD_CORE_MODULE 1
-#endif
-
-#include "_zstdmodule.h"
-
-#include <stddef.h> // offsetof()
-
-#define ZstdDict_CAST(op) ((ZstdDict *)op)
-
-static PyObject *
-_zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs))
-{
- ZstdDict *self;
- self = PyObject_GC_New(ZstdDict, type);
- if (self == NULL) {
- goto error;
- }
-
- self->dict_content = NULL;
- self->inited = 0;
- self->d_dict = NULL;
-
- /* ZSTD_CDict dict */
- self->c_dicts = PyDict_New();
- if (self->c_dicts == NULL) {
- goto error;
- }
-
- return (PyObject*)self;
-
-error:
- if (self != NULL) {
- PyObject_GC_Del(self);
- }
- return NULL;
-}
-
-static void
-ZstdDict_dealloc(PyObject *ob)
-{
- ZstdDict *self = ZstdDict_CAST(ob);
-
- PyObject_GC_UnTrack(self);
-
- /* Free ZSTD_DDict instance */
- ZSTD_freeDDict(self->d_dict);
-
- /* Release dict_content after Free ZSTD_CDict/ZSTD_DDict instances */
- Py_CLEAR(self->dict_content);
- Py_CLEAR(self->c_dicts);
-
- PyTypeObject *tp = Py_TYPE(self);
- PyObject_GC_Del(ob);
- Py_DECREF(tp);
-}
-
-/*[clinic input]
-_zstd.ZstdDict.__init__
-
- dict_content: object
- A bytes-like object, dictionary's content.
- is_raw: bool = False
- This parameter is for advanced user. True means dict_content
- argument is a "raw content" dictionary, free of any format
- restriction. False means dict_content argument is an ordinary
- zstd dictionary, was created by zstd functions, follow a
- specified format.
-
-Represents a zstd dictionary, which can be used for compression/decompression.
-
-It's thread-safe, and can be shared by multiple ZstdCompressor /
-ZstdDecompressor objects.
-[clinic start generated code]*/
-
-static int
-_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content,
- int is_raw)
-/*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/
-{
- /* Only called once */
- if (self->inited) {
- PyErr_SetString(PyExc_RuntimeError, init_twice_msg);
- return -1;
- }
- self->inited = 1;
-
- /* Check dict_content's type */
- self->dict_content = PyBytes_FromObject(dict_content);
- if (self->dict_content == NULL) {
- PyErr_SetString(PyExc_TypeError,
- "dict_content argument should be bytes-like object.");
- return -1;
- }
-
- /* Both ordinary dictionary and "raw content" dictionary should
- at least 8 bytes */
- if (Py_SIZE(self->dict_content) < 8) {
- PyErr_SetString(PyExc_ValueError,
- "Zstd dictionary content should at least 8 bytes.");
- return -1;
- }
-
- /* Get dict_id, 0 means "raw content" dictionary. */
- self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content),
- Py_SIZE(self->dict_content));
-
- /* Check validity for ordinary dictionary */
- if (!is_raw && self->dict_id == 0) {
- char *msg = "The dict_content argument is not a valid zstd "
- "dictionary. The first 4 bytes of a valid zstd dictionary "
- "should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n"
- "If you are an advanced user, and can be sure that "
- "dict_content argument is a \"raw content\" zstd "
- "dictionary, set is_raw parameter to True.";
- PyErr_SetString(PyExc_ValueError, msg);
- return -1;
- }
-
- // Can only track self once self->dict_content is included
- PyObject_GC_Track(self);
- return 0;
-}
-
-#define clinic_state() (get_zstd_state(type))
-#include "clinic/zdict.c.h"
-#undef clinic_state
-
-PyDoc_STRVAR(ZstdDict_dictid_doc,
-"ID of zstd dictionary, a 32-bit unsigned int value.\n\n"
-"Non-zero means ordinary dictionary, was created by zstd functions, follow\n"
-"a specified format.\n\n"
-"0 means a \"raw content\" dictionary, free of any format restriction, used\n"
-"for advanced user.");
-
-PyDoc_STRVAR(ZstdDict_dictcontent_doc,
-"The content of zstd dictionary, a bytes object, it's the same as dict_content\n"
-"argument in ZstdDict.__init__() method. It can be used with other programs.");
-
-static PyObject *
-ZstdDict_str(PyObject *ob)
-{
- ZstdDict *dict = ZstdDict_CAST(ob);
- return PyUnicode_FromFormat("<ZstdDict dict_id=%u dict_size=%zd>",
- dict->dict_id, Py_SIZE(dict->dict_content));
-}
-
-static PyMemberDef ZstdDict_members[] = {
- {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc},
- {"dict_content", Py_T_OBJECT_EX, offsetof(ZstdDict, dict_content), Py_READONLY, ZstdDict_dictcontent_doc},
- {0}
-};
-
-/*[clinic input]
-@critical_section
-@getter
-_zstd.ZstdDict.as_digested_dict
-
-Load as a digested dictionary to compressor.
-
-Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict)
-1. Some advanced compression parameters of compressor may be overridden
- by parameters of digested dictionary.
-2. ZstdDict has a digested dictionaries cache for each compression level.
- It's faster when loading again a digested dictionary with the same
- compression level.
-3. No need to use this for decompression.
-[clinic start generated code]*/
-
-static PyObject *
-_zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self)
-/*[clinic end generated code: output=09b086e7a7320dbb input=585448c79f31f74a]*/
-{
- return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED);
-}
-
-/*[clinic input]
-@critical_section
-@getter
-_zstd.ZstdDict.as_undigested_dict
-
-Load as an undigested dictionary to compressor.
-
-Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict)
-1. The advanced compression parameters of compressor will not be overridden.
-2. Loading an undigested dictionary is costly. If load an undigested dictionary
- multiple times, consider reusing a compressor object.
-3. No need to use this for decompression.
-[clinic start generated code]*/
-
-static PyObject *
-_zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self)
-/*[clinic end generated code: output=43c7a989e6d4253a input=022b0829ffb1c220]*/
-{
- return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED);
-}
-
-/*[clinic input]
-@critical_section
-@getter
-_zstd.ZstdDict.as_prefix
-
-Load as a prefix to compressor/decompressor.
-
-Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix)
-1. Prefix is compatible with long distance matching, while dictionary is not.
-2. It only works for the first frame, then the compressor/decompressor will
- return to no prefix state.
-3. When decompressing, must use the same prefix as when compressing."
-[clinic start generated code]*/
-
-static PyObject *
-_zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self)
-/*[clinic end generated code: output=6f7130c356595a16 input=09fb82a6a5407e87]*/
-{
- return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX);
-}
-
-static PyGetSetDef ZstdDict_getset[] = {
- _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF
-
- _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF
-
- _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF
-
- {0}
-};
-
-static Py_ssize_t
-ZstdDict_length(PyObject *ob)
-{
- ZstdDict *self = ZstdDict_CAST(ob);
- assert(PyBytes_Check(self->dict_content));
- return Py_SIZE(self->dict_content);
-}
-
-static int
-ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg)
-{
- ZstdDict *self = ZstdDict_CAST(ob);
- Py_VISIT(self->c_dicts);
- Py_VISIT(self->dict_content);
- return 0;
-}
-
-static int
-ZstdDict_clear(PyObject *ob)
-{
- ZstdDict *self = ZstdDict_CAST(ob);
- Py_CLEAR(self->dict_content);
- return 0;
-}
-
-static PyType_Slot zstddict_slots[] = {
- {Py_tp_members, ZstdDict_members},
- {Py_tp_getset, ZstdDict_getset},
- {Py_tp_new, _zstd_ZstdDict_new},
- {Py_tp_dealloc, ZstdDict_dealloc},
- {Py_tp_init, _zstd_ZstdDict___init__},
- {Py_tp_str, ZstdDict_str},
- {Py_tp_doc, (char*)_zstd_ZstdDict___init____doc__},
- {Py_sq_length, ZstdDict_length},
- {Py_tp_traverse, ZstdDict_traverse},
- {Py_tp_clear, ZstdDict_clear},
- {0}
-};
-
-PyType_Spec zstddict_type_spec = {
- .name = "_zstd.ZstdDict",
- .basicsize = sizeof(ZstdDict),
- .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
- .slots = zstddict_slots,
-};
diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c
new file mode 100644
index 00000000000..14f74aaed46
--- /dev/null
+++ b/Modules/_zstd/zstddict.c
@@ -0,0 +1,273 @@
+/* Low level interface to the Zstandard algorthm & the zstd library. */
+
+/* ZstdDict class definitions */
+
+/*[clinic input]
+module _zstd
+class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec"
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/
+
+#ifndef Py_BUILD_CORE_BUILTIN
+# define Py_BUILD_CORE_MODULE 1
+#endif
+
+#include "Python.h"
+
+#include "_zstdmodule.h"
+#include "clinic/zstddict.c.h"
+#include "internal/pycore_lock.h" // PyMutex_IsLocked
+
+#include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict()
+
+#define ZstdDict_CAST(op) ((ZstdDict *)op)
+
+/*[clinic input]
+@classmethod
+_zstd.ZstdDict.__new__ as _zstd_ZstdDict_new
+ dict_content: Py_buffer
+ The content of a Zstandard dictionary as a bytes-like object.
+ /
+ *
+ is_raw: bool = False
+ If true, perform no checks on *dict_content*, useful for some
+ advanced cases. Otherwise, check that the content represents
+ a Zstandard dictionary created by the zstd library or CLI.
+
+Represents a Zstandard dictionary.
+
+The dictionary can be used for compression or decompression, and can be shared
+by multiple ZstdCompressor or ZstdDecompressor objects.
+[clinic start generated code]*/
+
+static PyObject *
+_zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content,
+ int is_raw)
+/*[clinic end generated code: output=685b7406a48b0949 input=9e8c493e31c98383]*/
+{
+ /* All dictionaries must be at least 8 bytes */
+ if (dict_content->len < 8) {
+ PyErr_SetString(PyExc_ValueError,
+ "Zstandard dictionary content too short "
+ "(must have at least eight bytes)");
+ return NULL;
+ }
+
+ ZstdDict* self = PyObject_GC_New(ZstdDict, type);
+ if (self == NULL) {
+ return NULL;
+ }
+
+ self->d_dict = NULL;
+ self->dict_buffer = NULL;
+ self->dict_id = 0;
+ self->lock = (PyMutex){0};
+
+ /* ZSTD_CDict dict */
+ self->c_dicts = PyDict_New();
+ if (self->c_dicts == NULL) {
+ goto error;
+ }
+
+ self->dict_buffer = PyMem_Malloc(dict_content->len);
+ if (!self->dict_buffer) {
+ PyErr_NoMemory();
+ goto error;
+ }
+ memcpy(self->dict_buffer, dict_content->buf, dict_content->len);
+ self->dict_len = dict_content->len;
+
+ /* Get dict_id, 0 means "raw content" dictionary. */
+ self->dict_id = ZSTD_getDictID_fromDict(self->dict_buffer, self->dict_len);
+
+ /* Check validity for ordinary dictionary */
+ if (!is_raw && self->dict_id == 0) {
+ PyErr_SetString(PyExc_ValueError, "invalid Zstandard dictionary");
+ goto error;
+ }
+
+ PyObject_GC_Track(self);
+
+ return (PyObject *)self;
+
+error:
+ Py_XDECREF(self);
+ return NULL;
+}
+
+static void
+ZstdDict_dealloc(PyObject *ob)
+{
+ ZstdDict *self = ZstdDict_CAST(ob);
+
+ PyObject_GC_UnTrack(self);
+
+ /* Free ZSTD_DDict instance */
+ if (self->d_dict) {
+ ZSTD_freeDDict(self->d_dict);
+ }
+
+ assert(!PyMutex_IsLocked(&self->lock));
+
+ /* Release dict_buffer after freeing ZSTD_CDict/ZSTD_DDict instances */
+ PyMem_Free(self->dict_buffer);
+ Py_CLEAR(self->c_dicts);
+
+ PyTypeObject *tp = Py_TYPE(self);
+ tp->tp_free(self);
+ Py_DECREF(tp);
+}
+
+PyDoc_STRVAR(ZstdDict_dictid_doc,
+"the Zstandard dictionary, an int between 0 and 2**32.\n\n"
+"A non-zero value represents an ordinary Zstandard dictionary, "
+"conforming to the standardised format.\n\n"
+"The special value '0' means a 'raw content' dictionary,"
+"without any restrictions on format or content.");
+
+static PyObject *
+ZstdDict_repr(PyObject *ob)
+{
+ ZstdDict *dict = ZstdDict_CAST(ob);
+ return PyUnicode_FromFormat("<ZstdDict dict_id=%u dict_size=%zd>",
+ (unsigned int)dict->dict_id, dict->dict_len);
+}
+
+static PyMemberDef ZstdDict_members[] = {
+ {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc},
+ {NULL}
+};
+
+/*[clinic input]
+@getter
+_zstd.ZstdDict.dict_content
+
+The content of a Zstandard dictionary, as a bytes object.
+[clinic start generated code]*/
+
+static PyObject *
+_zstd_ZstdDict_dict_content_get_impl(ZstdDict *self)
+/*[clinic end generated code: output=0d05caa5b550eabb input=4ed526d1c151c596]*/
+{
+ return PyBytes_FromStringAndSize(self->dict_buffer, self->dict_len);
+}
+
+/*[clinic input]
+@getter
+_zstd.ZstdDict.as_digested_dict
+
+Load as a digested dictionary to compressor.
+
+Pass this attribute as zstd_dict argument:
+compress(dat, zstd_dict=zd.as_digested_dict)
+
+1. Some advanced compression parameters of compressor may be overridden
+ by parameters of digested dictionary.
+2. ZstdDict has a digested dictionaries cache for each compression level.
+ It's faster when loading again a digested dictionary with the same
+ compression level.
+3. No need to use this for decompression.
+[clinic start generated code]*/
+
+static PyObject *
+_zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self)
+/*[clinic end generated code: output=09b086e7a7320dbb input=ee45e1b4a48f6f2c]*/
+{
+ return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED);
+}
+
+/*[clinic input]
+@getter
+_zstd.ZstdDict.as_undigested_dict
+
+Load as an undigested dictionary to compressor.
+
+Pass this attribute as zstd_dict argument:
+compress(dat, zstd_dict=zd.as_undigested_dict)
+
+1. The advanced compression parameters of compressor will not be overridden.
+2. Loading an undigested dictionary is costly. If load an undigested dictionary
+ multiple times, consider reusing a compressor object.
+3. No need to use this for decompression.
+[clinic start generated code]*/
+
+static PyObject *
+_zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self)
+/*[clinic end generated code: output=43c7a989e6d4253a input=d39210eedec76fed]*/
+{
+ return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED);
+}
+
+/*[clinic input]
+@getter
+_zstd.ZstdDict.as_prefix
+
+Load as a prefix to compressor/decompressor.
+
+Pass this attribute as zstd_dict argument:
+compress(dat, zstd_dict=zd.as_prefix)
+
+1. Prefix is compatible with long distance matching, while dictionary is not.
+2. It only works for the first frame, then the compressor/decompressor will
+ return to no prefix state.
+3. When decompressing, must use the same prefix as when compressing."
+[clinic start generated code]*/
+
+static PyObject *
+_zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self)
+/*[clinic end generated code: output=6f7130c356595a16 input=d59757b0b5a9551a]*/
+{
+ return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX);
+}
+
+static PyGetSetDef ZstdDict_getset[] = {
+ _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF
+ _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF
+ _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF
+ _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF
+ {NULL}
+};
+
+static Py_ssize_t
+ZstdDict_length(PyObject *ob)
+{
+ ZstdDict *self = ZstdDict_CAST(ob);
+ return self->dict_len;
+}
+
+static int
+ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg)
+{
+ ZstdDict *self = ZstdDict_CAST(ob);
+ Py_VISIT(self->c_dicts);
+ return 0;
+}
+
+static int
+ZstdDict_clear(PyObject *ob)
+{
+ ZstdDict *self = ZstdDict_CAST(ob);
+ Py_CLEAR(self->c_dicts);
+ return 0;
+}
+
+static PyType_Slot zstddict_slots[] = {
+ {Py_tp_members, ZstdDict_members},
+ {Py_tp_getset, ZstdDict_getset},
+ {Py_tp_new, _zstd_ZstdDict_new},
+ {Py_tp_dealloc, ZstdDict_dealloc},
+ {Py_tp_repr, ZstdDict_repr},
+ {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__},
+ {Py_sq_length, ZstdDict_length},
+ {Py_tp_traverse, ZstdDict_traverse},
+ {Py_tp_clear, ZstdDict_clear},
+ {0, 0}
+};
+
+PyType_Spec zstd_dict_type_spec = {
+ .name = "compression.zstd.ZstdDict",
+ .basicsize = sizeof(ZstdDict),
+ .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE
+ | Py_TPFLAGS_HAVE_GC,
+ .slots = zstddict_slots,
+};
diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h
new file mode 100644
index 00000000000..4a403416dbd
--- /dev/null
+++ b/Modules/_zstd/zstddict.h
@@ -0,0 +1,29 @@
+/* Low level interface to the Zstandard algorthm & the zstd library. */
+
+#ifndef ZSTD_DICT_H
+#define ZSTD_DICT_H
+
+#include <zstd.h> // ZSTD_DDict
+
+typedef struct {
+ PyObject_HEAD
+
+ /* Reusable compress/decompress dictionary, they are created once and
+ can be shared by multiple threads concurrently, since its usage is
+ read-only.
+ c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */
+ ZSTD_DDict *d_dict;
+ PyObject *c_dicts;
+
+ /* Dictionary content. */
+ char *dict_buffer;
+ Py_ssize_t dict_len;
+
+ /* Dictionary id */
+ uint32_t dict_id;
+
+ /* Lock to protect the digested dictionaries */
+ PyMutex lock;
+} ZstdDict;
+
+#endif // !ZSTD_DICT_H
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 10:47:50 +0000