diff options
| -rw-r--r-- | Makefile.pre.in | 2 | ||||
| -rw-r--r-- | Modules/_zstd/_zstdmodule.c | 8 | ||||
| -rw-r--r-- | Modules/_zstd/_zstdmodule.h | 125 | ||||
| -rw-r--r-- | Modules/_zstd/buffer.h | 8 | ||||
| -rw-r--r-- | Modules/_zstd/compressor.c | 53 | ||||
| -rw-r--r-- | Modules/_zstd/decompressor.c | 76 | ||||
| -rw-r--r-- | Modules/_zstd/zstddict.c | 22 | ||||
| -rw-r--r-- | Modules/_zstd/zstddict.h | 31 | ||||
| -rw-r--r-- | PCbuild/_zstd.vcxproj | 1 | ||||
| -rw-r--r-- | PCbuild/_zstd.vcxproj.filters | 3 |
10 files changed, 165 insertions, 164 deletions
diff --git a/Makefile.pre.in b/Makefile.pre.in index fd3d0274bca..452c4ad35ac 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3346,7 +3346,7 @@ MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_tes MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h -MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h +MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h MODULE__CODECS_CN_DEPS=$(srcdir)/Modules/cjkcodecs/mappings_cn.h $(CODECS_COMMON_HEADERS) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 483e88d98b7..bddef0d0b86 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -7,7 +7,13 @@ Python module. # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include "zstddict.h" + +#include <zstd.h> // ZSTD_*() +#include <zdict.h> // ZDICT_*() /*[clinic input] module _zstd @@ -727,7 +733,7 @@ static struct PyModuleDef_Slot _zstd_slots[] = { {0, NULL}, }; -struct PyModuleDef _zstdmodule = { +static struct PyModuleDef _zstdmodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_zstd", .m_doc = "Implementation module for Zstandard compression.", diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index d5283ddcf56..00e0d2177f3 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,4 +1,3 @@ -#pragma once /* Low level interface to Meta's zstd library for use in the compression.zstd Python module. @@ -6,127 +5,25 @@ Python module. /* Declarations shared between different parts of the _zstd module*/ -#include "Python.h" - -#include "zstd.h" -#include "zdict.h" - - -/* Forward declaration of module state */ -typedef struct _zstd_state _zstd_state; - -/* Forward reference of module def */ -extern PyModuleDef _zstdmodule; - -/* For clinic type calculations */ -static inline _zstd_state * -get_zstd_state_from_type(PyTypeObject *type) -{ - PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule); - if (module == NULL) { - return NULL; - } - void *state = PyModule_GetState(module); - assert(state != NULL); - return (_zstd_state *)state; -} +#ifndef ZSTD_MODULE_H +#define ZSTD_MODULE_H +/* Type specs */ extern PyType_Spec zstd_dict_type_spec; extern PyType_Spec zstd_compressor_type_spec; extern PyType_Spec zstd_decompressor_type_spec; -struct _zstd_state { +typedef struct { + /* Module heap types. */ PyTypeObject *ZstdDict_type; PyTypeObject *ZstdCompressor_type; PyTypeObject *ZstdDecompressor_type; PyObject *ZstdError; + /* enum types set by set_parameter_types. */ PyTypeObject *CParameter_type; PyTypeObject *DParameter_type; -}; - -typedef struct { - PyObject_HEAD - - /* Reusable compress/decompress dictionary, they are created once and - can be shared by multiple threads concurrently, since its usage is - read-only. - c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ - ZSTD_DDict *d_dict; - PyObject *c_dicts; - - /* Content of the dictionary, bytes object. */ - PyObject *dict_content; - /* Dictionary id */ - uint32_t dict_id; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdDict; - -typedef struct { - PyObject_HEAD - - /* Compression context */ - ZSTD_CCtx *cctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Last mode, initialized to ZSTD_e_end */ - int last_mode; - - /* (nbWorker >= 1) ? 1 : 0 */ - int use_multithread; - - /* Compression level */ - int compression_level; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdCompressor; - -typedef struct { - PyObject_HEAD - - /* Decompression context */ - ZSTD_DCtx *dctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Unconsumed input data */ - char *input_buffer; - size_t input_buffer_size; - size_t in_begin, in_end; - - /* Unused data */ - PyObject *unused_data; - - /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ - char needs_input; - - /* For decompress(), 0 or 1. - 1 when both input and output streams are at a frame edge, means a - frame is completely decoded and fully flushed, or the decompressor - just be initialized. */ - char at_frame_edge; - - /* For ZstdDecompressor, 0 or 1. - 1 means the end of the first frame has been reached. */ - char eof; - - /* Used for fast reset above three variables */ - char _unused_char_for_align; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdDecompressor; - -typedef enum { - TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function -} decompress_type; +} _zstd_state; typedef enum { ERR_DECOMPRESS, @@ -149,12 +46,6 @@ typedef enum { DICT_TYPE_PREFIX = 2 } dictionary_type; -static inline int -mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) -{ - return in->size == in->pos && out->size != out->pos; -} - /* Format error message and set ZstdError. */ extern void set_zstd_error(const _zstd_state* const state, @@ -164,4 +55,4 @@ extern void set_parameter_error(const _zstd_state* const state, int is_compress, int key_v, int value_v); -static const char init_twice_msg[] = "__init__ method is called twice."; +#endif // !ZSTD_MODULE_H diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index 319b1214833..c902eef4f8e 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -3,9 +3,13 @@ Low level interface to Meta's zstd library for use in the compression.zstd Python module. */ -#include "_zstdmodule.h" +#ifndef ZSTD_BUFFER_H +#define ZSTD_BUFFER_H + #include "pycore_blocks_output_buffer.h" +#include <zstd.h> // ZSTD_outBuffer + /* Blocks output buffer wrapper code */ /* Initialize the buffer, and grow the buffer. @@ -102,3 +106,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob) return buffer->allocated == buffer->max_length; } + +#endif // !ZSTD_BUFFER_H diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index b92083cf78f..e70eb637b29 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -7,24 +7,50 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type" +class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/ - +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "zstddict.h" +#include <stdbool.h> // bool #include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Compression context */ + ZSTD_CCtx *cctx; + + /* ZstdDict object in use */ + PyObject *dict; + + /* Last mode, initialized to ZSTD_e_end */ + int last_mode; + + /* (nbWorker >= 1) ? 1 : 0 */ + int use_multithread; + /* Compression level */ + int compression_level; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdCompressor; #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) +#include "clinic/compressor.c.h" + static int _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, const char *arg_name, const char* arg_type) @@ -292,10 +318,6 @@ load: return 0; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/compressor.c.h" -#undef clinic_state - static PyObject * _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) { @@ -305,7 +327,7 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject goto error; } - self->inited = 0; + self->initialized = 0; self->dict = NULL; self->use_multithread = 0; @@ -372,12 +394,11 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, PyObject *options, PyObject *zstd_dict) /*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/ { - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; if (level != Py_None && options != Py_None) { PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); @@ -488,6 +509,12 @@ error: return NULL; } +static inline int +mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) +{ + return in->size == in->pos && out->size != out->pos; +} + static PyObject * compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) { diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index d2b637205be..2ed88cd3f23 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -7,22 +7,65 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompressor_type" +class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4e6eae327c0c0c76]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "zstddict.h" +#include <stdbool.h> // bool #include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Decompression context */ + ZSTD_DCtx *dctx; + + /* ZstdDict object in use */ + PyObject *dict; + + /* Unconsumed input data */ + char *input_buffer; + size_t input_buffer_size; + size_t in_begin, in_end; + + /* Unused data */ + PyObject *unused_data; + + /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ + char needs_input; + + /* For decompress(), 0 or 1. + 1 when both input and output streams are at a frame edge, means a + frame is completely decoded and fully flushed, or the decompressor + just be initialized. */ + char at_frame_edge; + + /* For ZstdDecompressor, 0 or 1. + 1 means the end of the first frame has been reached. */ + char eof; + + /* Used for fast reset above three variables */ + char _unused_char_for_align; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdDecompressor; #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) +#include "clinic/decompressor.c.h" + static inline ZSTD_DDict * _get_DDict(ZstdDict *self) { @@ -215,17 +258,14 @@ load: return 0; } - +typedef enum { + TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class + TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function +} decompress_type; /* - Given the two types of decompressors (defined in _zstdmodule.h): - - typedef enum { - TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function - } decompress_type; - - Decompress implementation for <D>, <E>, pseudo code: + Given the two types of decompressors (defined above), + decompress implementation for <D>, <E>, pseudo code: initialize_output_buffer while True: @@ -616,7 +656,7 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - self->inited = 0; + self->initialized = 0; self->dict = NULL; self->input_buffer = NULL; self->input_buffer_size = 0; @@ -695,11 +735,11 @@ _zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, /*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/ { /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; /* Load dictionary to decompression context */ if (zstd_dict != Py_None) { @@ -802,10 +842,6 @@ _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, return ret; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/decompressor.c.h" -#undef clinic_state - static PyMethodDef ZstdDecompressor_methods[] = { _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF {NULL, NULL} diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 402fb65fb00..99976cef85a 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -7,17 +7,21 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type" +class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5d1254c497e52ba]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include "zstddict.h" +#include "clinic/zstddict.c.h" -#include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict() #define ZstdDict_CAST(op) ((ZstdDict *)op) @@ -31,7 +35,7 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U } self->dict_content = NULL; - self->inited = 0; + self->initialized = 0; self->d_dict = NULL; /* ZSTD_CDict dict */ @@ -92,11 +96,11 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, /*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/ { /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; /* Check dict_content's type */ self->dict_content = PyBytes_FromObject(dict_content); @@ -135,10 +139,6 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, return 0; } -#define clinic_state() (get_zstd_state(type)) -#include "clinic/zstddict.c.h" -#undef clinic_state - PyDoc_STRVAR(ZstdDict_dictid_doc, "ID of zstd dictionary, a 32-bit unsigned int value.\n\n" "Non-zero means ordinary dictionary, was created by zstd functions, follow\n" diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h new file mode 100644 index 00000000000..e82498f5dd1 --- /dev/null +++ b/Modules/_zstd/zstddict.h @@ -0,0 +1,31 @@ +/* +Low level interface to Meta's zstd library for use in the compression.zstd +Python module. +*/ + +#ifndef ZSTD_DICT_H +#define ZSTD_DICT_H + +#include <stdbool.h> // bool +#include <zstd.h> // ZSTD_DDict + +typedef struct { + PyObject_HEAD + + /* Reusable compress/decompress dictionary, they are created once and + can be shared by multiple threads concurrently, since its usage is + read-only. + c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ + ZSTD_DDict *d_dict; + PyObject *c_dicts; + + /* Content of the dictionary, bytes object. */ + PyObject *dict_content; + /* Dictionary id */ + uint32_t dict_id; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdDict; + +#endif // !ZSTD_DICT_H diff --git a/PCbuild/_zstd.vcxproj b/PCbuild/_zstd.vcxproj index b53f93a6af8..6f91b8d05cc 100644 --- a/PCbuild/_zstd.vcxproj +++ b/PCbuild/_zstd.vcxproj @@ -137,6 +137,7 @@ <ItemGroup> <ClInclude Include="..\Modules\_zstd\_zstdmodule.h" /> <ClInclude Include="..\Modules\_zstd\buffer.h" /> + <ClInclude Include="..\Modules\_zstd\zstddict.h" /> <ClInclude Include="$(zstdDir)lib\common\bitstream.h" /> <ClInclude Include="$(zstdDir)lib\common\error_private.h" /> <ClInclude Include="$(zstdDir)lib\common\fse.h" /> diff --git a/PCbuild/_zstd.vcxproj.filters b/PCbuild/_zstd.vcxproj.filters index d4d36063c85..eec666e5eaf 100644 --- a/PCbuild/_zstd.vcxproj.filters +++ b/PCbuild/_zstd.vcxproj.filters @@ -128,6 +128,9 @@ <ClInclude Include="..\Modules\_zstd\buffer.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\Modules\_zstd\zstddict.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="$(zstdDir)lib\zstd.h"> <Filter>Header Files\zstd</Filter> </ClInclude> |