global: adapt to zstd 1.5.7

All our code changes required to support the upgrade from 1.5.6.

* Rust zstd upgraded to 1.5.7 to match C version.
* Test output changes.
* `zstd_errors.h` inclusion via `zstd.h` caused problems.
* MUSL 1.1 compatibility issues (see previous commit).

Author: bmwiedemann

Cargo.lock

@@ -152,7 +152,7 @@ void zstd_module_init(PyObject *m) {
     PyObject *features = NULL;
     PyObject *feature = NULL;
     unsigned zstd_ver_no = ZSTD_versionNumber();
-    unsigned our_hardcoded_version = 10506;
+    unsigned our_hardcoded_version = 10507;
     if (ZSTD_VERSION_NUMBER != our_hardcoded_version ||
         zstd_ver_no != our_hardcoded_version) {
         PyErr_Format(

c-ext/backend_c.c

@@ -14,6 +14,7 @@
 import tempfile
 
 import cffi
+import packaging.tags
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
@@ -22,7 +23,7 @@
 ]
 
 # Headers whose preprocessed output will be fed into cdef().
-HEADERS = [os.path.join(HERE, "zstd", p) for p in ("zstd.h", "zdict.h")]
+HEADERS = [os.path.join(HERE, "zstd", p) for p in ("zstd_errors.h", "zstd.h", "zdict.h")]
 
 INCLUDE_DIRS = [
     os.path.join(HERE, "zstd"),
@@ -76,8 +77,8 @@ def preprocess(path):
             # boilerplate. This can lead to duplicate declarations. So we strip
             # this include from the preprocessor invocation.
             #
-            # The same things happens for including zstd.h, so give it the same
-            # treatment.
+            # The same things happens for including zstd.h and zstd_errors.h, so
+            # give them the same treatment.
             #
             # We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline
             # #define in zstdmt_compress.h and results in a compiler warning. So drop
@@ -86,11 +87,17 @@ def preprocess(path):
                 (
                     b"#include <stddef.h>",
                     b'#include "zstd.h"',
+                    b'#include "zstd_errors.h"',
                     b"#define ZSTD_STATIC_LINKING_ONLY",
                 )
             ):
                 continue
 
+            # There's a naked `static` before the declaration of ZSTD_customMem that
+            # confuses the cffi parser. Strip it.
+            if line == b'static\n':
+                continue
+
             # The preprocessor environment on Windows doesn't define include
             # paths, so the #include of limits.h fails. We work around this
             # by removing that import and defining INT_MAX ourselves. This is
@@ -157,19 +164,25 @@ def normalize_output(output):
 
     return b"\n".join(lines)
 
+# musl 1.1 doesn't define qsort_r. We need to force using the C90
+# variant.
+musl_defines = ""
+for tag in packaging.tags.platform_tags():
+    if tag.startswith("musllinux_1_1_"):
+        musl_defines = "#define ZDICT_QSORT ZDICT_QSORT_C90"
+
 
 ffi = cffi.FFI()
-# zstd.h uses a possible undefined MIN(). Define it until
-# https://github.com/facebook/zstd/issues/976 is fixed.
 # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
 # when cffi uses the function. Since we statically link against zstd, even
 # if we use the deprecated functions it shouldn't be a huge problem.
 ffi.set_source(
     "zstandard._cffi",
-    """
-#define MIN(a,b) ((a)<(b) ? (a) : (b))
+    f"""
 #define ZSTD_STATIC_LINKING_ONLY
 #define ZSTD_DISABLE_DEPRECATE_WARNINGS
+{musl_defines}
+#include <zstd_errors.h>
 #include <zstd.h>
 #define ZDICT_STATIC_LINKING_ONLY
 #define ZDICT_DISABLE_DEPRECATE_WARNINGS
@@ -179,7 +192,7 @@ def normalize_output(output):
     include_dirs=INCLUDE_DIRS,
 )
 
-DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ")
+DEFINE = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)")
 
 sources = []
 
@@ -204,9 +217,14 @@ def normalize_output(output):
             if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
                 continue
 
+            # These defines create aliases from old (camelCase) type names
+            # to the new PascalCase names, which breaks CFFI.
+            if m.group(1).lower() == m.group(2).lower():
+                continue
+
             # The ... is magic syntax by the cdef parser to resolve the
             # value at compile time.
-            sources.append(m.group(0) + b" ...")
+            sources.append(b"#define " + m.group(1) + b" ...")
 
 cdeflines = b"\n".join(sources).splitlines()
 cdeflines = [line for line in cdeflines if line.strip()]

make_cffi.py

@@ -28,6 +28,7 @@ Documentation = "https://python-zstandard.readthedocs.io/en/latest/"
 [build-system]
 requires = [
     "cffi>=1.17.0",
+    "packaging",
     "setuptools",
 ]
 # Need to use legacy backend because setup_zstd.py breaks build isolation.

pyproject.toml

@@ -256,11 +256,11 @@ impl ZstdDecompressor {
         }
 
         let chunk_buffer: PyBuffer<u8> = PyBuffer::get(&chunk.as_borrowed())?;
-        let mut params = zstd_sys::ZSTD_frameHeader {
+        let mut params = zstd_sys::ZSTD_FrameHeader {
             frameContentSize: 0,
             windowSize: 0,
             blockSizeMax: 0,
-            frameType: zstd_sys::ZSTD_frameType_e::ZSTD_frame,
+            frameType: zstd_sys::ZSTD_FrameType_e::ZSTD_frame,
             headerSize: 0,
             dictID: 0,
             checksumFlag: 0,

rust-ext/src/decompressor.rs

@@ -12,7 +12,7 @@ use {
 
 #[pyclass(module = "zstandard.backend_rust")]
 struct FrameParameters {
-    header: zstd_sys::ZSTD_frameHeader,
+    header: zstd_sys::ZSTD_FrameHeader,
 }
 
 unsafe impl Sync for FrameParameters {}
@@ -75,11 +75,11 @@ fn get_frame_parameters(py: Python, buffer: PyBuffer<u8>) -> PyResult<Py<FramePa
         std::slice::from_raw_parts::<u8>(buffer.buf_ptr() as *const _, buffer.len_bytes())
     };
 
-    let mut header = zstd_sys::ZSTD_frameHeader {
+    let mut header = zstd_sys::ZSTD_FrameHeader {
         frameContentSize: 0,
         windowSize: 0,
         blockSizeMax: 0,
-        frameType: zstd_sys::ZSTD_frameType_e::ZSTD_frame,
+        frameType: zstd_sys::ZSTD_FrameType_e::ZSTD_frame,
         headerSize: 0,
         dictID: 0,
         checksumFlag: 0,

rust-ext/src/frame_parameters.rs

@@ -14,6 +14,8 @@
 import subprocess
 import sys
 
+import packaging.tags
+
 ext_includes = [
     "c-ext",
 ]
@@ -89,6 +91,13 @@ def get_c_extension(
     if not system_zstd and support_legacy:
         extra_args.append("-DZSTD_LEGACY_SUPPORT=1")
 
+    # musl 1.1 doesn't define qsort_r. We need to force using the C90
+    # variant. ZDICT_QSORT officially introduced in 1.5.8. But our
+    # vendored copy backported to 1.5.7.
+    for tag in packaging.tags.platform_tags():
+        if tag.startswith("musllinux_1_1_"):
+            extra_args.append("-DZDICT_QSORT=ZDICT_QSORT_C90")
+
     if warnings_as_errors:
         if compiler_type in ("unix", "mingw32"):
             extra_args.append("-Werror")

setup_zstd.py

@@ -52,7 +52,7 @@ def test_compress_large(self):
 
         cctx = zstd.ZstdCompressor(level=3, write_content_size=False)
         result = cctx.compress(b"".join(chunks))
-        self.assertEqual(len(result), 999)
+        self.assertEqual(len(result), 1029)
         self.assertEqual(result[0:4], b"\x28\xb5\x2f\xfd")
 
         # This matches the test for read_to_iter() below.

tests/test_compressor_compress.py

@@ -39,7 +39,7 @@ def test_compressobj_large(self):
         cobj = cctx.compressobj()
 
         result = cobj.compress(b"".join(chunks)) + cobj.flush()
-        self.assertEqual(len(result), 999)
+        self.assertEqual(len(result), 1029)
         self.assertEqual(result[0:4], b"\x28\xb5\x2f\xfd")
 
         params = zstd.get_frame_parameters(result)

tests/test_compressor_compressobj.py

@@ -50,7 +50,7 @@ def test_large_data(self):
         r, w = cctx.copy_stream(source, dest)
 
         self.assertEqual(r, 255 * 16384)
-        self.assertEqual(w, 999)
+        self.assertEqual(w, 1029)
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)