global: adapt to zstd 1.5.7

All our code changes required to support the upgrade from 1.5.6.

this changed compression results, so we have to adapt tests

Author: bmwiedemann

c-ext/backend_c.c

@@ -152,7 +152,7 @@ void zstd_module_init(PyObject *m) {
     PyObject *features = NULL;
     PyObject *feature = NULL;
     unsigned zstd_ver_no = ZSTD_versionNumber();
-    unsigned our_hardcoded_version = 10506;
+    unsigned our_hardcoded_version = 10507;
     if (ZSTD_VERSION_NUMBER != our_hardcoded_version ||
         zstd_ver_no != our_hardcoded_version) {
         PyErr_Format(

make_cffi.py

@@ -179,7 +179,7 @@ def normalize_output(output):
     include_dirs=INCLUDE_DIRS,
 )
 
-DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ")
+DEFINE = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)")
 
 sources = []
 
@@ -204,9 +204,14 @@ def normalize_output(output):
             if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
                 continue
 
+            # These defines create aliases from old (camelCase) type names
+            # to the new PascalCase names, which breaks CFFI.
+            if m.group(1).lower() == m.group(2).lower():
+                continue
+
             # The ... is magic syntax by the cdef parser to resolve the
             # value at compile time.
-            sources.append(m.group(0) + b" ...")
+            sources.append(b"#define " + m.group(1) + b" ...")
 
 cdeflines = b"\n".join(sources).splitlines()
 cdeflines = [line for line in cdeflines if line.strip()]

tests/test_compressor_compress.py

@@ -52,7 +52,7 @@ def test_compress_large(self):
 
         cctx = zstd.ZstdCompressor(level=3, write_content_size=False)
         result = cctx.compress(b"".join(chunks))
-        self.assertEqual(len(result), 999)
+        self.assertEqual(len(result), 1029)
         self.assertEqual(result[0:4], b"\x28\xb5\x2f\xfd")
 
         # This matches the test for read_to_iter() below.

tests/test_compressor_compressobj.py

@@ -39,7 +39,7 @@ def test_compressobj_large(self):
         cobj = cctx.compressobj()
 
         result = cobj.compress(b"".join(chunks)) + cobj.flush()
-        self.assertEqual(len(result), 999)
+        self.assertEqual(len(result), 1029)
         self.assertEqual(result[0:4], b"\x28\xb5\x2f\xfd")
 
         params = zstd.get_frame_parameters(result)

tests/test_compressor_copy_stream.py

@@ -50,7 +50,7 @@ def test_large_data(self):
         r, w = cctx.copy_stream(source, dest)
 
         self.assertEqual(r, 255 * 16384)
-        self.assertEqual(w, 999)
+        self.assertEqual(w, 1029)
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)

tests/test_compressor_stream_writer.py

@@ -301,7 +301,7 @@ def test_dictionary(self):
         d = zstd.train_dictionary(8192, samples)
 
         h = hashlib.sha1(d.as_bytes()).hexdigest()
-        self.assertEqual(h, "a46d2f7a3bc3357c9d717d3dadf9a26fde23e93d")
+        self.assertEqual(h, "f32ddfbe0878bbd428afc00b17810387c6752191")
 
         buffer = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=9, dict_data=d)

tests/test_module_attributes.py

@@ -5,7 +5,7 @@
 
 class TestModuleAttributes(unittest.TestCase):
     def test_version(self):
-        self.assertEqual(zstd.ZSTD_VERSION, (1, 5, 6))
+        self.assertEqual(zstd.ZSTD_VERSION, (1, 5, 7))
 
         self.assertEqual(zstd.__version__, "0.24.0.dev0")
 

zstandard/backend_cffi.py

@@ -2573,7 +2573,7 @@ def get_frame_parameters(data):
     :return:
        :py:class:`FrameParameters`
     """
-    params = ffi.new("ZSTD_frameHeader *")
+    params = ffi.new("ZSTD_FrameHeader *")
 
     data_buffer = ffi.from_buffer(data)
     zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer))
@@ -4288,7 +4288,7 @@ def decompress_content_dict_chain(self, frames):
 
         # All chunks should be zstd frames and should have content size set.
         chunk_buffer = ffi.from_buffer(chunk)
-        params = ffi.new("ZSTD_frameHeader *")
+        params = ffi.new("ZSTD_FrameHeader *")
         zresult = lib.ZSTD_getFrameHeader(
             params, chunk_buffer, len(chunk_buffer)
         )