From a4e5eeca895d43f08d0cb31cd57cce322a044b46 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 27 Jun 2026 16:51:57 +0200 Subject: [PATCH 1/2] feat: prefix dictionaries on contexts (refPrefix) for delta compression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bind ZSTD_CCtx_refPrefix / ZSTD_DCtx_refPrefix as ZstdCompressCtx.refPrefix(MemorySegment) / ZstdDecompressCtx.refPrefix(...): a single-use, by-reference raw-content dictionary for the next frame only — the building block for delta compression (compress a new version against a similar previous one). No digest, no copy, no dictionary ID written; the decompressor must reference the same prefix to decode. Segment-only by design. refPrefix references the prefix by pointer (unlike loadDictionary, which copies internally), so the prefix must outlive the next compression — a contract only a caller-owned native segment can honour. A byte[] overload would have to copy, defeating the zero-copy point and forcing a context-held arena to keep the copy alive; heap callers should use the copying loadDictionary instead. RefPrefixTest proves the prefix is actually applied (16 KiB of random data, identical to the prefix, compresses to tens of bytes at level 19 only with the prefix) and is required to decode — a round-trip-only test would pass even if refPrefix were a no-op. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 12 ++ docs/supported.md | 8 +- .../java/io/github/dfa1/zstd/Bindings.java | 9 ++ .../io/github/dfa1/zstd/ZstdCompressCtx.java | 32 ++++ .../github/dfa1/zstd/ZstdDecompressCtx.java | 30 ++++ .../io/github/dfa1/zstd/RefPrefixTest.java | 148 ++++++++++++++++++ 6 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e020cc..aeb49ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this project are documented here. Format loosely follows [Keep a Changelog](https://keepachangelog.com/); versions are released as `v*` git tags, which trigger publication to Maven Central. +## [Unreleased] + +### Added +- `ZstdCompressCtx.refPrefix(MemorySegment)` / `ZstdDecompressCtx.refPrefix(...)` + — reference native content as a single-use prefix (raw-content dictionary) for + the next frame only: the building block for delta compression (compress a new + version against a similar previous one). The prefix is referenced, not copied + or digested, and writes no dictionary ID; the decompressor must reference the + same prefix to decode. Binds `ZSTD_CCtx_refPrefix` / `ZSTD_DCtx_refPrefix`. + Segment-only by design — heap callers that need a copy should use + `loadDictionary` instead. + ## [0.5] ### Added diff --git a/docs/supported.md b/docs/supported.md index f548f15..b9058de 100644 --- a/docs/supported.md +++ b/docs/supported.md @@ -33,7 +33,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`. | Dictionary training (ZDICT) | 8 / 12 | trainFromBuffer, cover/fastCover optimizers, finalizeDictionary, getDictHeaderSize | | Streaming — compress | 3 / 22 | `ZstdOutputStream` (compressStream2 + buffer sizes) | | Streaming — decompress | 3 / 15 | `ZstdInputStream` (decompressStream + buffer sizes) | -| Advanced parameters | 12 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `c/dParam_getBounds`; MT inert on single-thread build | +| Advanced parameters | 14 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `C/DCtx_refPrefix`, `c/dParam_getBounds`; MT inert on single-thread build | | Frame inspection | 10 / 13 | `ZstdFrame` + getFrameProgression; `_advanced` not bound | | Memory sizing | 8 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size | | Low-level block | 0 / 12 | expert block/continue API not bound | @@ -233,7 +233,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa | `ZSTD_resetDStream` | — ᵈ | — | | `ZSTD_sizeof_DStream` | — | — | -### Advanced parameters (12/38) +### Advanced parameters (14/38) | Symbol | Bound | zstd-jni | |---|:---:|:---:| @@ -248,7 +248,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa | `ZSTD_CCtx_loadDictionary_advanced` | — | — | | `ZSTD_CCtx_loadDictionary_byReference` | — | — | | `ZSTD_CCtx_refCDict` | ✅ | ✅ | -| `ZSTD_CCtx_refPrefix` | — | — | +| `ZSTD_CCtx_refPrefix` | ✅ | — | | `ZSTD_CCtx_refPrefix_advanced` | — | — | | `ZSTD_CCtx_refThreadPool` | — | — | | `ZSTD_CCtx_reset` | ✅ | ✅ | @@ -263,7 +263,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa | `ZSTD_DCtx_loadDictionary_advanced` | — | — | | `ZSTD_DCtx_loadDictionary_byReference` | — | — | | `ZSTD_DCtx_refDDict` | ✅ | ✅ | -| `ZSTD_DCtx_refPrefix` | — | — | +| `ZSTD_DCtx_refPrefix` | ✅ | — | | `ZSTD_DCtx_refPrefix_advanced` | — | — | | `ZSTD_DCtx_reset` | ✅ | ✅ | | `ZSTD_DCtx_setFormat` | — ᵈ | — | diff --git a/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java b/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java index 268d033..8275dae 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java @@ -210,6 +210,15 @@ final class Bindings { NativeLibrary.lookup("ZSTD_DCtx_loadDictionary", FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG)); + // size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx*, const void* prefix, size_t prefixSize) + static final MethodHandle CCTX_REF_PREFIX = + NativeLibrary.lookup("ZSTD_CCtx_refPrefix", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG)); + // size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx*, const void* prefix, size_t prefixSize) + static final MethodHandle DCTX_REF_PREFIX = + NativeLibrary.lookup("ZSTD_DCtx_refPrefix", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG)); + // size_t ZSTD_CStreamInSize(void) / ZSTD_CStreamOutSize(void) static final MethodHandle CSTREAM_IN_SIZE = NativeLibrary.lookup("ZSTD_CStreamInSize", FunctionDescriptor.of(JAVA_LONG)); diff --git a/zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java b/zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java index e1ab1ea..5848dce 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java @@ -177,6 +177,38 @@ public ZstdCompressCtx refDictionary(ZstdCompressDict dict) { return this; } + /// References native `prefix` content as a single-use dictionary for the + /// **next** compression only — the building block for delta compression: + /// compress a new version against a similar previous one as the prefix, + /// storing little more than the difference. + /// + /// Unlike [#loadDictionary(ZstdDictionary)], a prefix is referenced (no + /// digest, no heap copy), writes no dictionary ID into the frame, and is + /// consumed by the next [#compress(MemorySegment, MemorySegment)] / + /// [#compress(byte[])] — it does not stick across frames. The decompressor + /// must reference the **same** prefix with + /// [ZstdDecompressCtx#refPrefix(MemorySegment)] to decode the frame. + /// + /// Because the prefix is referenced, `prefix` must stay valid until the next + /// compression completes. Heap callers that cannot manage native lifetime + /// should use a copying dictionary ([#loadDictionary(ZstdDictionary)]) instead. + /// + /// @param prefix native prefix content, or `null` / [MemorySegment#NULL] to clear it + /// @return `this`, for chaining + /// @throws ZstdException if the prefix cannot be referenced + public ZstdCompressCtx refPrefix(MemorySegment prefix) { + if (NativeCall.isNull(prefix)) { + return refPrefix(MemorySegment.NULL, 0L); + } + NativeCall.requireNative(prefix, "prefix"); + return refPrefix(prefix, prefix.byteSize()); + } + + private ZstdCompressCtx refPrefix(MemorySegment prefix, long size) { + NativeCall.checkReturnValue(() -> (long) Bindings.CCTX_REF_PREFIX.invokeExact(ptr(), prefix, size)); + return this; + } + /// Compresses `src` into a new zstd frame using this context and its /// advanced parameters. /// diff --git a/zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java b/zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java index 814c4b1..0e39962 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java @@ -126,6 +126,36 @@ public ZstdDecompressCtx refDictionary(ZstdDecompressDict dict) { return this; } + /// References native `prefix` content as a single-use dictionary for decoding + /// the **next** frame only — the decompression counterpart of + /// [ZstdCompressCtx#refPrefix(MemorySegment)]. It must be the **same** content + /// the frame was compressed against, or decoding fails. + /// + /// The prefix is referenced, not copied (no digest, no heap copy): `prefix` + /// must stay valid until the next [#decompress(byte[], int)] / + /// [#decompress(MemorySegment, MemorySegment)], which consumes it — it does not + /// stick across frames. Pass `null` / [MemorySegment#NULL] to clear a prefix + /// set but not yet consumed. + /// + /// Heap callers that cannot manage native lifetime should use a copying + /// dictionary ([#loadDictionary(ZstdDictionary)]) instead. + /// + /// @param prefix native prefix content, or `null` / [MemorySegment#NULL] to clear it + /// @return `this`, for chaining + /// @throws ZstdException if the prefix cannot be referenced + public ZstdDecompressCtx refPrefix(MemorySegment prefix) { + if (NativeCall.isNull(prefix)) { + return refPrefix(MemorySegment.NULL, 0L); + } + NativeCall.requireNative(prefix, "prefix"); + return refPrefix(prefix, prefix.byteSize()); + } + + private ZstdDecompressCtx refPrefix(MemorySegment prefix, long size) { + NativeCall.checkReturnValue(() -> (long) Bindings.DCTX_REF_PREFIX.invokeExact(ptr(), prefix, size)); + return this; + } + /// Decompresses a frame into a buffer of at most `maxSize` bytes. /// /// @param compressed a complete zstd frame diff --git a/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java b/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java new file mode 100644 index 0000000..7acacab --- /dev/null +++ b/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java @@ -0,0 +1,148 @@ +package io.github.dfa1.zstd; + +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.util.Random; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.Test; + +class RefPrefixTest { + + @Test + void roundTripsWithMatchingPrefix() { + // Given + byte[] prefixBytes = "the quick brown fox jumps over the lazy dog".getBytes(); + byte[] dataBytes = "the quick brown fox jumps over the lazy cat".getBytes(); + try (Arena arena = Arena.ofConfined(); + ZstdCompressCtx cctx = new ZstdCompressCtx(); + ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment prefix = copy(arena, prefixBytes); + MemorySegment src = copy(arena, dataBytes); + MemorySegment frame = arena.allocate(Zstd.compressBound(dataBytes.length)); + MemorySegment out = arena.allocate(dataBytes.length); + + // When + cctx.refPrefix(prefix); + long n = cctx.compress(frame, src); + dctx.refPrefix(prefix); + long m = dctx.decompress(out, frame.asSlice(0, n)); + + // Then + assertThat(m).isEqualTo(dataBytes.length); + assertThat(bytes(out, (int) m)).isEqualTo(dataBytes); + } + } + + @Test + void prefixIsAppliedAndRequiredToDecode() { + // Given — incompressible (random) data identical to the prefix. With no + // internal redundancy the encoder can only shrink it by matching against + // the prefix, so a much smaller frame proves the prefix is applied. + // (Level 19: the level-3 match finder is too weak to reach into the prefix + // for random content.) + byte[] random = randomBytes(0xBEEF, 16384); + try (Arena arena = Arena.ofConfined()) { + MemorySegment prefix = copy(arena, random); + MemorySegment src = copy(arena, random); + + long baseline; + try (ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { + baseline = cctx.compress(arena, src).byteSize(); + } + byte[] frame; + try (ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { + cctx.refPrefix(prefix); + MemorySegment f = cctx.compress(arena, src); + frame = bytes(f, (int) f.byteSize()); + } + + // Then — the prefix slashes the frame (16 KiB random → tens of bytes) + assertThat((long) frame.length).isLessThan(baseline / 10); + + // And — it round-trips with the same prefix + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment out = arena.allocate(random.length); + dctx.refPrefix(prefix); + long m = dctx.decompress(out, copy(arena, frame)); + assertThat(bytes(out, (int) m)).isEqualTo(random); + } + + // But — it cannot be recovered without the prefix + boolean reproduced; + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment out = arena.allocate(random.length); + long m = dctx.decompress(out, copy(arena, frame)); + reproduced = java.util.Arrays.equals(bytes(out, (int) m), random); + } catch (ZstdException e) { + reproduced = false; + } + assertThat(reproduced).isFalse(); + } + } + + @Test + void clearingPrefixWithNullCompressesPlainly() { + // Given + byte[] dataBytes = "the quick brown fox".getBytes(); + try (Arena arena = Arena.ofConfined(); + ZstdCompressCtx cctx = new ZstdCompressCtx()) { + MemorySegment prefix = copy(arena, "a prior version of the text".getBytes()); + MemorySegment src = copy(arena, dataBytes); + MemorySegment frame = arena.allocate(Zstd.compressBound(dataBytes.length)); + + // When — set then clear the prefix before compressing + cctx.refPrefix(prefix); + cctx.refPrefix((MemorySegment) null); + long n = cctx.compress(frame, src); + + // Then — a plain decoder with no prefix decodes it + byte[] restored; + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment out = arena.allocate(dataBytes.length); + long m = dctx.decompress(out, frame.asSlice(0, n)); + restored = bytes(out, (int) m); + } + assertThat(restored).isEqualTo(dataBytes); + } + } + + @Test + void refPrefixRejectsAHeapSegment() { + // Given + MemorySegment heap = MemorySegment.ofArray("prefix".getBytes()); + + // When + ThrowingCallable result = () -> { + try (ZstdCompressCtx cctx = new ZstdCompressCtx()) { + cctx.refPrefix(heap); + } + }; + + // Then + assertThatThrownBy(result) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("prefix"); + } + + private static MemorySegment copy(Arena arena, byte[] src) { + MemorySegment seg = arena.allocate(src.length); + MemorySegment.copy(src, 0, seg, JAVA_BYTE, 0, src.length); + return seg; + } + + private static byte[] bytes(MemorySegment seg, int len) { + byte[] out = new byte[len]; + MemorySegment.copy(seg, JAVA_BYTE, 0, out, 0, len); + return out; + } + + private static byte[] randomBytes(long seed, int n) { + byte[] b = new byte[n]; + new Random(seed).nextBytes(b); + return b; + } +} From fbc59025e7da824da61fc53b9b3277ce4c3069b7 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 27 Jun 2026 16:59:49 +0200 Subject: [PATCH 2/2] test: close refPrefix coverage gaps - prove refPrefix is single-use: prefix set once, two compressions, only the first frame shrinks (random data at level 19); second frame decodes with a plain decoder, confirming it does not stick across frames. - assert ZstdDecompressCtx.refPrefix also rejects a heap segment (was only covered on the compress side). - import java.util.Arrays instead of the fully-qualified inline reference. Co-Authored-By: Claude Opus 4.8 --- .../io/github/dfa1/zstd/RefPrefixTest.java | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java b/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java index 7acacab..9cf583f 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java @@ -6,6 +6,7 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.util.Arrays; import java.util.Random; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.Test; @@ -76,7 +77,7 @@ void prefixIsAppliedAndRequiredToDecode() { try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { MemorySegment out = arena.allocate(random.length); long m = dctx.decompress(out, copy(arena, frame)); - reproduced = java.util.Arrays.equals(bytes(out, (int) m), random); + reproduced = Arrays.equals(bytes(out, (int) m), random); } catch (ZstdException e) { reproduced = false; } @@ -111,7 +112,39 @@ void clearingPrefixWithNullCompressesPlainly() { } @Test - void refPrefixRejectsAHeapSegment() { + void prefixIsSingleUseAndDoesNotStickAcrossFrames() { + // Given — random data identical to the prefix (see prefixIsAppliedAndRequiredToDecode), + // and one context kept open across two compressions with the prefix set once. + byte[] random = randomBytes(0xCAFE, 16384); + try (Arena arena = Arena.ofConfined(); + ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { + MemorySegment prefix = copy(arena, random); + MemorySegment src = copy(arena, random); + MemorySegment first = arena.allocate(Zstd.compressBound(random.length)); + MemorySegment second = arena.allocate(Zstd.compressBound(random.length)); + + // When — first frame consumes the prefix; second is compressed with no re-set + cctx.refPrefix(prefix); + long n1 = cctx.compress(first, src); + long n2 = cctx.compress(second, src); + + // Then — the prefix shrank only the first frame; the second got no prefix + // (incompressible random with no prefix stays ~full size) + assertThat(n1).isLessThan(n2 / 10); + + // And — the second frame carries no prefix, so a plain decoder decodes it + byte[] restored; + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment out = arena.allocate(random.length); + long m = dctx.decompress(out, second.asSlice(0, n2)); + restored = bytes(out, (int) m); + } + assertThat(restored).isEqualTo(random); + } + } + + @Test + void compressRefPrefixRejectsAHeapSegment() { // Given MemorySegment heap = MemorySegment.ofArray("prefix".getBytes()); @@ -128,6 +161,24 @@ void refPrefixRejectsAHeapSegment() { .hasMessageContaining("prefix"); } + @Test + void decompressRefPrefixRejectsAHeapSegment() { + // Given + MemorySegment heap = MemorySegment.ofArray("prefix".getBytes()); + + // When + ThrowingCallable result = () -> { + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + dctx.refPrefix(heap); + } + }; + + // Then + assertThatThrownBy(result) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("prefix"); + } + private static MemorySegment copy(Arena arena, byte[] src) { MemorySegment seg = arena.allocate(src.length); MemorySegment.copy(src, 0, seg, JAVA_BYTE, 0, src.length);