feat: prefix dictionaries (refPrefix) for delta compression

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project are documented here. Format loosely follows
 [Keep a Changelog](https://keepachangelog.com/); versions are released as `v*`
 git tags, which trigger publication to Maven Central.
 
+## [Unreleased]
+
+### Added
+- `ZstdCompressCtx.refPrefix(MemorySegment)` / `ZstdDecompressCtx.refPrefix(...)`
+  — reference native content as a single-use prefix (raw-content dictionary) for
+  the next frame only: the building block for delta compression (compress a new
+  version against a similar previous one). The prefix is referenced, not copied
+  or digested, and writes no dictionary ID; the decompressor must reference the
+  same prefix to decode. Binds `ZSTD_CCtx_refPrefix` / `ZSTD_DCtx_refPrefix`.
+  Segment-only by design — heap callers that need a copy should use
+  `loadDictionary` instead.
+
 ## [0.5]
 
 ### Added

docs/supported.md

@@ -33,7 +33,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`.
 | Dictionary training (ZDICT) | 8 / 12 | trainFromBuffer, cover/fastCover optimizers, finalizeDictionary, getDictHeaderSize |
 | Streaming — compress | 3 / 22 | `ZstdOutputStream` (compressStream2 + buffer sizes) |
 | Streaming — decompress | 3 / 15 | `ZstdInputStream` (decompressStream + buffer sizes) |
-| Advanced parameters | 12 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `c/dParam_getBounds`; MT inert on single-thread build |
+| Advanced parameters | 14 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `C/DCtx_refPrefix`, `c/dParam_getBounds`; MT inert on single-thread build |
 | Frame inspection | 10 / 13 | `ZstdFrame` + getFrameProgression; `_advanced` not bound |
 | Memory sizing | 8 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size |
 | Low-level block | 0 / 12 | expert block/continue API not bound |
@@ -233,7 +233,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa
 | `ZSTD_resetDStream` | — ᵈ | — |
 | `ZSTD_sizeof_DStream` | — | — |
 
-### Advanced parameters (12/38)
+### Advanced parameters (14/38)
 
 | Symbol | Bound | zstd-jni |
 |---|:---:|:---:|
@@ -248,7 +248,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa
 | `ZSTD_CCtx_loadDictionary_advanced` | — | — |
 | `ZSTD_CCtx_loadDictionary_byReference` | — | — |
 | `ZSTD_CCtx_refCDict` | ✅ | ✅ |
-| `ZSTD_CCtx_refPrefix` | — | — |
+| `ZSTD_CCtx_refPrefix` | ✅ | — |
 | `ZSTD_CCtx_refPrefix_advanced` | — | — |
 | `ZSTD_CCtx_refThreadPool` | — | — |
 | `ZSTD_CCtx_reset` | ✅ | ✅ |
@@ -263,7 +263,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa
 | `ZSTD_DCtx_loadDictionary_advanced` | — | — |
 | `ZSTD_DCtx_loadDictionary_byReference` | — | — |
 | `ZSTD_DCtx_refDDict` | ✅ | ✅ |
-| `ZSTD_DCtx_refPrefix` | — | — |
+| `ZSTD_DCtx_refPrefix` | ✅ | — |
 | `ZSTD_DCtx_refPrefix_advanced` | — | — |
 | `ZSTD_DCtx_reset` | ✅ | ✅ |
 | `ZSTD_DCtx_setFormat` | — ᵈ | — |

zstd/src/main/java/io/github/dfa1/zstd/Bindings.java

@@ -210,6 +210,15 @@ final class Bindings {
             NativeLibrary.lookup("ZSTD_DCtx_loadDictionary",
                     FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG));
 
+    // size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx*, const void* prefix, size_t prefixSize)
+    static final MethodHandle CCTX_REF_PREFIX =
+            NativeLibrary.lookup("ZSTD_CCtx_refPrefix",
+                    FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG));
+    // size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx*, const void* prefix, size_t prefixSize)
+    static final MethodHandle DCTX_REF_PREFIX =
+            NativeLibrary.lookup("ZSTD_DCtx_refPrefix",
+                    FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG));
+
     // size_t ZSTD_CStreamInSize(void) / ZSTD_CStreamOutSize(void)
     static final MethodHandle CSTREAM_IN_SIZE =
             NativeLibrary.lookup("ZSTD_CStreamInSize", FunctionDescriptor.of(JAVA_LONG));

zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java

@@ -177,6 +177,38 @@ public ZstdCompressCtx refDictionary(ZstdCompressDict dict) {
         return this;
     }
 
+    /// References native `prefix` content as a single-use dictionary for the
+    /// **next** compression only — the building block for delta compression:
+    /// compress a new version against a similar previous one as the prefix,
+    /// storing little more than the difference.
+    ///
+    /// Unlike [#loadDictionary(ZstdDictionary)], a prefix is referenced (no
+    /// digest, no heap copy), writes no dictionary ID into the frame, and is
+    /// consumed by the next [#compress(MemorySegment, MemorySegment)] /
+    /// [#compress(byte[])] — it does not stick across frames. The decompressor
+    /// must reference the **same** prefix with
+    /// [ZstdDecompressCtx#refPrefix(MemorySegment)] to decode the frame.
+    ///
+    /// Because the prefix is referenced, `prefix` must stay valid until the next
+    /// compression completes. Heap callers that cannot manage native lifetime
+    /// should use a copying dictionary ([#loadDictionary(ZstdDictionary)]) instead.
+    ///
+    /// @param prefix native prefix content, or `null` / [MemorySegment#NULL] to clear it
+    /// @return `this`, for chaining
+    /// @throws ZstdException if the prefix cannot be referenced
+    public ZstdCompressCtx refPrefix(MemorySegment prefix) {
+        if (NativeCall.isNull(prefix)) {
+            return refPrefix(MemorySegment.NULL, 0L);
+        }
+        NativeCall.requireNative(prefix, "prefix");
+        return refPrefix(prefix, prefix.byteSize());
+    }
+
+    private ZstdCompressCtx refPrefix(MemorySegment prefix, long size) {
+        NativeCall.checkReturnValue(() -> (long) Bindings.CCTX_REF_PREFIX.invokeExact(ptr(), prefix, size));
+        return this;
+    }
+
     /// Compresses `src` into a new zstd frame using this context and its
     /// advanced parameters.
     ///

zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java

@@ -126,6 +126,36 @@ public ZstdDecompressCtx refDictionary(ZstdDecompressDict dict) {
         return this;
     }
 
+    /// References native `prefix` content as a single-use dictionary for decoding
+    /// the **next** frame only — the decompression counterpart of
+    /// [ZstdCompressCtx#refPrefix(MemorySegment)]. It must be the **same** content
+    /// the frame was compressed against, or decoding fails.
+    ///
+    /// The prefix is referenced, not copied (no digest, no heap copy): `prefix`
+    /// must stay valid until the next [#decompress(byte[], int)] /
+    /// [#decompress(MemorySegment, MemorySegment)], which consumes it — it does not
+    /// stick across frames. Pass `null` / [MemorySegment#NULL] to clear a prefix
+    /// set but not yet consumed.
+    ///
+    /// Heap callers that cannot manage native lifetime should use a copying
+    /// dictionary ([#loadDictionary(ZstdDictionary)]) instead.
+    ///
+    /// @param prefix native prefix content, or `null` / [MemorySegment#NULL] to clear it
+    /// @return `this`, for chaining
+    /// @throws ZstdException if the prefix cannot be referenced
+    public ZstdDecompressCtx refPrefix(MemorySegment prefix) {
+        if (NativeCall.isNull(prefix)) {
+            return refPrefix(MemorySegment.NULL, 0L);
+        }
+        NativeCall.requireNative(prefix, "prefix");
+        return refPrefix(prefix, prefix.byteSize());
+    }
+
+    private ZstdDecompressCtx refPrefix(MemorySegment prefix, long size) {
+        NativeCall.checkReturnValue(() -> (long) Bindings.DCTX_REF_PREFIX.invokeExact(ptr(), prefix, size));
+        return this;
+    }
+
     /// Decompresses a frame into a buffer of at most `maxSize` bytes.
     ///
     /// @param compressed a complete zstd frame

zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java

@@ -0,0 +1,199 @@
+package io.github.dfa1.zstd;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.util.Arrays;
+import java.util.Random;
+import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
+import org.junit.jupiter.api.Test;
+
+class RefPrefixTest {
+
+    @Test
+    void roundTripsWithMatchingPrefix() {
+        // Given
+        byte[] prefixBytes = "the quick brown fox jumps over the lazy dog".getBytes();
+        byte[] dataBytes = "the quick brown fox jumps over the lazy cat".getBytes();
+        try (Arena arena = Arena.ofConfined();
+             ZstdCompressCtx cctx = new ZstdCompressCtx();
+             ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {
+            MemorySegment prefix = copy(arena, prefixBytes);
+            MemorySegment src = copy(arena, dataBytes);
+            MemorySegment frame = arena.allocate(Zstd.compressBound(dataBytes.length));
+            MemorySegment out = arena.allocate(dataBytes.length);
+
+            // When
+            cctx.refPrefix(prefix);
+            long n = cctx.compress(frame, src);
+            dctx.refPrefix(prefix);
+            long m = dctx.decompress(out, frame.asSlice(0, n));
+
+            // Then
+            assertThat(m).isEqualTo(dataBytes.length);
+            assertThat(bytes(out, (int) m)).isEqualTo(dataBytes);
+        }
+    }
+
+    @Test
+    void prefixIsAppliedAndRequiredToDecode() {
+        // Given — incompressible (random) data identical to the prefix. With no
+        // internal redundancy the encoder can only shrink it by matching against
+        // the prefix, so a much smaller frame proves the prefix is applied.
+        // (Level 19: the level-3 match finder is too weak to reach into the prefix
+        // for random content.)
+        byte[] random = randomBytes(0xBEEF, 16384);
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment prefix = copy(arena, random);
+            MemorySegment src = copy(arena, random);
+
+            long baseline;
+            try (ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) {
+                baseline = cctx.compress(arena, src).byteSize();
+            }
+            byte[] frame;
+            try (ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) {
+                cctx.refPrefix(prefix);
+                MemorySegment f = cctx.compress(arena, src);
+                frame = bytes(f, (int) f.byteSize());
+            }
+
+            // Then — the prefix slashes the frame (16 KiB random → tens of bytes)
+            assertThat((long) frame.length).isLessThan(baseline / 10);
+
+            // And — it round-trips with the same prefix
+            try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {
+                MemorySegment out = arena.allocate(random.length);
+                dctx.refPrefix(prefix);
+                long m = dctx.decompress(out, copy(arena, frame));
+                assertThat(bytes(out, (int) m)).isEqualTo(random);
+            }
+
+            // But — it cannot be recovered without the prefix
+            boolean reproduced;
+            try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {
+                MemorySegment out = arena.allocate(random.length);
+                long m = dctx.decompress(out, copy(arena, frame));
+                reproduced = Arrays.equals(bytes(out, (int) m), random);
+            } catch (ZstdException e) {
+                reproduced = false;
+            }
+            assertThat(reproduced).isFalse();
+        }
+    }
+
+    @Test
+    void clearingPrefixWithNullCompressesPlainly() {
+        // Given
+        byte[] dataBytes = "the quick brown fox".getBytes();
+        try (Arena arena = Arena.ofConfined();
+             ZstdCompressCtx cctx = new ZstdCompressCtx()) {
+            MemorySegment prefix = copy(arena, "a prior version of the text".getBytes());
+            MemorySegment src = copy(arena, dataBytes);
+            MemorySegment frame = arena.allocate(Zstd.compressBound(dataBytes.length));
+
+            // When — set then clear the prefix before compressing
+            cctx.refPrefix(prefix);
+            cctx.refPrefix((MemorySegment) null);
+            long n = cctx.compress(frame, src);
+
+            // Then — a plain decoder with no prefix decodes it
+            byte[] restored;
+            try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {
+                MemorySegment out = arena.allocate(dataBytes.length);
+                long m = dctx.decompress(out, frame.asSlice(0, n));
+                restored = bytes(out, (int) m);
+            }
+            assertThat(restored).isEqualTo(dataBytes);
+        }
+    }
+
+    @Test
+    void prefixIsSingleUseAndDoesNotStickAcrossFrames() {
+        // Given — random data identical to the prefix (see prefixIsAppliedAndRequiredToDecode),
+        // and one context kept open across two compressions with the prefix set once.
+        byte[] random = randomBytes(0xCAFE, 16384);
+        try (Arena arena = Arena.ofConfined();
+             ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) {
+            MemorySegment prefix = copy(arena, random);
+            MemorySegment src = copy(arena, random);
+            MemorySegment first = arena.allocate(Zstd.compressBound(random.length));
+            MemorySegment second = arena.allocate(Zstd.compressBound(random.length));
+
+            // When — first frame consumes the prefix; second is compressed with no re-set
+            cctx.refPrefix(prefix);
+            long n1 = cctx.compress(first, src);
+            long n2 = cctx.compress(second, src);
+
+            // Then — the prefix shrank only the first frame; the second got no prefix
+            // (incompressible random with no prefix stays ~full size)
+            assertThat(n1).isLessThan(n2 / 10);
+
+            // And — the second frame carries no prefix, so a plain decoder decodes it
+            byte[] restored;
+            try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {
+                MemorySegment out = arena.allocate(random.length);
+                long m = dctx.decompress(out, second.asSlice(0, n2));
+                restored = bytes(out, (int) m);
+            }
+            assertThat(restored).isEqualTo(random);
+        }
+    }
+
+    @Test
+    void compressRefPrefixRejectsAHeapSegment() {
+        // Given
+        MemorySegment heap = MemorySegment.ofArray("prefix".getBytes());
+
+        // When
+        ThrowingCallable result = () -> {
+            try (ZstdCompressCtx cctx = new ZstdCompressCtx()) {
+                cctx.refPrefix(heap);
+            }
+        };
+
+        // Then
+        assertThatThrownBy(result)
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining("prefix");
+    }
+
+    @Test
+    void decompressRefPrefixRejectsAHeapSegment() {
+        // Given
+        MemorySegment heap = MemorySegment.ofArray("prefix".getBytes());
+
+        // When
+        ThrowingCallable result = () -> {
+            try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {
+                dctx.refPrefix(heap);
+            }
+        };
+
+        // Then
+        assertThatThrownBy(result)
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining("prefix");
+    }
+
+    private static MemorySegment copy(Arena arena, byte[] src) {
+        MemorySegment seg = arena.allocate(src.length);
+        MemorySegment.copy(src, 0, seg, JAVA_BYTE, 0, src.length);
+        return seg;
+    }
+
+    private static byte[] bytes(MemorySegment seg, int len) {
+        byte[] out = new byte[len];
+        MemorySegment.copy(seg, JAVA_BYTE, 0, out, 0, len);
+        return out;
+    }
+
+    private static byte[] randomBytes(long seed, int n) {
+        byte[] b = new byte[n];
+        new Random(seed).nextBytes(b);
+        return b;
+    }
+}