diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e020cc..aeb49ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this project are documented here. Format loosely follows [Keep a Changelog](https://keepachangelog.com/); versions are released as `v*` git tags, which trigger publication to Maven Central. +## [Unreleased] + +### Added +- `ZstdCompressCtx.refPrefix(MemorySegment)` / `ZstdDecompressCtx.refPrefix(...)` + — reference native content as a single-use prefix (raw-content dictionary) for + the next frame only: the building block for delta compression (compress a new + version against a similar previous one). The prefix is referenced, not copied + or digested, and writes no dictionary ID; the decompressor must reference the + same prefix to decode. Binds `ZSTD_CCtx_refPrefix` / `ZSTD_DCtx_refPrefix`. + Segment-only by design — heap callers that need a copy should use + `loadDictionary` instead. + ## [0.5] ### Added diff --git a/docs/supported.md b/docs/supported.md index f548f15..b9058de 100644 --- a/docs/supported.md +++ b/docs/supported.md @@ -33,7 +33,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`. | Dictionary training (ZDICT) | 8 / 12 | trainFromBuffer, cover/fastCover optimizers, finalizeDictionary, getDictHeaderSize | | Streaming — compress | 3 / 22 | `ZstdOutputStream` (compressStream2 + buffer sizes) | | Streaming — decompress | 3 / 15 | `ZstdInputStream` (decompressStream + buffer sizes) | -| Advanced parameters | 12 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `c/dParam_getBounds`; MT inert on single-thread build | +| Advanced parameters | 14 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `C/DCtx_refPrefix`, `c/dParam_getBounds`; MT inert on single-thread build | | Frame inspection | 10 / 13 | `ZstdFrame` + getFrameProgression; `_advanced` not bound | | Memory sizing | 8 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size | | Low-level block | 0 / 12 | expert block/continue API not bound | @@ -233,7 +233,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa | `ZSTD_resetDStream` | — ᵈ | — | | `ZSTD_sizeof_DStream` | — | — | -### Advanced parameters (12/38) +### Advanced parameters (14/38) | Symbol | Bound | zstd-jni | |---|:---:|:---:| @@ -248,7 +248,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa | `ZSTD_CCtx_loadDictionary_advanced` | — | — | | `ZSTD_CCtx_loadDictionary_byReference` | — | — | | `ZSTD_CCtx_refCDict` | ✅ | ✅ | -| `ZSTD_CCtx_refPrefix` | — | — | +| `ZSTD_CCtx_refPrefix` | ✅ | — | | `ZSTD_CCtx_refPrefix_advanced` | — | — | | `ZSTD_CCtx_refThreadPool` | — | — | | `ZSTD_CCtx_reset` | ✅ | ✅ | @@ -263,7 +263,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa | `ZSTD_DCtx_loadDictionary_advanced` | — | — | | `ZSTD_DCtx_loadDictionary_byReference` | — | — | | `ZSTD_DCtx_refDDict` | ✅ | ✅ | -| `ZSTD_DCtx_refPrefix` | — | — | +| `ZSTD_DCtx_refPrefix` | ✅ | — | | `ZSTD_DCtx_refPrefix_advanced` | — | — | | `ZSTD_DCtx_reset` | ✅ | ✅ | | `ZSTD_DCtx_setFormat` | — ᵈ | — | diff --git a/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java b/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java index 268d033..8275dae 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java @@ -210,6 +210,15 @@ final class Bindings { NativeLibrary.lookup("ZSTD_DCtx_loadDictionary", FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG)); + // size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx*, const void* prefix, size_t prefixSize) + static final MethodHandle CCTX_REF_PREFIX = + NativeLibrary.lookup("ZSTD_CCtx_refPrefix", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG)); + // size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx*, const void* prefix, size_t prefixSize) + static final MethodHandle DCTX_REF_PREFIX = + NativeLibrary.lookup("ZSTD_DCtx_refPrefix", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG)); + // size_t ZSTD_CStreamInSize(void) / ZSTD_CStreamOutSize(void) static final MethodHandle CSTREAM_IN_SIZE = NativeLibrary.lookup("ZSTD_CStreamInSize", FunctionDescriptor.of(JAVA_LONG)); diff --git a/zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java b/zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java index e1ab1ea..5848dce 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java @@ -177,6 +177,38 @@ public ZstdCompressCtx refDictionary(ZstdCompressDict dict) { return this; } + /// References native `prefix` content as a single-use dictionary for the + /// **next** compression only — the building block for delta compression: + /// compress a new version against a similar previous one as the prefix, + /// storing little more than the difference. + /// + /// Unlike [#loadDictionary(ZstdDictionary)], a prefix is referenced (no + /// digest, no heap copy), writes no dictionary ID into the frame, and is + /// consumed by the next [#compress(MemorySegment, MemorySegment)] / + /// [#compress(byte[])] — it does not stick across frames. The decompressor + /// must reference the **same** prefix with + /// [ZstdDecompressCtx#refPrefix(MemorySegment)] to decode the frame. + /// + /// Because the prefix is referenced, `prefix` must stay valid until the next + /// compression completes. Heap callers that cannot manage native lifetime + /// should use a copying dictionary ([#loadDictionary(ZstdDictionary)]) instead. + /// + /// @param prefix native prefix content, or `null` / [MemorySegment#NULL] to clear it + /// @return `this`, for chaining + /// @throws ZstdException if the prefix cannot be referenced + public ZstdCompressCtx refPrefix(MemorySegment prefix) { + if (NativeCall.isNull(prefix)) { + return refPrefix(MemorySegment.NULL, 0L); + } + NativeCall.requireNative(prefix, "prefix"); + return refPrefix(prefix, prefix.byteSize()); + } + + private ZstdCompressCtx refPrefix(MemorySegment prefix, long size) { + NativeCall.checkReturnValue(() -> (long) Bindings.CCTX_REF_PREFIX.invokeExact(ptr(), prefix, size)); + return this; + } + /// Compresses `src` into a new zstd frame using this context and its /// advanced parameters. /// diff --git a/zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java b/zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java index 814c4b1..0e39962 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java @@ -126,6 +126,36 @@ public ZstdDecompressCtx refDictionary(ZstdDecompressDict dict) { return this; } + /// References native `prefix` content as a single-use dictionary for decoding + /// the **next** frame only — the decompression counterpart of + /// [ZstdCompressCtx#refPrefix(MemorySegment)]. It must be the **same** content + /// the frame was compressed against, or decoding fails. + /// + /// The prefix is referenced, not copied (no digest, no heap copy): `prefix` + /// must stay valid until the next [#decompress(byte[], int)] / + /// [#decompress(MemorySegment, MemorySegment)], which consumes it — it does not + /// stick across frames. Pass `null` / [MemorySegment#NULL] to clear a prefix + /// set but not yet consumed. + /// + /// Heap callers that cannot manage native lifetime should use a copying + /// dictionary ([#loadDictionary(ZstdDictionary)]) instead. + /// + /// @param prefix native prefix content, or `null` / [MemorySegment#NULL] to clear it + /// @return `this`, for chaining + /// @throws ZstdException if the prefix cannot be referenced + public ZstdDecompressCtx refPrefix(MemorySegment prefix) { + if (NativeCall.isNull(prefix)) { + return refPrefix(MemorySegment.NULL, 0L); + } + NativeCall.requireNative(prefix, "prefix"); + return refPrefix(prefix, prefix.byteSize()); + } + + private ZstdDecompressCtx refPrefix(MemorySegment prefix, long size) { + NativeCall.checkReturnValue(() -> (long) Bindings.DCTX_REF_PREFIX.invokeExact(ptr(), prefix, size)); + return this; + } + /// Decompresses a frame into a buffer of at most `maxSize` bytes. /// /// @param compressed a complete zstd frame diff --git a/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java b/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java new file mode 100644 index 0000000..9cf583f --- /dev/null +++ b/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java @@ -0,0 +1,199 @@ +package io.github.dfa1.zstd; + +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.util.Arrays; +import java.util.Random; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.Test; + +class RefPrefixTest { + + @Test + void roundTripsWithMatchingPrefix() { + // Given + byte[] prefixBytes = "the quick brown fox jumps over the lazy dog".getBytes(); + byte[] dataBytes = "the quick brown fox jumps over the lazy cat".getBytes(); + try (Arena arena = Arena.ofConfined(); + ZstdCompressCtx cctx = new ZstdCompressCtx(); + ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment prefix = copy(arena, prefixBytes); + MemorySegment src = copy(arena, dataBytes); + MemorySegment frame = arena.allocate(Zstd.compressBound(dataBytes.length)); + MemorySegment out = arena.allocate(dataBytes.length); + + // When + cctx.refPrefix(prefix); + long n = cctx.compress(frame, src); + dctx.refPrefix(prefix); + long m = dctx.decompress(out, frame.asSlice(0, n)); + + // Then + assertThat(m).isEqualTo(dataBytes.length); + assertThat(bytes(out, (int) m)).isEqualTo(dataBytes); + } + } + + @Test + void prefixIsAppliedAndRequiredToDecode() { + // Given — incompressible (random) data identical to the prefix. With no + // internal redundancy the encoder can only shrink it by matching against + // the prefix, so a much smaller frame proves the prefix is applied. + // (Level 19: the level-3 match finder is too weak to reach into the prefix + // for random content.) + byte[] random = randomBytes(0xBEEF, 16384); + try (Arena arena = Arena.ofConfined()) { + MemorySegment prefix = copy(arena, random); + MemorySegment src = copy(arena, random); + + long baseline; + try (ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { + baseline = cctx.compress(arena, src).byteSize(); + } + byte[] frame; + try (ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { + cctx.refPrefix(prefix); + MemorySegment f = cctx.compress(arena, src); + frame = bytes(f, (int) f.byteSize()); + } + + // Then — the prefix slashes the frame (16 KiB random → tens of bytes) + assertThat((long) frame.length).isLessThan(baseline / 10); + + // And — it round-trips with the same prefix + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment out = arena.allocate(random.length); + dctx.refPrefix(prefix); + long m = dctx.decompress(out, copy(arena, frame)); + assertThat(bytes(out, (int) m)).isEqualTo(random); + } + + // But — it cannot be recovered without the prefix + boolean reproduced; + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment out = arena.allocate(random.length); + long m = dctx.decompress(out, copy(arena, frame)); + reproduced = Arrays.equals(bytes(out, (int) m), random); + } catch (ZstdException e) { + reproduced = false; + } + assertThat(reproduced).isFalse(); + } + } + + @Test + void clearingPrefixWithNullCompressesPlainly() { + // Given + byte[] dataBytes = "the quick brown fox".getBytes(); + try (Arena arena = Arena.ofConfined(); + ZstdCompressCtx cctx = new ZstdCompressCtx()) { + MemorySegment prefix = copy(arena, "a prior version of the text".getBytes()); + MemorySegment src = copy(arena, dataBytes); + MemorySegment frame = arena.allocate(Zstd.compressBound(dataBytes.length)); + + // When — set then clear the prefix before compressing + cctx.refPrefix(prefix); + cctx.refPrefix((MemorySegment) null); + long n = cctx.compress(frame, src); + + // Then — a plain decoder with no prefix decodes it + byte[] restored; + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment out = arena.allocate(dataBytes.length); + long m = dctx.decompress(out, frame.asSlice(0, n)); + restored = bytes(out, (int) m); + } + assertThat(restored).isEqualTo(dataBytes); + } + } + + @Test + void prefixIsSingleUseAndDoesNotStickAcrossFrames() { + // Given — random data identical to the prefix (see prefixIsAppliedAndRequiredToDecode), + // and one context kept open across two compressions with the prefix set once. + byte[] random = randomBytes(0xCAFE, 16384); + try (Arena arena = Arena.ofConfined(); + ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { + MemorySegment prefix = copy(arena, random); + MemorySegment src = copy(arena, random); + MemorySegment first = arena.allocate(Zstd.compressBound(random.length)); + MemorySegment second = arena.allocate(Zstd.compressBound(random.length)); + + // When — first frame consumes the prefix; second is compressed with no re-set + cctx.refPrefix(prefix); + long n1 = cctx.compress(first, src); + long n2 = cctx.compress(second, src); + + // Then — the prefix shrank only the first frame; the second got no prefix + // (incompressible random with no prefix stays ~full size) + assertThat(n1).isLessThan(n2 / 10); + + // And — the second frame carries no prefix, so a plain decoder decodes it + byte[] restored; + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment out = arena.allocate(random.length); + long m = dctx.decompress(out, second.asSlice(0, n2)); + restored = bytes(out, (int) m); + } + assertThat(restored).isEqualTo(random); + } + } + + @Test + void compressRefPrefixRejectsAHeapSegment() { + // Given + MemorySegment heap = MemorySegment.ofArray("prefix".getBytes()); + + // When + ThrowingCallable result = () -> { + try (ZstdCompressCtx cctx = new ZstdCompressCtx()) { + cctx.refPrefix(heap); + } + }; + + // Then + assertThatThrownBy(result) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("prefix"); + } + + @Test + void decompressRefPrefixRejectsAHeapSegment() { + // Given + MemorySegment heap = MemorySegment.ofArray("prefix".getBytes()); + + // When + ThrowingCallable result = () -> { + try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + dctx.refPrefix(heap); + } + }; + + // Then + assertThatThrownBy(result) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("prefix"); + } + + private static MemorySegment copy(Arena arena, byte[] src) { + MemorySegment seg = arena.allocate(src.length); + MemorySegment.copy(src, 0, seg, JAVA_BYTE, 0, src.length); + return seg; + } + + private static byte[] bytes(MemorySegment seg, int len) { + byte[] out = new byte[len]; + MemorySegment.copy(seg, JAVA_BYTE, 0, out, 0, len); + return out; + } + + private static byte[] randomBytes(long seed, int n) { + byte[] b = new byte[n]; + new Random(seed).nextBytes(b); + return b; + } +}