diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c52e3f..514e905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ git tags, which trigger publication to Maven Central. - `ZstdFrame.headerSize(byte[])` / `ZstdFrame.headerSize(MemorySegment)` — the size of a frame's header computed from just its leading bytes (as few as 5), without a full parse. Binds `ZSTD_frameHeaderSize`. +- `ZstdFrame.decompressionMargin(byte[])` / `ZstdFrame.decompressionMargin(MemorySegment)` + — the extra room needed to decompress a frame **in place** (output buffer overlaps + the compressed input at its tail), sized `decompressedSize + margin`. Binds + `ZSTD_decompressionMargin`. - `ZstdDictionary.compressDict(int)` / `compressDict()` / `decompressDict()` — factories for digested dictionaries, e.g. `dict.compressDict(19)` instead of `new ZstdCompressDict(dict, 19)`. They signal that the result is `AutoCloseable` diff --git a/docs/supported.md b/docs/supported.md index 7a6e0ac..92753fc 100644 --- a/docs/supported.md +++ b/docs/supported.md @@ -35,7 +35,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`. | Streaming — decompress | 3 / 15 | `ZstdInputStream` (decompressStream + buffer sizes) | | Advanced parameters | 14 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `C/DCtx_refPrefix`, `c/dParam_getBounds`; MT inert on single-thread build | | Frame inspection | 12 / 13 | `ZstdFrame` + getFrameProgression; `_advanced` not bound | -| Memory sizing | 8 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size | +| Memory sizing | 9 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size, in-place decompression margin | | Low-level block | 0 / 12 | expert block/continue API not bound | | Sequences | 0 / 5 | sequence producer API not bound | | Misc / experimental | 0 / 10 | static-buffer init, param helpers, `copy*` not bound | @@ -294,12 +294,12 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa | `ZSTD_readSkippableFrame` | ✅ | — | | `ZSTD_writeSkippableFrame` | ✅ | — | -### Memory sizing (8/14) +### Memory sizing (9/14) | Symbol | Bound | zstd-jni | |---|:---:|:---:| | `ZSTD_decodingBufferSize_min` | — | — | -| `ZSTD_decompressionMargin` | — | — | +| `ZSTD_decompressionMargin` | ✅ | — | | `ZSTD_estimateCCtxSize` | ✅ | — | | `ZSTD_estimateCCtxSize_usingCParams` | — | — | | `ZSTD_estimateCDictSize` | ✅ | — | diff --git a/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java b/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java index de48da0..29514fb 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java @@ -59,6 +59,11 @@ final class Bindings { NativeLibrary.lookup("ZSTD_findDecompressedSize", FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG)); + // size_t ZSTD_decompressionMargin(const void* src, size_t srcSize) + static final MethodHandle DECOMPRESSION_MARGIN = + NativeLibrary.lookup("ZSTD_decompressionMargin", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG)); + // unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) static final MethodHandle GET_DICT_ID_FROM_FRAME = NativeLibrary.lookup("ZSTD_getDictID_fromFrame", diff --git a/zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java b/zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java index 119ba12..0cd89bf 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java @@ -102,6 +102,34 @@ public static long decompressedSize(MemorySegment data) { return decompressedSize(data, data.byteSize()); } + /// Extra room needed for **in-place** decompression of `data`, where the input + /// and output buffers overlap. + /// + /// zstd can decompress into a buffer that also holds the compressed input, with + /// no second allocation: make the output buffer at least + /// `decompressedSize + margin` bytes and place the compressed frame(s) at its + /// **end**, then decompress from that tail slice into the start. This returns + /// that `margin`. Supports multi-frame input. + /// + /// @param data one or more concatenated zstd frames + /// @return the in-place decompression margin in bytes + /// @throws ZstdException if the input is not valid zstd data + public static long decompressionMargin(byte[] data) { + try (Arena arena = Arena.ofConfined()) { + return decompressionMargin(Zstd.copyIn(arena, data), data.length); + } + } + + /// Extra room needed for in-place decompression of native `data`. + /// Otherwise identical to [#decompressionMargin(byte[])]. + /// + /// @param data one or more concatenated zstd frames + /// @return the in-place decompression margin in bytes + /// @throws ZstdException if the input is not valid zstd data + public static long decompressionMargin(MemorySegment data) { + return decompressionMargin(data, data.byteSize()); + } + /// Dictionary id recorded in the first frame's header. /// /// @param data a zstd frame @@ -281,6 +309,10 @@ private static long decompressedSize(MemorySegment data, long size) { return Zstd.requireStoredContentSize(total); } + private static long decompressionMargin(MemorySegment data, long size) { + return NativeCall.checkReturnValue(() -> (long) Bindings.DECOMPRESSION_MARGIN.invokeExact(data, size)); + } + private static long headerSize(MemorySegment data, long size) { return NativeCall.checkReturnValue(() -> (long) Bindings.FRAME_HEADER_SIZE.invokeExact(data, size)); } diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java index fa5d102..e912fb8 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java @@ -7,6 +7,7 @@ import java.io.ByteArrayOutputStream; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -209,6 +210,63 @@ void rejectsInputShorterThanThePrefix() { } } + @Nested + class DecompressionMargin { + + @Test + void isPositiveForAValidFrame() { + // Given a frame + byte[] frame = Zstd.compress(PAYLOAD); + + // Then its in-place margin is a real, positive size + assertThat(ZstdFrame.decompressionMargin(frame)).isPositive(); + } + + @Test + void matchesThroughTheSegmentOverload() { + byte[] frame = Zstd.compress(PAYLOAD); + try (Arena arena = Arena.ofConfined()) { + MemorySegment seg = Zstd.copyIn(arena, frame); + assertThat(ZstdFrame.decompressionMargin(seg)).isEqualTo(ZstdFrame.decompressionMargin(frame)); + } + } + + @Test + void enablesInPlaceDecompression() { + // Given a frame and a single buffer sized output + margin + byte[] frame = Zstd.compress(PAYLOAD); + long margin = ZstdFrame.decompressionMargin(frame); + try (Arena arena = Arena.ofConfined(); + ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { + MemorySegment buf = arena.allocate(PAYLOAD.length + margin); + + // and the compressed frame placed at the very end of that buffer + MemorySegment in = buf.asSlice(buf.byteSize() - frame.length, frame.length); + MemorySegment.copy(frame, 0, in, ValueLayout.JAVA_BYTE, 0, frame.length); + + // When decompressed in place (output overlaps the input tail) + long n = dctx.decompress(buf, in); + + // Then the original is recovered at the start of the same buffer + byte[] out = new byte[(int) n]; + MemorySegment.copy(buf, ValueLayout.JAVA_BYTE, 0, out, 0, (int) n); + assertThat(out).isEqualTo(PAYLOAD); + } + } + + @Test + void rejectsGarbage() { + // Given bytes that are not valid zstd data + byte[] garbage = "xx".getBytes(StandardCharsets.UTF_8); + + // When the margin is requested + ThrowingCallable result = () -> ZstdFrame.decompressionMargin(garbage); + + // Then it fails + assertThatThrownBy(result).isInstanceOf(ZstdException.class); + } + } + @Nested class Header {