Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ git tags, which trigger publication to Maven Central.
- `ZstdFrame.headerSize(byte[])` / `ZstdFrame.headerSize(MemorySegment)` — the size
of a frame's header computed from just its leading bytes (as few as 5), without a
full parse. Binds `ZSTD_frameHeaderSize`.
- `ZstdFrame.decompressionMargin(byte[])` / `ZstdFrame.decompressionMargin(MemorySegment)`
— the extra room needed to decompress a frame **in place** (output buffer overlaps
the compressed input at its tail), sized `decompressedSize + margin`. Binds
`ZSTD_decompressionMargin`.
- `ZstdDictionary.compressDict(int)` / `compressDict()` / `decompressDict()` —
factories for digested dictionaries, e.g. `dict.compressDict(19)` instead of
`new ZstdCompressDict(dict, 19)`. They signal that the result is `AutoCloseable`
Expand Down
6 changes: 3 additions & 3 deletions docs/supported.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`.
| Streaming — decompress | 3 / 15 | `ZstdInputStream` (decompressStream + buffer sizes) |
| Advanced parameters | 14 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `C/DCtx_refPrefix`, `c/dParam_getBounds`; MT inert on single-thread build |
| Frame inspection | 12 / 13 | `ZstdFrame` + getFrameProgression; `_advanced` not bound |
| Memory sizing | 8 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size |
| Memory sizing | 9 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size, in-place decompression margin |
| Low-level block | 0 / 12 | expert block/continue API not bound |
| Sequences | 0 / 5 | sequence producer API not bound |
| Misc / experimental | 0 / 10 | static-buffer init, param helpers, `copy*` not bound |
Expand Down Expand Up @@ -294,12 +294,12 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa
| `ZSTD_readSkippableFrame` | ✅ | — |
| `ZSTD_writeSkippableFrame` | ✅ | — |

### Memory sizing (8/14)
### Memory sizing (9/14)

| Symbol | Bound | zstd-jni |
|---|:---:|:---:|
| `ZSTD_decodingBufferSize_min` | — | — |
| `ZSTD_decompressionMargin` | | — |
| `ZSTD_decompressionMargin` | | — |
| `ZSTD_estimateCCtxSize` | ✅ | — |
| `ZSTD_estimateCCtxSize_usingCParams` | — | — |
| `ZSTD_estimateCDictSize` | ✅ | — |
Expand Down
5 changes: 5 additions & 0 deletions zstd/src/main/java/io/github/dfa1/zstd/Bindings.java
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ final class Bindings {
NativeLibrary.lookup("ZSTD_findDecompressedSize",
FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG));

// size_t ZSTD_decompressionMargin(const void* src, size_t srcSize)
static final MethodHandle DECOMPRESSION_MARGIN =
NativeLibrary.lookup("ZSTD_decompressionMargin",
FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG));

// unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
static final MethodHandle GET_DICT_ID_FROM_FRAME =
NativeLibrary.lookup("ZSTD_getDictID_fromFrame",
Expand Down
32 changes: 32 additions & 0 deletions zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,34 @@ public static long decompressedSize(MemorySegment data) {
return decompressedSize(data, data.byteSize());
}

/// Extra room needed for **in-place** decompression of `data`, where the input
/// and output buffers overlap.
///
/// zstd can decompress into a buffer that also holds the compressed input, with
/// no second allocation: make the output buffer at least
/// `decompressedSize + margin` bytes and place the compressed frame(s) at its
/// **end**, then decompress from that tail slice into the start. This returns
/// that `margin`. Supports multi-frame input.
///
/// @param data one or more concatenated zstd frames
/// @return the in-place decompression margin in bytes
/// @throws ZstdException if the input is not valid zstd data
public static long decompressionMargin(byte[] data) {
try (Arena arena = Arena.ofConfined()) {
return decompressionMargin(Zstd.copyIn(arena, data), data.length);
}
}

/// Extra room needed for in-place decompression of native `data`.
/// Otherwise identical to [#decompressionMargin(byte[])].
///
/// @param data one or more concatenated zstd frames
/// @return the in-place decompression margin in bytes
/// @throws ZstdException if the input is not valid zstd data
public static long decompressionMargin(MemorySegment data) {
return decompressionMargin(data, data.byteSize());
}

/// Dictionary id recorded in the first frame's header.
///
/// @param data a zstd frame
Expand Down Expand Up @@ -281,6 +309,10 @@ private static long decompressedSize(MemorySegment data, long size) {
return Zstd.requireStoredContentSize(total);
}

private static long decompressionMargin(MemorySegment data, long size) {
return NativeCall.checkReturnValue(() -> (long) Bindings.DECOMPRESSION_MARGIN.invokeExact(data, size));
}

private static long headerSize(MemorySegment data, long size) {
return NativeCall.checkReturnValue(() -> (long) Bindings.FRAME_HEADER_SIZE.invokeExact(data, size));
}
Expand Down
58 changes: 58 additions & 0 deletions zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.io.ByteArrayOutputStream;
import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -209,6 +210,63 @@ void rejectsInputShorterThanThePrefix() {
}
}

@Nested
class DecompressionMargin {

@Test
void isPositiveForAValidFrame() {
// Given a frame
byte[] frame = Zstd.compress(PAYLOAD);

// Then its in-place margin is a real, positive size
assertThat(ZstdFrame.decompressionMargin(frame)).isPositive();
}

@Test
void matchesThroughTheSegmentOverload() {
byte[] frame = Zstd.compress(PAYLOAD);
try (Arena arena = Arena.ofConfined()) {
MemorySegment seg = Zstd.copyIn(arena, frame);
assertThat(ZstdFrame.decompressionMargin(seg)).isEqualTo(ZstdFrame.decompressionMargin(frame));
}
}

@Test
void enablesInPlaceDecompression() {
// Given a frame and a single buffer sized output + margin
byte[] frame = Zstd.compress(PAYLOAD);
long margin = ZstdFrame.decompressionMargin(frame);
try (Arena arena = Arena.ofConfined();
ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {
MemorySegment buf = arena.allocate(PAYLOAD.length + margin);

// and the compressed frame placed at the very end of that buffer
MemorySegment in = buf.asSlice(buf.byteSize() - frame.length, frame.length);
MemorySegment.copy(frame, 0, in, ValueLayout.JAVA_BYTE, 0, frame.length);

// When decompressed in place (output overlaps the input tail)
long n = dctx.decompress(buf, in);

// Then the original is recovered at the start of the same buffer
byte[] out = new byte[(int) n];
MemorySegment.copy(buf, ValueLayout.JAVA_BYTE, 0, out, 0, (int) n);
assertThat(out).isEqualTo(PAYLOAD);
}
}

@Test
void rejectsGarbage() {
// Given bytes that are not valid zstd data
byte[] garbage = "xx".getBytes(StandardCharsets.UTF_8);

// When the margin is requested
ThrowingCallable result = () -> ZstdFrame.decompressionMargin(garbage);

// Then it fails
assertThatThrownBy(result).isInstanceOf(ZstdException.class);
}
}

@Nested
class Header {

Expand Down
Loading