Skip to content

Commit fb47200

Browse files
dfa1claude
andcommitted
feat: bind ZSTD_decompressionMargin for in-place decompression
Adds ZstdFrame.decompressionMargin(byte[] / MemorySegment): the extra room needed to decompress a frame in place, where the output buffer overlaps the compressed input at its tail. Size the output buffer at decompressedSize + margin, place the frame at its end, and decompress from that tail slice into the start — no second allocation. Supports multi-frame input. Binds ZSTD_decompressionMargin. The test proves it end to end: it actually decompresses in place through the single-pass ZstdDecompressCtx.decompress(dst, src) and recovers the original from one overlapping buffer. docs/supported.md: memory sizing 8 -> 9/14. CHANGELOG updated. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 367556e commit fb47200

5 files changed

Lines changed: 102 additions & 3 deletions

File tree

CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ git tags, which trigger publication to Maven Central.
2727
- `ZstdFrame.headerSize(byte[])` / `ZstdFrame.headerSize(MemorySegment)` — the size
2828
of a frame's header computed from just its leading bytes (as few as 5), without a
2929
full parse. Binds `ZSTD_frameHeaderSize`.
30+
- `ZstdFrame.decompressionMargin(byte[])` / `ZstdFrame.decompressionMargin(MemorySegment)`
31+
— the extra room needed to decompress a frame **in place** (output buffer overlaps
32+
the compressed input at its tail), sized `decompressedSize + margin`. Binds
33+
`ZSTD_decompressionMargin`.
3034
- `ZstdDictionary.compressDict(int)` / `compressDict()` / `decompressDict()`
3135
factories for digested dictionaries, e.g. `dict.compressDict(19)` instead of
3236
`new ZstdCompressDict(dict, 19)`. They signal that the result is `AutoCloseable`

docs/supported.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`.
3535
| Streaming — decompress | 3 / 15 | `ZstdInputStream` (decompressStream + buffer sizes) |
3636
| Advanced parameters | 14 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `C/DCtx_refPrefix`, `c/dParam_getBounds`; MT inert on single-thread build |
3737
| Frame inspection | 12 / 13 | `ZstdFrame` + getFrameProgression; `_advanced` not bound |
38-
| Memory sizing | 8 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size |
38+
| Memory sizing | 9 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size, in-place decompression margin |
3939
| Low-level block | 0 / 12 | expert block/continue API not bound |
4040
| Sequences | 0 / 5 | sequence producer API not bound |
4141
| Misc / experimental | 0 / 10 | static-buffer init, param helpers, `copy*` not bound |
@@ -294,12 +294,12 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa
294294
| `ZSTD_readSkippableFrame` |||
295295
| `ZSTD_writeSkippableFrame` |||
296296

297-
### Memory sizing (8/14)
297+
### Memory sizing (9/14)
298298

299299
| Symbol | Bound | zstd-jni |
300300
|---|:---:|:---:|
301301
| `ZSTD_decodingBufferSize_min` |||
302-
| `ZSTD_decompressionMargin` | ||
302+
| `ZSTD_decompressionMargin` | ||
303303
| `ZSTD_estimateCCtxSize` |||
304304
| `ZSTD_estimateCCtxSize_usingCParams` |||
305305
| `ZSTD_estimateCDictSize` |||

zstd/src/main/java/io/github/dfa1/zstd/Bindings.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ final class Bindings {
5959
NativeLibrary.lookup("ZSTD_findDecompressedSize",
6060
FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG));
6161

62+
// size_t ZSTD_decompressionMargin(const void* src, size_t srcSize)
63+
static final MethodHandle DECOMPRESSION_MARGIN =
64+
NativeLibrary.lookup("ZSTD_decompressionMargin",
65+
FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG));
66+
6267
// unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
6368
static final MethodHandle GET_DICT_ID_FROM_FRAME =
6469
NativeLibrary.lookup("ZSTD_getDictID_fromFrame",

zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,34 @@ public static long decompressedSize(MemorySegment data) {
102102
return decompressedSize(data, data.byteSize());
103103
}
104104

105+
/// Extra room needed for **in-place** decompression of `data`, where the input
106+
/// and output buffers overlap.
107+
///
108+
/// zstd can decompress into a buffer that also holds the compressed input, with
109+
/// no second allocation: make the output buffer at least
110+
/// `decompressedSize + margin` bytes and place the compressed frame(s) at its
111+
/// **end**, then decompress from that tail slice into the start. This returns
112+
/// that `margin`. Supports multi-frame input.
113+
///
114+
/// @param data one or more concatenated zstd frames
115+
/// @return the in-place decompression margin in bytes
116+
/// @throws ZstdException if the input is not valid zstd data
117+
public static long decompressionMargin(byte[] data) {
118+
try (Arena arena = Arena.ofConfined()) {
119+
return decompressionMargin(Zstd.copyIn(arena, data), data.length);
120+
}
121+
}
122+
123+
/// Extra room needed for in-place decompression of native `data`.
124+
/// Otherwise identical to [#decompressionMargin(byte[])].
125+
///
126+
/// @param data one or more concatenated zstd frames
127+
/// @return the in-place decompression margin in bytes
128+
/// @throws ZstdException if the input is not valid zstd data
129+
public static long decompressionMargin(MemorySegment data) {
130+
return decompressionMargin(data, data.byteSize());
131+
}
132+
105133
/// Dictionary id recorded in the first frame's header.
106134
///
107135
/// @param data a zstd frame
@@ -281,6 +309,10 @@ private static long decompressedSize(MemorySegment data, long size) {
281309
return Zstd.requireStoredContentSize(total);
282310
}
283311

312+
private static long decompressionMargin(MemorySegment data, long size) {
313+
return NativeCall.checkReturnValue(() -> (long) Bindings.DECOMPRESSION_MARGIN.invokeExact(data, size));
314+
}
315+
284316
private static long headerSize(MemorySegment data, long size) {
285317
return NativeCall.checkReturnValue(() -> (long) Bindings.FRAME_HEADER_SIZE.invokeExact(data, size));
286318
}

zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import java.io.ByteArrayOutputStream;
88
import java.lang.foreign.Arena;
99
import java.lang.foreign.MemorySegment;
10+
import java.lang.foreign.ValueLayout;
1011
import java.nio.charset.StandardCharsets;
1112
import java.util.ArrayList;
1213
import java.util.Arrays;
@@ -209,6 +210,63 @@ void rejectsInputShorterThanThePrefix() {
209210
}
210211
}
211212

213+
@Nested
214+
class DecompressionMargin {
215+
216+
@Test
217+
void isPositiveForAValidFrame() {
218+
// Given a frame
219+
byte[] frame = Zstd.compress(PAYLOAD);
220+
221+
// Then its in-place margin is a real, positive size
222+
assertThat(ZstdFrame.decompressionMargin(frame)).isPositive();
223+
}
224+
225+
@Test
226+
void matchesThroughTheSegmentOverload() {
227+
byte[] frame = Zstd.compress(PAYLOAD);
228+
try (Arena arena = Arena.ofConfined()) {
229+
MemorySegment seg = Zstd.copyIn(arena, frame);
230+
assertThat(ZstdFrame.decompressionMargin(seg)).isEqualTo(ZstdFrame.decompressionMargin(frame));
231+
}
232+
}
233+
234+
@Test
235+
void enablesInPlaceDecompression() {
236+
// Given a frame and a single buffer sized output + margin
237+
byte[] frame = Zstd.compress(PAYLOAD);
238+
long margin = ZstdFrame.decompressionMargin(frame);
239+
try (Arena arena = Arena.ofConfined();
240+
ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {
241+
MemorySegment buf = arena.allocate(PAYLOAD.length + margin);
242+
243+
// and the compressed frame placed at the very end of that buffer
244+
MemorySegment in = buf.asSlice(buf.byteSize() - frame.length, frame.length);
245+
MemorySegment.copy(frame, 0, in, ValueLayout.JAVA_BYTE, 0, frame.length);
246+
247+
// When decompressed in place (output overlaps the input tail)
248+
long n = dctx.decompress(buf, in);
249+
250+
// Then the original is recovered at the start of the same buffer
251+
byte[] out = new byte[(int) n];
252+
MemorySegment.copy(buf, ValueLayout.JAVA_BYTE, 0, out, 0, (int) n);
253+
assertThat(out).isEqualTo(PAYLOAD);
254+
}
255+
}
256+
257+
@Test
258+
void rejectsGarbage() {
259+
// Given bytes that are not valid zstd data
260+
byte[] garbage = "xx".getBytes(StandardCharsets.UTF_8);
261+
262+
// When the margin is requested
263+
ThrowingCallable result = () -> ZstdFrame.decompressionMargin(garbage);
264+
265+
// Then it fails
266+
assertThatThrownBy(result).isInstanceOf(ZstdException.class);
267+
}
268+
}
269+
212270
@Nested
213271
class Header {
214272

0 commit comments

Comments
 (0)