diff --git a/CHANGELOG.md b/CHANGELOG.md index ad8efbe..6b0f508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,13 @@ git tags, which trigger publication to Maven Central. Binds `ZSTD_getDictID_fromDict`. - `ZstdDictionaryId` value type — a `record` wrapping the 32-bit dictionary id with an unsigned `value()`, `isPresent()`, and the `NONE` sentinel for "no id". +- `ZstdFrame.decompressedSize(byte[])` / `ZstdFrame.decompressedSize(MemorySegment)` + — the exact combined decompressed size of all concatenated frames, summed from + each frame header (throws if any frame does not record its size). Complements + `decompressedBound` (upper bound). Binds `ZSTD_findDecompressedSize`. +- `ZstdFrame.headerSize(byte[])` / `ZstdFrame.headerSize(MemorySegment)` — the size + of a frame's header computed from just its leading bytes (as few as 5), without a + full parse. Binds `ZSTD_frameHeaderSize`. ### Changed - Every dictionary-id accessor now returns `ZstdDictionaryId` instead of `int`: diff --git a/docs/supported.md b/docs/supported.md index e033e2d..7a6e0ac 100644 --- a/docs/supported.md +++ b/docs/supported.md @@ -34,7 +34,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`. | Streaming — compress | 3 / 22 | `ZstdOutputStream` (compressStream2 + buffer sizes) | | Streaming — decompress | 3 / 15 | `ZstdInputStream` (decompressStream + buffer sizes) | | Advanced parameters | 14 / 38 | all `ZSTD_cParameter` + `ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter`; `compress2`, `C/DCtx_setParameter`, `C/DCtx_reset`, `C/DCtx_loadDictionary`, `CCtx_refCDict`/`DCtx_refDDict`, `C/DCtx_refPrefix`, `c/dParam_getBounds`; MT inert on single-thread build | -| Frame inspection | 10 / 13 | `ZstdFrame` + getFrameProgression; `_advanced` not bound | +| Frame inspection | 12 / 13 | `ZstdFrame` + getFrameProgression; `_advanced` not bound | | Memory sizing | 8 / 14 | sizeof_C/DCtx, sizeof_C/DDict, estimate C/DCtx + C/DDict size | | Low-level block | 0 / 12 | expert block/continue API not bound | | Sequences | 0 / 5 | sequence producer API not bound | @@ -81,7 +81,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`. 1. ~~**Streaming**~~ — done: `ZstdOutputStream` / `ZstdInputStream` (`compressStream2`, `decompressStream`, bounded buffers, dictionary constructors, `pledgedSrcSize` via `withPledgedSize`). Remaining: `MemorySegment`-buffer driver. 2. ~~**Advanced parameters**~~ — done: every `ZSTD_cParameter`/`ZSTD_dParameter` via `ZstdCompressParameter`/`ZstdDecompressParameter` (+ `bounds()`), on both contexts; `pledgedSrcSize`. `nbWorkers` is settable but inert until the native build enables multithreading. -3. ~~**Frame inspection**~~ — done: `ZstdFrame` (`isFrame`, `header`, `compressedSize`, `decompressedBound`, `dictId`, skippable, `getFrameProgression`); dict-id from raw/CDict/DDict. +3. ~~**Frame inspection**~~ — done: `ZstdFrame` (`isFrame`, `header`, `compressedSize`, `decompressedBound`, `decompressedSize`, `dictId`, skippable, `getFrameProgression`); dict-id from raw/CDict/DDict. 4. ~~**Better dictionaries**~~ — done: COVER / fast-COVER optimisers, `finalizeDictionary`, `getDictHeaderSize`. 5. ~~**Typed errors**~~ — done: `ZstdException.code()` returns `ZstdErrorCode` (via `getErrorCode`). @@ -276,15 +276,15 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa | `ZSTD_estimateCCtxSize_usingCCtxParams` | — | — | | `ZSTD_freeCCtxParams` | — | — | -### Frame inspection (10/13) +### Frame inspection (12/13) | Symbol | Bound | zstd-jni | |---|:---:|:---:| | `ZSTD_getFrameContentSize` | ✅ | ✅ | | `ZSTD_decompressBound` | ✅ | — | -| `ZSTD_findDecompressedSize` | — | — | +| `ZSTD_findDecompressedSize` | ✅ | — | | `ZSTD_findFrameCompressedSize` | ✅ | ✅ | -| `ZSTD_frameHeaderSize` | — | — | +| `ZSTD_frameHeaderSize` | ✅ | — | | `ZSTD_getDecompressedSize` | — ᵈ | — | | `ZSTD_getFrameHeader` | ✅ | — | | `ZSTD_getFrameHeader_advanced` | — | ✅ | diff --git a/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java b/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java index 8073811..86a95d9 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/Bindings.java @@ -54,6 +54,11 @@ final class Bindings { NativeLibrary.lookup("ZSTD_decompressBound", FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG)); + // unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) + static final MethodHandle FIND_DECOMPRESSED_SIZE = + NativeLibrary.lookup("ZSTD_findDecompressedSize", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG)); + // unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) static final MethodHandle GET_DICT_ID_FROM_FRAME = NativeLibrary.lookup("ZSTD_getDictID_fromFrame", @@ -64,6 +69,11 @@ final class Bindings { NativeLibrary.lookup("ZSTD_getFrameHeader", FunctionDescriptor.of(JAVA_LONG, ADDRESS, ADDRESS, JAVA_LONG)); + // size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) + static final MethodHandle FRAME_HEADER_SIZE = + NativeLibrary.lookup("ZSTD_frameHeaderSize", + FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG)); + // unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) static final MethodHandle IS_SKIPPABLE_FRAME = NativeLibrary.lookup("ZSTD_isSkippableFrame", diff --git a/zstd/src/main/java/io/github/dfa1/zstd/Zstd.java b/zstd/src/main/java/io/github/dfa1/zstd/Zstd.java index fa2a7ea..892bebf 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/Zstd.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/Zstd.java @@ -59,13 +59,7 @@ public static byte[] compress(byte[] src, int level) { /// use {@link #decompress(byte[], int)} for the latter public static byte[] decompress(byte[] compressed) { Objects.requireNonNull(compressed, "compressed"); - long size = frameContentSize(compressed); - if (size == CONTENTSIZE_UNKNOWN) { - throw new ZstdException("decompressed size not stored in frame; call decompress(src, maxSize)"); - } - if (size == CONTENTSIZE_ERROR) { - throw new ZstdException("not a valid zstd frame"); - } + long size = requireStoredContentSize(frameContentSize(compressed)); return decompress(compressed, Math.toIntExact(size)); } @@ -102,6 +96,16 @@ public static long decompressedSize(MemorySegment frame) { } catch (Throwable t) { throw NativeCall.rethrow(t); } + return requireStoredContentSize(size); + } + + /// Maps a raw `ZSTD_getFrameContentSize` / `ZSTD_findDecompressedSize` result to + /// a usable length, turning zstd's negative sentinels into exceptions. + /// + /// @param size a content size, or a `CONTENTSIZE_UNKNOWN` / `CONTENTSIZE_ERROR` sentinel + /// @return `size` when it is a real length + /// @throws ZstdException if the size is not stored, or the input is not valid zstd data + static long requireStoredContentSize(long size) { if (size == CONTENTSIZE_UNKNOWN) { throw new ZstdException("decompressed size not stored in frame"); } diff --git a/zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java b/zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java index f0843b6..fb70f65 100644 --- a/zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java +++ b/zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java @@ -73,6 +73,35 @@ public static long decompressedBound(MemorySegment data) { return decompressedBound(data, data.byteSize()); } + /// Exact total decompressed size of all frames in `data`, summed from each + /// frame header. + /// + /// Unlike [#decompressedBound(byte[])], which over-estimates so a buffer never + /// overflows, this is the precise combined size — but only when every frame + /// records it. Use it to size an exact destination; fall back to the bound when + /// a frame was produced by a stream that did not pledge its size. + /// + /// @param data one or more concatenated zstd frames + /// @return the exact combined decompressed size + /// @throws ZstdException if the input is not valid zstd data, or any frame does + /// not record its decompressed size + public static long decompressedSize(byte[] data) { + try (Arena arena = Arena.ofConfined()) { + return decompressedSize(Zstd.copyIn(arena, data), data.length); + } + } + + /// Exact total decompressed size of all frames in native `data`. + /// Otherwise identical to [#decompressedSize(byte[])]. + /// + /// @param data one or more concatenated zstd frames + /// @return the exact combined decompressed size + /// @throws ZstdException if the input is not valid zstd data, or any frame does + /// not record its decompressed size + public static long decompressedSize(MemorySegment data) { + return decompressedSize(data, data.byteSize()); + } + /// Dictionary id recorded in the first frame's header. /// /// @param data a zstd frame @@ -92,6 +121,34 @@ public static ZstdDictionaryId dictId(MemorySegment data) { return dictId(data, data.byteSize()); } + /// Size in bytes of the first frame's header, without parsing it. + /// + /// A lighter probe than [#header(byte[])]: it reads only the few leading bytes + /// needed to compute the header length (as few as 5 for a standard frame), so + /// you can learn how many bytes to buffer before a full [#header(byte[])] parse. + /// For an already-parsed header the same value is [ZstdFrameHeader#headerSize()]. + /// + /// @param data the start of a zstd frame, at least its first few bytes + /// @return the header size in bytes + /// @throws ZstdException if `data` is too short to determine the header size, or + /// is not a valid zstd frame + public static long headerSize(byte[] data) { + try (Arena arena = Arena.ofConfined()) { + return headerSize(Zstd.copyIn(arena, data), data.length); + } + } + + /// Size in bytes of the first frame's header in native `data`. + /// Otherwise identical to [#headerSize(byte[])]. + /// + /// @param data the start of a zstd frame, at least its first few bytes + /// @return the header size in bytes + /// @throws ZstdException if `data` is too short to determine the header size, or + /// is not a valid zstd frame + public static long headerSize(MemorySegment data) { + return headerSize(data, data.byteSize()); + } + /// Parses the header of the first frame in `data` without decompressing it. /// /// @param data a complete zstd frame (or at least its header) @@ -210,10 +267,22 @@ private static long decompressedBound(MemorySegment data, long size) { } catch (Throwable t) { throw NativeCall.rethrow(t); } - if (bound == Zstd.CONTENTSIZE_ERROR) { - throw new ZstdException("not valid zstd data"); + // ZSTD_decompressBound never returns CONTENTSIZE_UNKNOWN, only a bound or the error sentinel + return Zstd.requireStoredContentSize(bound); + } + + private static long decompressedSize(MemorySegment data, long size) { + long total; + try { + total = (long) Bindings.FIND_DECOMPRESSED_SIZE.invokeExact(data, size); + } catch (Throwable t) { + throw NativeCall.rethrow(t); } - return bound; + return Zstd.requireStoredContentSize(total); + } + + private static long headerSize(MemorySegment data, long size) { + return NativeCall.checkReturnValue(() -> (long) Bindings.FRAME_HEADER_SIZE.invokeExact(data, size)); } private static ZstdDictionaryId dictId(MemorySegment data, long size) { diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java index 465efb2..fa5d102 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java @@ -94,6 +94,121 @@ void rejectsGarbage() { } } + @Nested + class DecompressedSize { + + @Test + void equalsContentSizeForSingleFrame() { + // Given one frame that records its content size + byte[] frame = Zstd.compress(PAYLOAD); + + // Then the exact decompressed size is the payload length + assertThat(ZstdFrame.decompressedSize(frame)).isEqualTo(PAYLOAD.length); + } + + @Test + void sumsAcrossConcatenatedFrames() throws Exception { + // Given two frames back to back + byte[] second = "second payload".getBytes(StandardCharsets.UTF_8); + ByteArrayOutputStream both = new ByteArrayOutputStream(); + both.write(Zstd.compress(PAYLOAD)); + both.write(Zstd.compress(second)); + + // Then the total is the sum of both decompressed sizes + assertThat(ZstdFrame.decompressedSize(both.toByteArray())) + .isEqualTo((long) PAYLOAD.length + second.length); + } + + @Test + void matchesThroughTheSegmentOverload() { + // Given a frame in a native segment + byte[] frame = Zstd.compress(PAYLOAD); + try (Arena arena = Arena.ofConfined()) { + MemorySegment seg = Zstd.copyIn(arena, frame); + + // Then the segment overload agrees with the byte[] overload + assertThat(ZstdFrame.decompressedSize(seg)).isEqualTo(ZstdFrame.decompressedSize(frame)); + } + } + + @Test + void failsWhenAFrameDoesNotRecordItsSize() throws Exception { + // Given a streamed frame with no pledged size + ByteArrayOutputStream sink = new ByteArrayOutputStream(); + try (ZstdOutputStream zout = new ZstdOutputStream(sink)) { + zout.write(PAYLOAD); + } + byte[] frame = sink.toByteArray(); + + // When the exact total is requested + ThrowingCallable result = () -> ZstdFrame.decompressedSize(frame); + + // Then it fails because the size was never recorded + assertThatThrownBy(result) + .isInstanceOf(ZstdException.class) + .hasMessageContaining("not stored"); + } + + @Test + void rejectsGarbage() { + // Given bytes that are not valid zstd data + byte[] garbage = "xx".getBytes(StandardCharsets.UTF_8); + + // When the exact total is requested + ThrowingCallable result = () -> ZstdFrame.decompressedSize(garbage); + + // Then it fails + assertThatThrownBy(result).isInstanceOf(ZstdException.class); + } + } + + @Nested + class HeaderSize { + + @Test + void matchesTheParsedHeaderSize() { + // Given a frame + byte[] frame = Zstd.compress(PAYLOAD); + + // Then the light probe agrees with the fully parsed header + assertThat(ZstdFrame.headerSize(frame)).isEqualTo(ZstdFrame.header(frame).headerSize()); + } + + @Test + void computedFromJustTheHeaderPrefix() { + // Given only the first 5 bytes of a frame (the minimum prefix) + byte[] frame = Zstd.compress(PAYLOAD); + byte[] prefix = Arrays.copyOf(frame, 5); + + // Then the header size can still be determined, matching the full frame + assertThat(ZstdFrame.headerSize(prefix)).isEqualTo(ZstdFrame.headerSize(frame)); + } + + @Test + void matchesThroughTheSegmentOverload() { + // Given a frame in a native segment + byte[] frame = Zstd.compress(PAYLOAD); + try (Arena arena = Arena.ofConfined()) { + MemorySegment seg = Zstd.copyIn(arena, frame); + + // Then the segment overload agrees with the byte[] overload + assertThat(ZstdFrame.headerSize(seg)).isEqualTo(ZstdFrame.headerSize(frame)); + } + } + + @Test + void rejectsInputShorterThanThePrefix() { + // Given fewer bytes than the header-size prefix + byte[] tooShort = Arrays.copyOf(Zstd.compress(PAYLOAD), 2); + + // When the header size is requested + ThrowingCallable result = () -> ZstdFrame.headerSize(tooShort); + + // Then it fails instead of guessing + assertThatThrownBy(result).isInstanceOf(ZstdException.class); + } + } + @Nested class Header {