Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,17 @@ git tags, which trigger publication to Maven Central.
same prefix to decode. Binds `ZSTD_CCtx_refPrefix` / `ZSTD_DCtx_refPrefix`.
Segment-only by design — heap callers that need a copy should use
`loadDictionary` instead.
- `Zstd.dictId(byte[])` / `Zstd.dictId(MemorySegment)` — read the dictionary id
stamped in raw dictionary bytes without wrapping them in a `ZstdDictionary`.
Binds `ZSTD_getDictID_fromDict`.
- `ZstdDictionaryId` value type — a `record` wrapping the 32-bit dictionary id
with an unsigned `value()`, `isPresent()`, and the `NONE` sentinel for "no id".

### Changed
- Every dictionary-id accessor now returns `ZstdDictionaryId` instead of `int`:
`ZstdDictionary.id()`, `ZstdCompressDict.id()`, `ZstdDecompressDict.id()`,
`ZstdFrame.dictId(...)`, and `ZstdFrameHeader.dictId()`. The `0` sentinel is now
`ZstdDictionaryId.NONE`, and the id reads as unsigned via `value()`.

## [0.5]

Expand Down
6 changes: 3 additions & 3 deletions docs/supported.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ rather than the deprecated `ZSTD_getDecompressedSize`.
| Version | 2 / 2 | complete |
| Errors | 4 / 4 | complete: name, typed `ZstdErrorCode`, and `description()` |
| Reusable contexts | 6 / 8 | CCtx/DCtx create/free/compress/decompress |
| Dictionary — simple | 10 / 23 | raw + digested (CDict/DDict) + dict-id queries; `_advanced`/`_byReference`/`Begin` variants not bound |
| Dictionary — simple | 11 / 23 | raw + digested (CDict/DDict) + dict-id queries; `_advanced`/`_byReference`/`Begin` variants not bound |
| Dictionary training (ZDICT) | 8 / 12 | trainFromBuffer, cover/fastCover optimizers, finalizeDictionary, getDictHeaderSize |
| Streaming — compress | 3 / 22 | `ZstdOutputStream` (compressStream2 + buffer sizes) |
| Streaming — decompress | 3 / 15 | `ZstdInputStream` (decompressStream + buffer sizes) |
Expand Down Expand Up @@ -141,7 +141,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa
| `ZSTD_createCCtx_advanced` | — | — |
| `ZSTD_createDCtx_advanced` | — | — |

### Dictionary — simple (10/23)
### Dictionary — simple (11/23)

| Symbol | Bound | zstd-jni |
|---|:---:|:---:|
Expand All @@ -166,7 +166,7 @@ sequence-producer hooks vs this library's frame-inspection and typed-error surfa
| `ZSTD_decompressBegin_usingDict` | — | — |
| `ZSTD_getDictID_fromCDict` | ✅ | — |
| `ZSTD_getDictID_fromDDict` | ✅ | — |
| `ZSTD_getDictID_fromDict` | | ✅ |
| `ZSTD_getDictID_fromDict` | | ✅ |
| `ZSTD_getDictID_fromFrame` | ✅ | ✅ |

### Dictionary training, ZDICT (8/12)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.github.dfa1.zstd.it;

import io.github.dfa1.zstd.ZstdDictionaryId;
import io.github.dfa1.zstd.Zstd;
import io.github.dfa1.zstd.ZstdCompressCtx;
import io.github.dfa1.zstd.ZstdDecompressCtx;
Expand Down Expand Up @@ -250,11 +251,11 @@ void dictIdRidesWithFrame(Path file) {
}

// When
int frameDictId = ZstdFrame.dictId(frame);
ZstdDictionaryId frameDictId = ZstdFrame.dictId(frame);

// Then
assertThat(frameDictId).isEqualTo(dict.id());
assertThat(frameDictId).isNotZero();
assertThat(frameDictId).isNotEqualTo(ZstdDictionaryId.NONE);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.github.dfa1.zstd.it;

import com.github.luben.zstd.ZstdCompressCtx;
import io.github.dfa1.zstd.ZstdDictionaryId;
import io.github.dfa1.zstd.Zstd;
import io.github.dfa1.zstd.ZstdDictionary;
import io.github.dfa1.zstd.ZstdException;
Expand Down Expand Up @@ -219,11 +220,11 @@ void javaReadsDictIdFromJniDictFrame() {
byte[] frame = com.github.luben.zstd.Zstd.compress(record(7), jniDict);

// When
int dictId = ZstdFrame.dictId(frame);
ZstdDictionaryId dictId = ZstdFrame.dictId(frame);

// Then
assertThat(dictId).isEqualTo(dict.id());
assertThat(dictId).isNotZero();
assertThat(dictId).isNotEqualTo(ZstdDictionaryId.NONE);
}

private ZstdDictionary trainDict() {
Expand Down
4 changes: 4 additions & 0 deletions zstd/src/main/java/io/github/dfa1/zstd/Bindings.java
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,10 @@ final class Bindings {
// unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict*)
static final MethodHandle GET_DICT_ID_FROM_DDICT =
NativeLibrary.lookup("ZSTD_getDictID_fromDDict", FunctionDescriptor.of(JAVA_INT, ADDRESS));
// unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
static final MethodHandle GET_DICT_ID_FROM_DICT =
NativeLibrary.lookup("ZSTD_getDictID_fromDict",
FunctionDescriptor.of(JAVA_INT, ADDRESS, JAVA_LONG));

// ZSTD_bounds { size_t error; int lowerBound; int upperBound; } — returned by value
static final MemoryLayout BOUNDS_LAYOUT =
Expand Down
36 changes: 36 additions & 0 deletions zstd/src/main/java/io/github/dfa1/zstd/Zstd.java
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,42 @@ public static long decompressedSize(MemorySegment frame) {
return size;
}

/// Dictionary id stamped in raw dictionary `bytes`, read with the core
/// `libzstd` reader.
///
/// This reads the id a *dictionary* records, not the id a frame references —
/// for the latter use [ZstdFrame#dictId(byte[])]. It returns the same value as
/// [ZstdDictionary#id()] (which uses the equivalent `ZDICT` reader); prefer this
/// when you only hold raw bytes and do not want to wrap them in a
/// [ZstdDictionary].
///
/// @param bytes raw dictionary content
/// @return the dictionary id, or [ZstdDictionaryId#NONE] if `bytes` is not a standard zstd
/// dictionary (for example a raw/content-only dictionary with no header)
public static ZstdDictionaryId dictId(byte[] bytes) {
try (Arena arena = Arena.ofConfined()) {
return dictId(copyIn(arena, bytes), bytes.length);
}
}

/// Dictionary id stamped in native dictionary `bytes`, read with no copy.
/// Otherwise identical to [#dictId(byte[])].
///
/// @param bytes native dictionary content
/// @return the dictionary id, or [ZstdDictionaryId#NONE] if `bytes` is not a standard zstd dictionary
public static ZstdDictionaryId dictId(MemorySegment bytes) {
NativeCall.requireNative(bytes, "bytes");
return dictId(bytes, bytes.byteSize());
}

private static ZstdDictionaryId dictId(MemorySegment bytes, long size) {
try {
return ZstdDictionaryId.of((int) Bindings.GET_DICT_ID_FROM_DICT.invokeExact(bytes, size));
} catch (Throwable t) {
throw NativeCall.rethrow(t);
}
}

/// Maximum compressed size for an input of `srcSize` bytes — the buffer
/// size guaranteed to never overflow during compression.
///
Expand Down
6 changes: 3 additions & 3 deletions zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressDict.java
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,10 @@ public int level() {

/// The dictionary id stamped into frames compressed with this dictionary.
///
/// @return the dictionary id, or `0` for a content-only dictionary
public int id() {
/// @return the dictionary id, or [ZstdDictionaryId#NONE] for a content-only dictionary
public ZstdDictionaryId id() {
try {
return (int) Bindings.GET_DICT_ID_FROM_CDICT.invokeExact(ptr());
return ZstdDictionaryId.of((int) Bindings.GET_DICT_ID_FROM_CDICT.invokeExact(ptr()));
} catch (Throwable t) {
throw NativeCall.rethrow(t);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ private static MemorySegment create(MemorySegment dict) {

/// The dictionary id this dictionary decodes frames for.
///
/// @return the dictionary id, or `0` for a content-only dictionary
public int id() {
/// @return the dictionary id, or [ZstdDictionaryId#NONE] for a content-only dictionary
public ZstdDictionaryId id() {
try {
return (int) Bindings.GET_DICT_ID_FROM_DDICT.invokeExact(ptr());
return ZstdDictionaryId.of((int) Bindings.GET_DICT_ID_FROM_DDICT.invokeExact(ptr()));
} catch (Throwable t) {
throw NativeCall.rethrow(t);
}
Expand Down
8 changes: 4 additions & 4 deletions zstd/src/main/java/io/github/dfa1/zstd/ZstdDictionary.java
Original file line number Diff line number Diff line change
Expand Up @@ -253,13 +253,13 @@ private static ZstdDictionary toDictionary(MemorySegment dictBuf, long produced,
}

/// The dictionary id zstd stamps into frames compressed with this dictionary,
/// or `0` for a raw/content-only dictionary with no header.
/// or [ZstdDictionaryId#NONE] for a raw/content-only dictionary with no header.
///
/// @return the dictionary id, or `0` if none
public int id() {
/// @return the dictionary id, or [ZstdDictionaryId#NONE] if none
public ZstdDictionaryId id() {
try (Arena arena = Arena.ofConfined()) {
MemorySegment seg = Zstd.copyIn(arena, bytes);
return (int) Bindings.ZDICT_GET_DICT_ID.invokeExact(seg, (long) bytes.length);
return ZstdDictionaryId.of((int) Bindings.ZDICT_GET_DICT_ID.invokeExact(seg, (long) bytes.length));
} catch (Throwable t) {
throw NativeCall.rethrow(t);
}
Expand Down
41 changes: 41 additions & 0 deletions zstd/src/main/java/io/github/dfa1/zstd/ZstdDictionaryId.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package io.github.dfa1.zstd;

/// A zstd dictionary id: the 32-bit value a dictionary stamps into the frames it
/// compresses, letting a decompressor pick the matching dictionary.
///
/// zstd treats the id as a C `unsigned`, so the wire value can exceed
/// [Integer#MAX_VALUE]. The raw 32-bit pattern is kept in [#raw()]; read it as an
/// unsigned magnitude with [#value()]. The value `0` is the sentinel for "no
/// dictionary id" — a raw/content-only dictionary, or a frame that records none —
/// exposed as [#NONE] and tested with [#isPresent()].
///
/// @param raw the 32-bit id as zstd stores it, possibly negative when read as a
/// signed `int`; `0` means no id
public record ZstdDictionaryId(int raw) {

/// The absent id: no dictionary, or a dictionary with no recorded id.
public static final ZstdDictionaryId NONE = new ZstdDictionaryId(0);

/// Wraps a raw 32-bit id, returning [#NONE] for `0`.
///
/// @param raw the 32-bit id as zstd stores it
/// @return the wrapped id, or [#NONE] if `raw` is `0`
public static ZstdDictionaryId of(int raw) {
return raw == 0 ? NONE : new ZstdDictionaryId(raw);
}

/// Whether an id is present (non-zero).
///
/// @return `true` unless this is [#NONE]
public boolean isPresent() {
return raw != 0;
}

/// The id as an unsigned magnitude in `[0, 2^32)`, widening the raw 32-bit
/// pattern without sign extension.
///
/// @return the unsigned id value
public long value() {
return Integer.toUnsignedLong(raw);
}
}
16 changes: 8 additions & 8 deletions zstd/src/main/java/io/github/dfa1/zstd/ZstdFrame.java
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ public static long decompressedBound(MemorySegment data) {
/// Dictionary id recorded in the first frame's header.
///
/// @param data a zstd frame
/// @return the dictionary id, or `0` if the frame uses no dictionary or does
/// not record one
public static int dictId(byte[] data) {
/// @return the dictionary id, or [ZstdDictionaryId#NONE] if the frame uses no dictionary
/// or does not record one
public static ZstdDictionaryId dictId(byte[] data) {
try (Arena arena = Arena.ofConfined()) {
return dictId(Zstd.copyIn(arena, data), data.length);
}
Expand All @@ -87,8 +87,8 @@ public static int dictId(byte[] data) {
/// Dictionary id recorded in native frame `data`.
///
/// @param data a zstd frame
/// @return the dictionary id, or `0` if none
public static int dictId(MemorySegment data) {
/// @return the dictionary id, or [ZstdDictionaryId#NONE] if none
public static ZstdDictionaryId dictId(MemorySegment data) {
return dictId(data, data.byteSize());
}

Expand Down Expand Up @@ -178,7 +178,7 @@ private static ZstdFrameHeader header(MemorySegment data, long size) {
zfh.get(JAVA_INT, 16) & 0xFFFFFFFFL, // blockSizeMax
ZstdFrameType.of(zfh.get(JAVA_INT, 20)), // frameType
zfh.get(JAVA_INT, 24), // headerSize
zfh.get(JAVA_INT, 28), // dictID
ZstdDictionaryId.of(zfh.get(JAVA_INT, 28)), // dictID
zfh.get(JAVA_INT, 32) != 0); // checksumFlag
}
}
Expand Down Expand Up @@ -216,9 +216,9 @@ private static long decompressedBound(MemorySegment data, long size) {
return bound;
}

private static int dictId(MemorySegment data, long size) {
private static ZstdDictionaryId dictId(MemorySegment data, long size) {
try {
return (int) Bindings.GET_DICT_ID_FROM_FRAME.invokeExact(data, size);
return ZstdDictionaryId.of((int) Bindings.GET_DICT_ID_FROM_FRAME.invokeExact(data, size));
} catch (Throwable t) {
throw NativeCall.rethrow(t);
}
Expand Down
6 changes: 3 additions & 3 deletions zstd/src/main/java/io/github/dfa1/zstd/ZstdFrameHeader.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@
/// @param blockSizeMax the maximum block size used in the frame
/// @param frameType whether this is a standard or skippable frame
/// @param headerSize the size of the frame header in bytes
/// @param dictId the dictionary id, or `0` if none (for a skippable frame,
/// the magic variant 0..15)
/// @param dictId the dictionary id, or [ZstdDictionaryId#NONE] if none (for a
/// skippable frame, [ZstdDictionaryId#raw()] is the magic variant 0..15)
/// @param hasChecksum whether a 4-byte content checksum follows the frame
public record ZstdFrameHeader(
long frameContentSize,
long windowSize,
long blockSizeMax,
ZstdFrameType frameType,
int headerSize,
int dictId,
ZstdDictionaryId dictId,
boolean hasChecksum) {

/// The decompressed size, if the frame records it.
Expand Down
48 changes: 48 additions & 0 deletions zstd/src/test/java/io/github/dfa1/zstd/ZstdDictionaryIdTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package io.github.dfa1.zstd;

import static org.assertj.core.api.Assertions.assertThat;

import org.junit.jupiter.api.Test;

class ZstdDictionaryIdTest {

@Test
void ofZeroIsTheNoneSingleton() {
// When wrapping the zero sentinel
ZstdDictionaryId sut = ZstdDictionaryId.of(0);

// Then it is the shared NONE and reports no id
assertThat(sut).isEqualTo(ZstdDictionaryId.NONE);
assertThat(sut.isPresent()).isFalse();
assertThat(sut.value()).isZero();
}

@Test
void ofNonZeroKeepsTheRawValueAndIsPresent() {
// When wrapping a positive id
ZstdDictionaryId sut = ZstdDictionaryId.of(42);

// Then it is present and exposes the value
assertThat(sut.isPresent()).isTrue();
assertThat(sut.raw()).isEqualTo(42);
assertThat(sut.value()).isEqualTo(42L);
}

@Test
void valueReadsTheRawPatternAsUnsigned() {
// Given an id whose top bit is set (negative as a signed int)
ZstdDictionaryId sut = ZstdDictionaryId.of(0xFFFFFFFF);

// Then raw stays the signed pattern while value widens without sign extension
assertThat(sut.raw()).isEqualTo(-1);
assertThat(sut.value()).isEqualTo(4_294_967_295L);
assertThat(sut.isPresent()).isTrue();
}

@Test
void equalIdsAreEqual() {
// Then records with the same raw value compare equal
assertThat(ZstdDictionaryId.of(7)).isEqualTo(ZstdDictionaryId.of(7));
assertThat(ZstdDictionaryId.of(7)).isNotEqualTo(ZstdDictionaryId.of(8));
}
}
12 changes: 6 additions & 6 deletions zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ void reportsContentSizeAndNoChecksumByDefault() {
assertThat(header.frameType()).isEqualTo(ZstdFrameType.STANDARD);
assertThat(header.contentSize()).hasValue(PAYLOAD.length);
assertThat(header.hasChecksum()).isFalse();
assertThat(header.dictId()).isZero();
assertThat(header.dictId()).isEqualTo(ZstdDictionaryId.NONE);
assertThat(header.windowSize()).isPositive();
}

Expand Down Expand Up @@ -199,11 +199,11 @@ void contentHasValueEqualityOverTheBytesNotArrayIdentity() {
}

@Nested
class DictId {
class DictIdLookup {

@Test
void isZeroForDictionarylessFrame() {
assertThat(ZstdFrame.dictId(Zstd.compress(PAYLOAD))).isZero();
void isNoneForDictionarylessFrame() {
assertThat(ZstdFrame.dictId(Zstd.compress(PAYLOAD))).isEqualTo(ZstdDictionaryId.NONE);
}

@Test
Expand All @@ -230,8 +230,8 @@ void matchesThroughTheSegmentOverload() {
try (Arena arena = Arena.ofConfined()) {
MemorySegment seg = Zstd.copyIn(arena, frame);

// Then the segment overload reports the same non-zero id
assertThat(ZstdFrame.dictId(seg)).isNotZero().isEqualTo(dict.id());
// Then the segment overload reports the same present id
assertThat(ZstdFrame.dictId(seg)).isNotEqualTo(ZstdDictionaryId.NONE).isEqualTo(dict.id());
}
}

Expand Down
Loading
Loading