Skip to content

Commit 7108536

Browse files
dfa1claude
andcommitted
feat: memory accounting — sizeOf() + estimate*Size
- live memory: sizeOf() on ZstdCompressCtx/DecompressCtx, ZstdCompressDict/ DecompressDict, ZstdCompressStream/DecompressStream (ZSTD_sizeof_C/DCtx, ZSTD_sizeof_C/DDict) - static estimates for budgeting before allocation: Zstd.estimateCompress/ DecompressContextSize, estimateCompress/DecompressDictSize (ZSTD_estimate*Size) - 68/186 symbols bound Tests: estimates positive and monotonic in level; context size grows after use; contexts/dicts/streams all report a positive live size. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent b3b4d4d commit 7108536

9 files changed

Lines changed: 227 additions & 0 deletions

File tree

zstd/src/main/java/io/github/dfa1/zstd/Bindings.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,30 @@ final class Bindings {
285285
NativeLibrary.lookup("ZDICT_getDictHeaderSize",
286286
FunctionDescriptor.of(JAVA_LONG, ADDRESS, JAVA_LONG));
287287

288+
// --- memory accounting ---
289+
290+
// size_t ZSTD_sizeof_CCtx / DCtx / CDict / DDict (live memory of an object)
291+
static final MethodHandle SIZEOF_CCTX =
292+
NativeLibrary.lookup("ZSTD_sizeof_CCtx", FunctionDescriptor.of(JAVA_LONG, ADDRESS));
293+
static final MethodHandle SIZEOF_DCTX =
294+
NativeLibrary.lookup("ZSTD_sizeof_DCtx", FunctionDescriptor.of(JAVA_LONG, ADDRESS));
295+
static final MethodHandle SIZEOF_CDICT =
296+
NativeLibrary.lookup("ZSTD_sizeof_CDict", FunctionDescriptor.of(JAVA_LONG, ADDRESS));
297+
static final MethodHandle SIZEOF_DDICT =
298+
NativeLibrary.lookup("ZSTD_sizeof_DDict", FunctionDescriptor.of(JAVA_LONG, ADDRESS));
299+
300+
// size_t ZSTD_estimateCCtxSize(int level) / ZSTD_estimateDCtxSize(void)
301+
static final MethodHandle ESTIMATE_CCTX_SIZE =
302+
NativeLibrary.lookup("ZSTD_estimateCCtxSize", FunctionDescriptor.of(JAVA_LONG, JAVA_INT));
303+
static final MethodHandle ESTIMATE_DCTX_SIZE =
304+
NativeLibrary.lookup("ZSTD_estimateDCtxSize", FunctionDescriptor.of(JAVA_LONG));
305+
// size_t ZSTD_estimateCDictSize(size_t dictSize, int level)
306+
static final MethodHandle ESTIMATE_CDICT_SIZE =
307+
NativeLibrary.lookup("ZSTD_estimateCDictSize", FunctionDescriptor.of(JAVA_LONG, JAVA_LONG, JAVA_INT));
308+
// size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dlm)
309+
static final MethodHandle ESTIMATE_DDICT_SIZE =
310+
NativeLibrary.lookup("ZSTD_estimateDDictSize", FunctionDescriptor.of(JAVA_LONG, JAVA_LONG, JAVA_INT));
311+
288312
private Bindings() {
289313
// no instances
290314
}

zstd/src/main/java/io/github/dfa1/zstd/Zstd.java

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,57 @@ public static int defaultCompressionLevel() {
162162
}
163163
}
164164

165+
/// Estimates the memory a compression context will use at `level`, before
166+
/// creating one — useful for budgeting.
167+
///
168+
/// @param level the compression level
169+
/// @return the estimated context size in bytes
170+
public static long estimateCompressContextSize(int level) {
171+
try {
172+
return (long) Bindings.ESTIMATE_CCTX_SIZE.invokeExact(level);
173+
} catch (Throwable t) {
174+
throw sneaky(t);
175+
}
176+
}
177+
178+
/// Estimates the memory a decompression context will use.
179+
///
180+
/// @return the estimated context size in bytes
181+
public static long estimateDecompressContextSize() {
182+
try {
183+
return (long) Bindings.ESTIMATE_DCTX_SIZE.invokeExact();
184+
} catch (Throwable t) {
185+
throw sneaky(t);
186+
}
187+
}
188+
189+
/// Estimates the memory a digested compression dictionary of `dictSize` bytes
190+
/// will use at `level`.
191+
///
192+
/// @param dictSize the raw dictionary size in bytes
193+
/// @param level the compression level
194+
/// @return the estimated digested-dictionary size in bytes
195+
public static long estimateCompressDictSize(long dictSize, int level) {
196+
try {
197+
return (long) Bindings.ESTIMATE_CDICT_SIZE.invokeExact(dictSize, level);
198+
} catch (Throwable t) {
199+
throw sneaky(t);
200+
}
201+
}
202+
203+
/// Estimates the memory a digested decompression dictionary of `dictSize`
204+
/// bytes will use.
205+
///
206+
/// @param dictSize the raw dictionary size in bytes
207+
/// @return the estimated digested-dictionary size in bytes
208+
public static long estimateDecompressDictSize(long dictSize) {
209+
try {
210+
return (long) Bindings.ESTIMATE_DDICT_SIZE.invokeExact(dictSize, 0); // ZSTD_dlm_byCopy
211+
} catch (Throwable t) {
212+
throw sneaky(t);
213+
}
214+
}
215+
165216
/// Runtime zstd version, e.g. `"1.6.0"`.
166217
///
167218
/// @return the linked zstd library version as an `x.y.z` string

zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressCtx.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,17 @@ public MemorySegment compress(Arena arena, MemorySegment src, ZstdCompressDict d
205205
return dst.asSlice(0, written);
206206
}
207207

208+
/// Current native memory used by this context, in bytes.
209+
///
210+
/// @return the live context size
211+
public long sizeOf() {
212+
try {
213+
return (long) Bindings.SIZEOF_CCTX.invokeExact(ptr());
214+
} catch (Throwable t) {
215+
throw rethrow(t);
216+
}
217+
}
218+
208219
@Override
209220
protected void tryClose(MemorySegment ptr) throws Throwable {
210221
long ignored = (long) Bindings.FREE_CCTX.invokeExact(ptr);

zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressDict.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,17 @@ public int id() {
6262
}
6363
}
6464

65+
/// Current native memory used by this digested dictionary, in bytes.
66+
///
67+
/// @return the live dictionary size
68+
public long sizeOf() {
69+
try {
70+
return (long) Bindings.SIZEOF_CDICT.invokeExact(ptr());
71+
} catch (Throwable t) {
72+
throw rethrow(t);
73+
}
74+
}
75+
6576
@Override
6677
protected void tryClose(MemorySegment ptr) throws Throwable {
6778
long ignored = (long) Bindings.FREE_CDICT.invokeExact(ptr);

zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressStream.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,17 @@ public ZstdFrameProgression progress() {
113113
}
114114
}
115115

116+
/// Current native memory used by this stream's context, in bytes.
117+
///
118+
/// @return the live context size
119+
public long sizeOf() {
120+
try {
121+
return (long) Bindings.SIZEOF_CCTX.invokeExact(ptr());
122+
} catch (Throwable t) {
123+
throw rethrow(t);
124+
}
125+
}
126+
116127
@Override
117128
protected void tryClose(MemorySegment ptr) throws Throwable {
118129
try {

zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressCtx.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,17 @@ public MemorySegment decompress(Arena arena, MemorySegment frame, ZstdDecompress
165165
return out;
166166
}
167167

168+
/// Current native memory used by this context, in bytes.
169+
///
170+
/// @return the live context size
171+
public long sizeOf() {
172+
try {
173+
return (long) Bindings.SIZEOF_DCTX.invokeExact(ptr());
174+
} catch (Throwable t) {
175+
throw rethrow(t);
176+
}
177+
}
178+
168179
@Override
169180
protected void tryClose(MemorySegment ptr) throws Throwable {
170181
long ignored = (long) Bindings.FREE_DCTX.invokeExact(ptr);

zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressDict.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,17 @@ public int id() {
4343
}
4444
}
4545

46+
/// Current native memory used by this digested dictionary, in bytes.
47+
///
48+
/// @return the live dictionary size
49+
public long sizeOf() {
50+
try {
51+
return (long) Bindings.SIZEOF_DDICT.invokeExact(ptr());
52+
} catch (Throwable t) {
53+
throw rethrow(t);
54+
}
55+
}
56+
4657
@Override
4758
protected void tryClose(MemorySegment ptr) throws Throwable {
4859
long ignored = (long) Bindings.FREE_DDICT.invokeExact(ptr);

zstd/src/main/java/io/github/dfa1/zstd/ZstdDecompressStream.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,17 @@ public ZstdStreamResult decompress(MemorySegment dst, MemorySegment src) {
6666
return new ZstdStreamResult(in.pos(), out.pos(), remaining);
6767
}
6868

69+
/// Current native memory used by this stream's context, in bytes.
70+
///
71+
/// @return the live context size
72+
public long sizeOf() {
73+
try {
74+
return (long) Bindings.SIZEOF_DCTX.invokeExact(ptr());
75+
} catch (Throwable t) {
76+
throw rethrow(t);
77+
}
78+
}
79+
6980
@Override
7081
protected void tryClose(MemorySegment ptr) throws Throwable {
7182
try {
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package io.github.dfa1.zstd;
2+
3+
import org.junit.jupiter.api.Nested;
4+
import org.junit.jupiter.api.Test;
5+
6+
import java.nio.charset.StandardCharsets;
7+
import java.util.ArrayList;
8+
import java.util.List;
9+
10+
import static org.assertj.core.api.Assertions.assertThat;
11+
12+
class ZstdMemoryTest {
13+
14+
private static final byte[] PAYLOAD = "memory accounting ".repeat(500).getBytes(StandardCharsets.UTF_8);
15+
16+
@Nested
17+
class Estimates {
18+
19+
@Test
20+
void contextEstimatesArePositive() {
21+
assertThat(Zstd.estimateCompressContextSize(3)).isPositive();
22+
assertThat(Zstd.estimateDecompressContextSize()).isPositive();
23+
}
24+
25+
@Test
26+
void higherLevelEstimatesAtLeastAsLarge() {
27+
assertThat(Zstd.estimateCompressContextSize(19))
28+
.isGreaterThanOrEqualTo(Zstd.estimateCompressContextSize(1));
29+
}
30+
31+
@Test
32+
void dictionaryEstimatesArePositive() {
33+
assertThat(Zstd.estimateCompressDictSize(64 * 1024, 3)).isPositive();
34+
assertThat(Zstd.estimateDecompressDictSize(64 * 1024)).isPositive();
35+
}
36+
}
37+
38+
@Nested
39+
class LiveSizes {
40+
41+
@Test
42+
void contextSizeGrowsAfterUse() {
43+
try (ZstdCompressCtx ctx = new ZstdCompressCtx()) {
44+
long before = ctx.sizeOf();
45+
ctx.compress(PAYLOAD);
46+
long after = ctx.sizeOf();
47+
assertThat(before).isPositive();
48+
assertThat(after).isGreaterThanOrEqualTo(before);
49+
}
50+
}
51+
52+
@Test
53+
void decompressContextHasSize() {
54+
try (ZstdDecompressCtx ctx = new ZstdDecompressCtx()) {
55+
assertThat(ctx.sizeOf()).isPositive();
56+
}
57+
}
58+
59+
@Test
60+
void digestedDictionariesHaveSize() {
61+
ZstdDictionary dict = trainDict();
62+
try (ZstdCompressDict cdict = new ZstdCompressDict(dict);
63+
ZstdDecompressDict ddict = new ZstdDecompressDict(dict)) {
64+
assertThat(cdict.sizeOf()).isPositive();
65+
assertThat(ddict.sizeOf()).isPositive();
66+
}
67+
}
68+
69+
@Test
70+
void streamsReportContextSize() {
71+
try (ZstdCompressStream cs = new ZstdCompressStream();
72+
ZstdDecompressStream ds = new ZstdDecompressStream()) {
73+
assertThat(cs.sizeOf()).isPositive();
74+
assertThat(ds.sizeOf()).isPositive();
75+
}
76+
}
77+
78+
private ZstdDictionary trainDict() {
79+
List<byte[]> samples = new ArrayList<>();
80+
for (int i = 0; i < 2000; i++) {
81+
samples.add(("{\"id\":" + i + ",\"k\":\"v" + (i % 20) + "\"}").getBytes(StandardCharsets.UTF_8));
82+
}
83+
return ZstdDictionary.train(samples, 8 * 1024);
84+
}
85+
}
86+
}

0 commit comments

Comments
 (0)