Skip to content

Commit 3f4a13b

Browse files
dfa1claude
andcommitted
feat: live stream progress — ZstdCompressStream.progress()
ZstdCompressStream.progress() returns a ZstdFrameProgression snapshot (ingested, consumed, produced, flushed, currentJobId, activeWorkers) via ZSTD_getFrameProgression (a struct returned by value through FFM) — byte counters for reporting on a long compression stream. Tests: counters are zero on a fresh stream; after a complete END step consumed == ingested == input length and flushed == produced; activeWorkers is 0 on the single-threaded build. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent fbe7dc4 commit 3f4a13b

4 files changed

Lines changed: 78 additions & 0 deletions

File tree

zstd/src/main/java/io/github/dfa1/zstd/Bindings.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,14 @@ final class Bindings {
166166
static final MethodHandle DPARAM_GET_BOUNDS =
167167
NativeLibrary.lookup("ZSTD_dParam_getBounds", FunctionDescriptor.of(BOUNDS_LAYOUT, JAVA_INT));
168168

169+
// ZSTD_frameProgression { u64 ingested, consumed, produced, flushed; u32 currentJobID, nbActiveWorkers; }
170+
private static final MemoryLayout FRAME_PROGRESSION_LAYOUT =
171+
MemoryLayout.structLayout(JAVA_LONG, JAVA_LONG, JAVA_LONG, JAVA_LONG, JAVA_INT, JAVA_INT);
172+
// ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx*) — returned by value
173+
static final MethodHandle GET_FRAME_PROGRESSION =
174+
NativeLibrary.lookup("ZSTD_getFrameProgression",
175+
FunctionDescriptor.of(FRAME_PROGRESSION_LAYOUT, ADDRESS));
176+
169177
// size_t ZSTD_compressStream2(ZSTD_CCtx*, ZSTD_outBuffer*, ZSTD_inBuffer*, ZSTD_EndDirective)
170178
static final MethodHandle COMPRESS_STREAM2 =
171179
NativeLibrary.lookup("ZSTD_compressStream2",

zstd/src/main/java/io/github/dfa1/zstd/ZstdCompressStream.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
import java.lang.foreign.Arena;
44
import java.lang.foreign.MemorySegment;
5+
import java.lang.foreign.SegmentAllocator;
6+
7+
import static java.lang.foreign.ValueLayout.JAVA_INT;
8+
import static java.lang.foreign.ValueLayout.JAVA_LONG;
59

610
/// A low-level, zero-copy streaming compressor driven with native
711
/// {@link MemorySegment} buffers — no heap `byte[]` bounce.
@@ -89,6 +93,26 @@ public ZstdStreamResult compress(MemorySegment dst, MemorySegment src, ZstdEndDi
8993
return new ZstdStreamResult(in.pos(), out.pos(), remaining);
9094
}
9195

96+
/// A live snapshot of how much this stream has ingested, compressed, produced,
97+
/// and flushed so far — useful for progress reporting on a long stream.
98+
///
99+
/// @return the current frame progression
100+
public ZstdFrameProgression progress() {
101+
try (Arena scratch = Arena.ofConfined()) {
102+
MemorySegment p = (MemorySegment) Bindings.GET_FRAME_PROGRESSION.invokeExact(
103+
(SegmentAllocator) scratch, ptr());
104+
return new ZstdFrameProgression(
105+
p.get(JAVA_LONG, 0), // ingested
106+
p.get(JAVA_LONG, 8), // consumed
107+
p.get(JAVA_LONG, 16), // produced
108+
p.get(JAVA_LONG, 24), // flushed
109+
p.get(JAVA_INT, 32), // currentJobID
110+
p.get(JAVA_INT, 36)); // nbActiveWorkers
111+
} catch (Throwable t) {
112+
throw rethrow(t);
113+
}
114+
}
115+
92116
@Override
93117
protected void tryClose(MemorySegment ptr) throws Throwable {
94118
try {
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package io.github.dfa1.zstd;
2+
3+
/// A live snapshot of a streaming compressor's progress, from
4+
/// `ZSTD_getFrameProgression`.
5+
///
6+
/// @param ingested input bytes read and buffered so far
7+
/// @param consumed input bytes actually compressed so far
8+
/// @param produced output bytes generated so far, including still-buffered
9+
/// @param flushed output bytes flushed out so far
10+
/// @param currentJobId current multithreading job id (0 when single-threaded)
11+
/// @param activeWorkers number of worker threads currently active (0 when single-threaded)
12+
public record ZstdFrameProgression(
13+
long ingested,
14+
long consumed,
15+
long produced,
16+
long flushed,
17+
int currentJobId,
18+
int activeWorkers) {
19+
}

zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentStreamTest.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,33 @@ private byte[] drive(byte[] input, int chunk, boolean compress) {
8888
}
8989
}
9090

91+
@Nested
92+
class Progress {
93+
94+
@Test
95+
void tracksByteCounters() {
96+
byte[] original = "progress payload ".repeat(500).getBytes(StandardCharsets.UTF_8);
97+
try (Arena arena = Arena.ofConfined();
98+
ZstdCompressStream cs = new ZstdCompressStream()) {
99+
100+
MemorySegment src = segment(arena, original);
101+
MemorySegment dst = arena.allocate(Zstd.compressBound(original.length));
102+
103+
// fresh stream: nothing moved yet
104+
assertThat(cs.progress().consumed()).isZero();
105+
106+
ZstdStreamResult r = cs.compress(dst, src, ZstdEndDirective.END);
107+
108+
// after a complete END step the counters reflect the whole frame
109+
ZstdFrameProgression p = cs.progress();
110+
assertThat(p.consumed()).isEqualTo(original.length);
111+
assertThat(p.ingested()).isEqualTo(original.length);
112+
assertThat(p.flushed()).isEqualTo(r.bytesProduced());
113+
assertThat(p.activeWorkers()).isZero(); // single-threaded build
114+
}
115+
}
116+
}
117+
91118
@Nested
92119
class WithDictionary {
93120

0 commit comments

Comments
 (0)