Skip to content

Commit a110d8b

Browse files
dfa1claude
andcommitted
fix(stream): detect truncated frames; drop per-byte allocations
ZstdInputStream silently returned clean EOF when the underlying stream ended mid-frame, losing data without error. Track zstd's outstanding-input hint from decompressStream and throw ZstdException on a non-zero hint at EOF. Empty input stays a clean EOF. Also reuse a single-byte scratch buffer in read()/write(int) instead of allocating new byte[1] per call (streams are already non-thread-safe). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 08bca75 commit a110d8b

3 files changed

Lines changed: 60 additions & 9 deletions

File tree

zstd/src/main/java/io/github/dfa1/zstd/ZstdInputStream.java

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,14 @@ public final class ZstdInputStream extends InputStream {
3535
private final ZstdStreamBuffer outBufView = new ZstdStreamBuffer(arena);
3636
private final byte[] feed;
3737
private final byte[] hold;
38+
private final byte[] single = new byte[1];
3839
private int holdStart;
3940
private int holdEnd;
4041
private boolean inputEof;
4142
private boolean closed;
43+
/// zstd's "bytes still expected" hint from the last decompressStream call;
44+
/// non-zero at EOF means the final frame was truncated.
45+
private long lastHint;
4246

4347
/// Wraps `in`, decompressing the zstd frame it carries.
4448
///
@@ -84,9 +88,8 @@ private void loadDictionary(ZstdDictionary dictionary) {
8488

8589
@Override
8690
public int read() throws IOException {
87-
byte[] one = new byte[1];
88-
int n = read(one, 0, 1);
89-
return n == -1 ? -1 : (one[0] & 0xFF);
91+
int n = read(single, 0, 1);
92+
return n == -1 ? -1 : (single[0] & 0xFF);
9093
}
9194

9295
@Override
@@ -112,13 +115,20 @@ private boolean produce() throws IOException {
112115
int r = inputEof ? -1 : in.read(feed);
113116
if (r == -1) {
114117
inputEof = true;
118+
// The frame boundary is clean only when the last decompressStream
119+
// call reported nothing outstanding; otherwise the stream was cut
120+
// mid-frame and the remaining bytes are lost.
121+
if (lastHint != 0) {
122+
throw new ZstdException("truncated zstd stream: " + lastHint
123+
+ " more input byte(s) expected");
124+
}
115125
return false;
116126
}
117127
MemorySegment.copy(feed, 0, inSeg, JAVA_BYTE, 0, r);
118128
inBuf.set(inSeg, r, 0);
119129
}
120130
outBufView.set(outSeg, outCap, 0);
121-
Zstd.call(() -> (long) Bindings.DECOMPRESS_STREAM.invokeExact(
131+
lastHint = Zstd.call(() -> (long) Bindings.DECOMPRESS_STREAM.invokeExact(
122132
dctx, outBufView.segment(), inBuf.segment()));
123133
int produced = Math.toIntExact(outBufView.pos());
124134
if (produced > 0) {
@@ -127,10 +137,17 @@ private boolean produce() throws IOException {
127137
holdEnd = produced;
128138
return true;
129139
}
130-
// nothing produced: if input is exhausted and the underlying stream is
131-
// at EOF, there is no more output to come.
132-
if (inputEof && inBuf.pos() == inBuf.size()) {
133-
return false;
140+
// Nothing produced. If the decoder neither advanced its input nor wants
141+
// more, it cannot make progress on this input — stop to avoid spinning.
142+
if (inBuf.pos() == inBuf.size()) {
143+
if (inputEof) {
144+
if (lastHint != 0) {
145+
throw new ZstdException("truncated zstd stream: " + lastHint
146+
+ " more input byte(s) expected");
147+
}
148+
return false;
149+
}
150+
// input drained but frame wants more: loop to refill from `in`.
134151
}
135152
}
136153
}

zstd/src/main/java/io/github/dfa1/zstd/ZstdOutputStream.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public final class ZstdOutputStream extends OutputStream {
4141
private final ZstdStreamBuffer in = new ZstdStreamBuffer(arena);
4242
private final ZstdStreamBuffer outBuf = new ZstdStreamBuffer(arena);
4343
private final byte[] drain;
44+
private final byte[] single = new byte[1];
4445
private boolean closed;
4546

4647
/// Wraps `out`, compressing at the library default level.
@@ -124,7 +125,8 @@ private void loadDictionary(ZstdDictionary dictionary) {
124125

125126
@Override
126127
public void write(int b) throws IOException {
127-
write(new byte[]{(byte) b}, 0, 1);
128+
single[0] = (byte) b;
129+
write(single, 0, 1);
128130
}
129131

130132
@Override

zstd/src/test/java/io/github/dfa1/zstd/ZstdStreamTest.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import java.util.Random;
1313

1414
import static org.assertj.core.api.Assertions.assertThat;
15+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
1516

1617
class ZstdStreamTest {
1718

@@ -148,6 +149,37 @@ private byte[] record(int i) {
148149
}
149150
}
150151

152+
@Nested
153+
class Truncation {
154+
155+
@ParameterizedTest
156+
@ValueSource(ints = {1, 8, 64})
157+
void throwsWhenFinalFrameIsCutShort(int bytesDropped) throws IOException {
158+
// Given a valid frame with its tail bytes lopped off (random data so the
159+
// frame stays large enough to drop bytes from)
160+
byte[] original = randomBytes(50_000);
161+
byte[] frame = streamCompress(original, 6);
162+
byte[] cut = java.util.Arrays.copyOf(frame, frame.length - bytesDropped);
163+
164+
// When the streaming decompressor drains it
165+
// Then it reports the truncation instead of returning a clean EOF
166+
try (ZstdInputStream zin = new ZstdInputStream(new ByteArrayInputStream(cut))) {
167+
assertThatThrownBy(zin::readAllBytes)
168+
.isInstanceOf(ZstdException.class)
169+
.hasMessageContaining("truncated");
170+
}
171+
}
172+
173+
@Test
174+
void emptyInputIsCleanEofNotTruncation() throws IOException {
175+
// Given no input at all
176+
// When read, it is end-of-stream, not an error
177+
try (ZstdInputStream zin = new ZstdInputStream(new ByteArrayInputStream(new byte[0]))) {
178+
assertThat(zin.read()).isEqualTo(-1);
179+
}
180+
}
181+
}
182+
151183
@Nested
152184
class PledgedSize {
153185

0 commit comments

Comments
 (0)