diff --git a/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java b/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java index a501290..9c7f9bc 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/RefPrefixTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.zstd; -import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static io.github.dfa1.zstd.ZstdTestSupport.bytesOf; +import static io.github.dfa1.zstd.ZstdTestSupport.segmentOf; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -21,8 +22,8 @@ void roundTripsWithMatchingPrefix() { try (Arena arena = Arena.ofConfined(); ZstdCompressCtx cctx = new ZstdCompressCtx(); ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { - MemorySegment prefix = copy(arena, prefixBytes); - MemorySegment src = copy(arena, dataBytes); + MemorySegment prefix = segmentOf(arena, prefixBytes); + MemorySegment src = segmentOf(arena, dataBytes); MemorySegment frame = arena.allocate(Zstd.compressBound(dataBytes.length)); MemorySegment out = arena.allocate(dataBytes.length); @@ -34,7 +35,7 @@ void roundTripsWithMatchingPrefix() { // Then assertThat(m).isEqualTo(dataBytes.length); - assertThat(bytes(out, (int) m)).isEqualTo(dataBytes); + assertThat(bytesOf(out, (int) m)).isEqualTo(dataBytes); } } @@ -47,8 +48,8 @@ void prefixIsAppliedAndRequiredToDecode() { // for random content.) byte[] random = randomBytes(0xBEEF, 16384); try (Arena arena = Arena.ofConfined()) { - MemorySegment prefix = copy(arena, random); - MemorySegment src = copy(arena, random); + MemorySegment prefix = segmentOf(arena, random); + MemorySegment src = segmentOf(arena, random); long baseline; try (ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { @@ -58,7 +59,7 @@ void prefixIsAppliedAndRequiredToDecode() { try (ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { cctx.refPrefix(prefix); MemorySegment f = cctx.compress(arena, src); - frame = bytes(f, (int) f.byteSize()); + frame = bytesOf(f, (int) f.byteSize()); } // Then — the prefix slashes the frame (16 KiB random → tens of bytes) @@ -68,16 +69,16 @@ void prefixIsAppliedAndRequiredToDecode() { try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { MemorySegment out = arena.allocate(random.length); dctx.refPrefix(prefix); - long m = dctx.decompress(out, copy(arena, frame)); - assertThat(bytes(out, (int) m)).isEqualTo(random); + long m = dctx.decompress(out, segmentOf(arena, frame)); + assertThat(bytesOf(out, (int) m)).isEqualTo(random); } // But — it cannot be recovered without the prefix boolean reproduced; try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { MemorySegment out = arena.allocate(random.length); - long m = dctx.decompress(out, copy(arena, frame)); - reproduced = Arrays.equals(bytes(out, (int) m), random); + long m = dctx.decompress(out, segmentOf(arena, frame)); + reproduced = Arrays.equals(bytesOf(out, (int) m), random); } catch (ZstdException e) { reproduced = false; } @@ -91,8 +92,8 @@ void clearingPrefixWithNullCompressesPlainly() { byte[] dataBytes = "the quick brown fox".getBytes(); try (Arena arena = Arena.ofConfined(); ZstdCompressCtx cctx = new ZstdCompressCtx()) { - MemorySegment prefix = copy(arena, "a prior version of the text".getBytes()); - MemorySegment src = copy(arena, dataBytes); + MemorySegment prefix = segmentOf(arena, "a prior version of the text".getBytes()); + MemorySegment src = segmentOf(arena, dataBytes); MemorySegment frame = arena.allocate(Zstd.compressBound(dataBytes.length)); // When — set then clear the prefix before compressing @@ -105,7 +106,7 @@ void clearingPrefixWithNullCompressesPlainly() { try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { MemorySegment out = arena.allocate(dataBytes.length); long m = dctx.decompress(out, frame.asSlice(0, n)); - restored = bytes(out, (int) m); + restored = bytesOf(out, (int) m); } assertThat(restored).isEqualTo(dataBytes); } @@ -118,8 +119,8 @@ void prefixIsSingleUseAndDoesNotStickAcrossFrames() { byte[] random = randomBytes(0xCAFE, 16384); try (Arena arena = Arena.ofConfined(); ZstdCompressCtx cctx = new ZstdCompressCtx().level(19)) { - MemorySegment prefix = copy(arena, random); - MemorySegment src = copy(arena, random); + MemorySegment prefix = segmentOf(arena, random); + MemorySegment src = segmentOf(arena, random); MemorySegment first = arena.allocate(Zstd.compressBound(random.length)); MemorySegment second = arena.allocate(Zstd.compressBound(random.length)); @@ -137,7 +138,7 @@ void prefixIsSingleUseAndDoesNotStickAcrossFrames() { try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { MemorySegment out = arena.allocate(random.length); long m = dctx.decompress(out, second.asSlice(0, n2)); - restored = bytes(out, (int) m); + restored = bytesOf(out, (int) m); } assertThat(restored).isEqualTo(random); } @@ -221,7 +222,7 @@ void refPrefixReturnsTheSameContextForChaining() { try (Arena arena = Arena.ofConfined(); ZstdCompressCtx cctx = new ZstdCompressCtx(); ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { - MemorySegment prefix = copy(arena, "a prior version".getBytes()); + MemorySegment prefix = segmentOf(arena, "a prior version".getBytes()); // When setting and then clearing a prefix on both contexts ZstdCompressCtx cSet = cctx.refPrefix(prefix); @@ -243,7 +244,7 @@ void loadDictionaryFromSegmentReturnsTheSameContextForChaining() { try (Arena arena = Arena.ofConfined(); ZstdCompressCtx cctx = new ZstdCompressCtx(); ZstdDecompressCtx dctx = new ZstdDecompressCtx()) { - MemorySegment dict = copy(arena, "dictionary sample payload ".repeat(64).getBytes()); + MemorySegment dict = segmentOf(arena, "dictionary sample payload ".repeat(64).getBytes()); // When loading and then clearing a segment dictionary on both contexts ZstdCompressCtx cSet = cctx.loadDictionary(dict); @@ -259,18 +260,6 @@ void loadDictionaryFromSegmentReturnsTheSameContextForChaining() { } } - private static MemorySegment copy(Arena arena, byte[] src) { - MemorySegment seg = arena.allocate(src.length); - MemorySegment.copy(src, 0, seg, JAVA_BYTE, 0, src.length); - return seg; - } - - private static byte[] bytes(MemorySegment seg, int len) { - byte[] out = new byte[len]; - MemorySegment.copy(seg, JAVA_BYTE, 0, out, 0, len); - return out; - } - private static byte[] randomBytes(long seed, int n) { byte[] b = new byte[n]; new Random(seed).nextBytes(b); diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdDictionaryTest.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdDictionaryTest.java index 905054f..e993f27 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/ZstdDictionaryTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdDictionaryTest.java @@ -429,12 +429,6 @@ void rejectsHeapCompressDictSegment() { // Then it fails fast assertThatThrownBy(result).isInstanceOf(IllegalArgumentException.class); } - - private MemorySegment nativeDict(Arena arena, byte[] raw) { - MemorySegment seg = arena.allocate(raw.length); - MemorySegment.copy(raw, 0, seg, ValueLayout.JAVA_BYTE, 0, raw.length); - return seg; - } } @Nested @@ -614,12 +608,6 @@ void loadAndRefReturnTheSameDecompressContext() { assertThat(dctx.refDictionary(null)).isSameAs(dctx); } } - - private MemorySegment nativeDict(Arena arena, byte[] raw) { - MemorySegment seg = arena.allocate(raw.length); - MemorySegment.copy(raw, 0, seg, ValueLayout.JAVA_BYTE, 0, raw.length); - return seg; - } } private static byte[] record(int i) { @@ -629,4 +617,10 @@ private static byte[] record(int i) { + ",\"score\":" + (i * 7 % 1000) + ",\"tag\":\"event\"}").getBytes(StandardCharsets.UTF_8); } + + private static MemorySegment nativeDict(Arena arena, byte[] raw) { + MemorySegment seg = arena.allocate(raw.length); + MemorySegment.copy(raw, 0, seg, ValueLayout.JAVA_BYTE, 0, raw.length); + return seg; + } } diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java index e912fb8..d3aafaa 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdFrameTest.java @@ -9,10 +9,9 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; +import static io.github.dfa1.zstd.ZstdTestSupport.trainDictionary; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -409,11 +408,7 @@ void matchesThroughTheSegmentOverload() { } private ZstdDictionary trainDict() { - List samples = new ArrayList<>(); - for (int i = 0; i < 3000; i++) { - samples.add(("{\"id\":" + i + ",\"k\":\"v" + (i % 30) + "\"}").getBytes(StandardCharsets.UTF_8)); - } - return ZstdDictionary.train(samples, 8 * 1024); + return trainDictionary(3000); } } diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdMemoryTest.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdMemoryTest.java index ebf51f3..6d24988 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/ZstdMemoryTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdMemoryTest.java @@ -4,9 +4,8 @@ import org.junit.jupiter.api.Test; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; +import static io.github.dfa1.zstd.ZstdTestSupport.trainDictionary; import static org.assertj.core.api.Assertions.assertThat; class ZstdMemoryTest { @@ -76,11 +75,7 @@ void streamsReportContextSize() { } private ZstdDictionary trainDict() { - List samples = new ArrayList<>(); - for (int i = 0; i < 2000; i++) { - samples.add(("{\"id\":" + i + ",\"k\":\"v" + (i % 20) + "\"}").getBytes(StandardCharsets.UTF_8)); - } - return ZstdDictionary.train(samples, 8 * 1024); + return trainDictionary(2000); } } } diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentStreamTest.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentStreamTest.java index 2b68a0c..32c4864 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentStreamTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentStreamTest.java @@ -7,11 +7,11 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; import java.util.Random; -import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static io.github.dfa1.zstd.ZstdTestSupport.bytesOf; +import static io.github.dfa1.zstd.ZstdTestSupport.segmentOf; +import static io.github.dfa1.zstd.ZstdTestSupport.trainDictionary; import static org.assertj.core.api.Assertions.assertThat; class ZstdSegmentStreamTest { @@ -26,7 +26,7 @@ void roundTripsInOneCallEach() { ZstdCompressStream cs = new ZstdCompressStream(); ZstdDecompressStream ds = new ZstdDecompressStream()) { - MemorySegment src = segment(arena, original); + MemorySegment src = segmentOf(arena, original); MemorySegment dst = arena.allocate(Zstd.compressBound(original.length)); // When compressed in one END step @@ -42,7 +42,7 @@ void roundTripsInOneCallEach() { assertThat(d.isComplete()).isTrue(); assertThat(d.bytesProduced()).isEqualTo(original.length); - assertThat(bytes(out, d.bytesProduced())).isEqualTo(original); + assertThat(bytesOf(out, d.bytesProduced())).isEqualTo(original); } } } @@ -71,7 +71,7 @@ private byte[] drive(byte[] input, int chunk, boolean compress) { ZstdCompressStream cs = compress ? new ZstdCompressStream() : null; ZstdDecompressStream ds = compress ? null : new ZstdDecompressStream()) { - MemorySegment src = segment(arena, input); + MemorySegment src = segmentOf(arena, input); MemorySegment dst = arena.allocate(chunk); long srcOff = 0; ZstdStreamResult r; @@ -81,7 +81,7 @@ private byte[] drive(byte[] input, int chunk, boolean compress) { ? cs.compress(dst, srcSlice, ZstdEndDirective.END) : ds.decompress(dst, srcSlice); srcOff += r.bytesConsumed(); - collected.writeBytes(bytes(dst, r.bytesProduced())); + collected.writeBytes(bytesOf(dst, r.bytesProduced())); } while (!r.isComplete()); return collected.toByteArray(); } @@ -97,7 +97,7 @@ void tracksByteCounters() { try (Arena arena = Arena.ofConfined(); ZstdCompressStream cs = new ZstdCompressStream()) { - MemorySegment src = segment(arena, original); + MemorySegment src = segmentOf(arena, original); MemorySegment dst = arena.allocate(Zstd.compressBound(original.length)); // fresh stream: nothing moved yet @@ -127,35 +127,18 @@ void roundTripsAgainstDictionary() { ZstdCompressStream cs = new ZstdCompressStream(3, dict); ZstdDecompressStream ds = new ZstdDecompressStream(dict)) { - MemorySegment src = segment(arena, record); + MemorySegment src = segmentOf(arena, record); MemorySegment dst = arena.allocate(Zstd.compressBound(record.length)); ZstdStreamResult c = cs.compress(dst, src, ZstdEndDirective.END); MemorySegment out = arena.allocate(record.length); ds.decompress(out, dst.asSlice(0, c.bytesProduced())); - assertThat(bytes(out, record.length)).isEqualTo(record); + assertThat(bytesOf(out, record.length)).isEqualTo(record); } } private ZstdDictionary trainDict() { - List samples = new ArrayList<>(); - for (int i = 0; i < 3000; i++) { - samples.add(("{\"id\":" + i + ",\"user\":\"u" + (i % 30) + "\",\"event\":\"x\"}") - .getBytes(StandardCharsets.UTF_8)); - } - return ZstdDictionary.train(samples, 8 * 1024); + return trainDictionary(3000); } } - - private static MemorySegment segment(Arena arena, byte[] bytes) { - MemorySegment seg = arena.allocate(Math.max(bytes.length, 1)); - MemorySegment.copy(bytes, 0, seg, JAVA_BYTE, 0, bytes.length); - return seg; - } - - private static byte[] bytes(MemorySegment seg, long len) { - byte[] out = new byte[Math.toIntExact(len)]; - MemorySegment.copy(seg, JAVA_BYTE, 0, out, 0, out.length); - return out; - } } diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentTest.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentTest.java index 979aa15..8ba42c6 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdSegmentTest.java @@ -7,10 +7,10 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static io.github.dfa1.zstd.ZstdTestSupport.bytesOf; +import static io.github.dfa1.zstd.ZstdTestSupport.segmentOf; +import static io.github.dfa1.zstd.ZstdTestSupport.trainDictionary; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -77,7 +77,7 @@ class WithDictionary { @Test void roundTripsWithDigestedDictionary() { // Given a digested dictionary and a record in a native segment - ZstdDictionary dict = trainSmallDictionary(); + ZstdDictionary dict = trainDictionary(2000); byte[] record = "{\"id\":42,\"user\":\"u\",\"active\":true}".getBytes(StandardCharsets.UTF_8); try (Arena arena = Arena.ofConfined(); @@ -101,7 +101,7 @@ void roundTripsWithDigestedDictionary() { @Test void arenaAllocatingDecompressSizesOutputFromTheDigestedDictionaryFrame() { // Given a digested-dictionary frame that stores its decompressed size - ZstdDictionary dict = trainSmallDictionary(); + ZstdDictionary dict = trainDictionary(2000); byte[] record = "{\"id\":99,\"user\":\"u\",\"active\":false}".getBytes(StandardCharsets.UTF_8); try (Arena arena = Arena.ofConfined(); @@ -201,7 +201,7 @@ void decompressRejectsHeapSource() { @Test void compressWithDictionaryRejectsHeapSource() { - ZstdDictionary dict = trainSmallDictionary(); + ZstdDictionary dict = trainDictionary(2000); try (Arena arena = Arena.ofConfined(); ZstdCompressCtx sut = new ZstdCompressCtx(); ZstdCompressDict cdict = new ZstdCompressDict(dict)) { @@ -221,7 +221,7 @@ void compressWithDictionaryRejectsHeapSource() { @Test void compressWithDictionaryRejectsHeapDestination() { - ZstdDictionary dict = trainSmallDictionary(); + ZstdDictionary dict = trainDictionary(2000); try (Arena arena = Arena.ofConfined(); ZstdCompressCtx sut = new ZstdCompressCtx(); ZstdCompressDict cdict = new ZstdCompressDict(dict)) { @@ -241,7 +241,7 @@ void compressWithDictionaryRejectsHeapDestination() { @Test void decompressWithDictionaryRejectsHeapSource() { - ZstdDictionary dict = trainSmallDictionary(); + ZstdDictionary dict = trainDictionary(2000); try (Arena arena = Arena.ofConfined(); ZstdDecompressCtx sut = new ZstdDecompressCtx(); ZstdDecompressDict ddict = new ZstdDecompressDict(dict)) { @@ -261,7 +261,7 @@ void decompressWithDictionaryRejectsHeapSource() { @Test void decompressWithDictionaryRejectsHeapDestination() { - ZstdDictionary dict = trainSmallDictionary(); + ZstdDictionary dict = trainDictionary(2000); try (Arena arena = Arena.ofConfined(); ZstdDecompressCtx sut = new ZstdDecompressCtx(); ZstdDecompressDict ddict = new ZstdDecompressDict(dict)) { @@ -294,24 +294,4 @@ void decompressedSizeRejectsHeapFrame() { } } - private static MemorySegment segmentOf(Arena arena, byte[] bytes) { - MemorySegment seg = arena.allocate(Math.max(bytes.length, 1)); - MemorySegment.copy(bytes, 0, seg, JAVA_BYTE, 0, bytes.length); - return seg; - } - - private static byte[] bytesOf(MemorySegment seg, long len) { - byte[] out = new byte[Math.toIntExact(len)]; - MemorySegment.copy(seg, JAVA_BYTE, 0, out, 0, out.length); - return out; - } - - private static ZstdDictionary trainSmallDictionary() { - List samples = new ArrayList<>(); - for (int i = 0; i < 2000; i++) { - samples.add(("{\"id\":" + i + ",\"user\":\"u\",\"active\":" + (i % 2 == 0) + "}") - .getBytes(StandardCharsets.UTF_8)); - } - return ZstdDictionary.train(samples, 8 * 1024); - } } diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdStreamTest.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdStreamTest.java index 0b6baec..99e21ca 100644 --- a/zstd/src/test/java/io/github/dfa1/zstd/ZstdStreamTest.java +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdStreamTest.java @@ -16,6 +16,7 @@ import java.nio.charset.StandardCharsets; import java.util.Random; +import static io.github.dfa1.zstd.ZstdTestSupport.trainDictionary; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -141,11 +142,7 @@ void streamWithDictionaryDecodesWithOneShot() throws IOException { } private ZstdDictionary trainDict() { - java.util.List samples = new java.util.ArrayList<>(); - for (int i = 0; i < 3000; i++) { - samples.add(record(i)); - } - return ZstdDictionary.train(samples, 8 * 1024); + return trainDictionary(3000); } private byte[] record(int i) { diff --git a/zstd/src/test/java/io/github/dfa1/zstd/ZstdTestSupport.java b/zstd/src/test/java/io/github/dfa1/zstd/ZstdTestSupport.java new file mode 100644 index 0000000..762070e --- /dev/null +++ b/zstd/src/test/java/io/github/dfa1/zstd/ZstdTestSupport.java @@ -0,0 +1,51 @@ +package io.github.dfa1.zstd; + +import static java.lang.foreign.ValueLayout.JAVA_BYTE; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/// Shared low-level helpers for the segment-based tests: copy a `byte[]` into a +/// native segment, read it back out, and train a small dictionary from a common +/// structured schema. Kept in one place so the helpers cannot drift apart. +final class ZstdTestSupport { + + /// Copy `bytes` into a freshly allocated native segment in `arena`. + /// Allocates at least one byte so a zero-length payload still yields a valid + /// (non-null) segment. + static MemorySegment segmentOf(Arena arena, byte[] bytes) { + MemorySegment seg = arena.allocate(Math.max(bytes.length, 1)); + MemorySegment.copy(bytes, 0, seg, JAVA_BYTE, 0, bytes.length); + return seg; + } + + /// Read the first `len` bytes out of `seg` into a heap `byte[]`. + static byte[] bytesOf(MemorySegment seg, long len) { + byte[] out = new byte[Math.toIntExact(len)]; + MemorySegment.copy(seg, JAVA_BYTE, 0, out, 0, out.length); + return out; + } + + /// `int`-length overload of [ZstdTestSupport#bytesOf(MemorySegment,long)]. + static byte[] bytesOf(MemorySegment seg, int len) { + return bytesOf(seg, (long) len); + } + + /// Train a dictionary from `sampleCount` structured sample records using an + /// 8 KiB dictionary buffer. + static ZstdDictionary trainDictionary(int sampleCount) { + List samples = new ArrayList<>(); + for (int i = 0; i < sampleCount; i++) { + samples.add(("{\"id\":" + i + ",\"user\":\"user_" + (i % 100) + "\",\"event\":\"click\"}") + .getBytes(StandardCharsets.UTF_8)); + } + return ZstdDictionary.train(samples, 8 * 1024); + } + + private ZstdTestSupport() { + // no instances + } +}