Skip to content

Commit 8a1b5db

Browse files
dfa1claude
andcommitted
test(reader): bump HTTP fixtures to v0.75.0; smoke-test all encodings
Bump the S3 compatibility bucket from v0.72.0 to v0.75.0 and add a parameterized smoke test that opens and decodes every published encoding fixture, catching upstream wire-format drift across the full set. zstd.vortex is now dictionary-compressed, which the pure-Java decoder cannot handle (aircompressor has no Zstd dictionary support); assert the fail-fast VortexException in a dedicated test instead of decoding it. Tracked by #104 / airlift/aircompressor#119. masked/patched/variant fixtures are still unpublished at v0.75.0 (decoders already exist), so that test stays @disabled with an updated reason and a corrected, honest method name. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 365c406 commit 8a1b5db

1 file changed

Lines changed: 65 additions & 5 deletions

File tree

reader/src/test/java/io/github/dfa1/vortex/reader/VortexHttpReaderIT.java

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import io.github.dfa1.vortex.core.DType;
44
import io.github.dfa1.vortex.core.PType;
5+
import io.github.dfa1.vortex.core.VortexException;
56
import io.github.dfa1.vortex.core.VortexFormat;
67
import io.github.dfa1.vortex.reader.array.ListArray;
78
import io.github.dfa1.vortex.reader.array.ListViewArray;
@@ -15,6 +16,7 @@
1516
import java.util.List;
1617

1718
import static org.assertj.core.api.Assertions.assertThat;
19+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
1820

1921
/// Integration test: reads real Vortex files from the public S3 compatibility bucket
2022
/// via HTTP Range requests and validates structure + data.
@@ -23,7 +25,7 @@
2325
@Tag("integration")
2426
class VortexHttpReaderIT {
2527

26-
private static final URI BASE = URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/v0.72.0/arrays/");
28+
private static final URI BASE = URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/v0.75.0/arrays/");
2729

2830
private static final URI TPCH_LINEITEM = BASE.resolve("tpch_lineitem.compact.vortex");
2931

@@ -78,13 +80,71 @@ void scan_forVortex_decodesAllRows() throws Exception {
7880
assertThat(totalRows).isGreaterThan(0);
7981
}
8082

81-
// vortex.masked / vortex.patched / vortex.variant: IDs known, decoders not yet implemented.
82-
// No S3 fixture exists in v0.72.0 — enable these tests once fixtures are published.
83+
// Smoke test: every published encoding fixture must open and decode all its rows.
84+
// Catches upstream wire-format drift across the full encoding set in one place; the
85+
// structural tests below (for/list/listview) assert shape for the trickier layouts.
86+
// Excludes the multi-MB clickbench/tpch fixtures (covered by dedicated tests) to keep
87+
// this fast over HTTP.
88+
@ParameterizedTest
89+
@ValueSource(strings = {
90+
"alp.vortex", "alprd.vortex", "bitpacked.vortex", "booleans.vortex", "bytebool.vortex",
91+
"chunked.vortex", "constant.vortex", "datetime.vortex", "datetimeparts.vortex",
92+
"decimal.vortex", "decimal_byte_parts.vortex", "dict.vortex", "fixed_size_list.vortex",
93+
"fsst.vortex", "null.vortex", "pco.vortex", "primitives.vortex", "rle.vortex",
94+
"runend.vortex", "sequence.vortex", "sparse.vortex", "struct_nested.vortex",
95+
"varbin.vortex", "varbinview.vortex", "zigzag.vortex"
96+
// zstd.vortex omitted: dictionary-compressed, unsupported by the pure-Java decoder
97+
// (see scan_zstdVortex_rejectsDictionaryCompression).
98+
})
99+
void scan_publishedFixture_decodesAllRows(String fixture) throws Exception {
100+
// Given
101+
URI uri = BASE.resolve(fixture);
102+
103+
// When
104+
long totalRows = 0;
105+
try (var sut = VortexHttpReader.open(uri);
106+
var iter = sut.scan(ScanOptions.all())) {
107+
while (iter.hasNext()) {
108+
try (var c = iter.next()) {
109+
totalRows += c.rowCount();
110+
}
111+
}
112+
}
113+
114+
// Then
115+
assertThat(totalRows).isGreaterThan(0);
116+
}
117+
118+
// The published zstd.vortex fixture is dictionary-compressed; the pure-Java decoder has no
119+
// Zstd dictionary support and must fail fast with a clear message rather than mis-decode.
120+
// Tracked by https://github.com/dfa1/vortex-java/issues/104 (upstream airlift/aircompressor#119).
121+
@Test
122+
void scan_zstdVortex_rejectsDictionaryCompression() throws Exception {
123+
// Given
124+
URI uri = BASE.resolve("zstd.vortex");
125+
126+
// When / Then
127+
try (var sut = VortexHttpReader.open(uri);
128+
var iter = sut.scan(ScanOptions.all())) {
129+
assertThatThrownBy(() -> {
130+
while (iter.hasNext()) {
131+
try (var c = iter.next()) {
132+
c.rowCount();
133+
}
134+
}
135+
})
136+
.isInstanceOf(VortexException.class)
137+
.hasMessageContaining("dictionary-compressed Zstd segments are not supported");
138+
}
139+
}
140+
141+
// vortex.masked / vortex.patched / vortex.variant: decoders implemented, but no S3 fixture
142+
// is published (still absent at v0.75.0) — enable this test once fixtures exist upstream.
83143

84-
@Disabled("no S3 fixture in v0.72.0")
144+
@Disabled("no S3 fixture through v0.75.0")
85145
@ParameterizedTest
86146
@ValueSource(strings = {"masked.vortex", "patched.vortex", "variant.vortex"})
87-
void scan_unimplementedEncoding_decodesAllRows(String fixture) throws Exception {
147+
void scan_unpublishedFixture_decodesAllRows(String fixture) throws Exception {
88148
// Given
89149
URI uri = BASE.resolve(fixture);
90150

0 commit comments

Comments
 (0)