|
2 | 2 |
|
3 | 3 | import io.github.dfa1.vortex.core.DType; |
4 | 4 | import io.github.dfa1.vortex.core.PType; |
| 5 | +import io.github.dfa1.vortex.core.VortexException; |
5 | 6 | import io.github.dfa1.vortex.core.VortexFormat; |
6 | 7 | import io.github.dfa1.vortex.reader.array.ListArray; |
7 | 8 | import io.github.dfa1.vortex.reader.array.ListViewArray; |
|
15 | 16 | import java.util.List; |
16 | 17 |
|
17 | 18 | import static org.assertj.core.api.Assertions.assertThat; |
| 19 | +import static org.assertj.core.api.Assertions.assertThatThrownBy; |
18 | 20 |
|
19 | 21 | /// Integration test: reads real Vortex files from the public S3 compatibility bucket |
20 | 22 | /// via HTTP Range requests and validates structure + data. |
|
23 | 25 | @Tag("integration") |
24 | 26 | class VortexHttpReaderIT { |
25 | 27 |
|
26 | | - private static final URI BASE = URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/v0.72.0/arrays/"); |
| 28 | + private static final URI BASE = URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/v0.75.0/arrays/"); |
27 | 29 |
|
28 | 30 | private static final URI TPCH_LINEITEM = BASE.resolve("tpch_lineitem.compact.vortex"); |
29 | 31 |
|
@@ -78,13 +80,71 @@ void scan_forVortex_decodesAllRows() throws Exception { |
78 | 80 | assertThat(totalRows).isGreaterThan(0); |
79 | 81 | } |
80 | 82 |
|
81 | | - // vortex.masked / vortex.patched / vortex.variant: IDs known, decoders not yet implemented. |
82 | | - // No S3 fixture exists in v0.72.0 — enable these tests once fixtures are published. |
| 83 | + // Smoke test: every published encoding fixture must open and decode all its rows. |
| 84 | + // Catches upstream wire-format drift across the full encoding set in one place; the |
| 85 | + // structural tests below (for/list/listview) assert shape for the trickier layouts. |
| 86 | + // Excludes the multi-MB clickbench/tpch fixtures (covered by dedicated tests) to keep |
| 87 | + // this fast over HTTP. |
| 88 | + @ParameterizedTest |
| 89 | + @ValueSource(strings = { |
| 90 | + "alp.vortex", "alprd.vortex", "bitpacked.vortex", "booleans.vortex", "bytebool.vortex", |
| 91 | + "chunked.vortex", "constant.vortex", "datetime.vortex", "datetimeparts.vortex", |
| 92 | + "decimal.vortex", "decimal_byte_parts.vortex", "dict.vortex", "fixed_size_list.vortex", |
| 93 | + "fsst.vortex", "null.vortex", "pco.vortex", "primitives.vortex", "rle.vortex", |
| 94 | + "runend.vortex", "sequence.vortex", "sparse.vortex", "struct_nested.vortex", |
| 95 | + "varbin.vortex", "varbinview.vortex", "zigzag.vortex" |
| 96 | + // zstd.vortex omitted: dictionary-compressed, unsupported by the pure-Java decoder |
| 97 | + // (see scan_zstdVortex_rejectsDictionaryCompression). |
| 98 | + }) |
| 99 | + void scan_publishedFixture_decodesAllRows(String fixture) throws Exception { |
| 100 | + // Given |
| 101 | + URI uri = BASE.resolve(fixture); |
| 102 | + |
| 103 | + // When |
| 104 | + long totalRows = 0; |
| 105 | + try (var sut = VortexHttpReader.open(uri); |
| 106 | + var iter = sut.scan(ScanOptions.all())) { |
| 107 | + while (iter.hasNext()) { |
| 108 | + try (var c = iter.next()) { |
| 109 | + totalRows += c.rowCount(); |
| 110 | + } |
| 111 | + } |
| 112 | + } |
| 113 | + |
| 114 | + // Then |
| 115 | + assertThat(totalRows).isGreaterThan(0); |
| 116 | + } |
| 117 | + |
| 118 | + // The published zstd.vortex fixture is dictionary-compressed; the pure-Java decoder has no |
| 119 | + // Zstd dictionary support and must fail fast with a clear message rather than mis-decode. |
| 120 | + // Tracked by https://github.com/dfa1/vortex-java/issues/104 (upstream airlift/aircompressor#119). |
| 121 | + @Test |
| 122 | + void scan_zstdVortex_rejectsDictionaryCompression() throws Exception { |
| 123 | + // Given |
| 124 | + URI uri = BASE.resolve("zstd.vortex"); |
| 125 | + |
| 126 | + // When / Then |
| 127 | + try (var sut = VortexHttpReader.open(uri); |
| 128 | + var iter = sut.scan(ScanOptions.all())) { |
| 129 | + assertThatThrownBy(() -> { |
| 130 | + while (iter.hasNext()) { |
| 131 | + try (var c = iter.next()) { |
| 132 | + c.rowCount(); |
| 133 | + } |
| 134 | + } |
| 135 | + }) |
| 136 | + .isInstanceOf(VortexException.class) |
| 137 | + .hasMessageContaining("dictionary-compressed Zstd segments are not supported"); |
| 138 | + } |
| 139 | + } |
| 140 | + |
| 141 | + // vortex.masked / vortex.patched / vortex.variant: decoders implemented, but no S3 fixture |
| 142 | + // is published (still absent at v0.75.0) — enable this test once fixtures exist upstream. |
83 | 143 |
|
84 | | - @Disabled("no S3 fixture in v0.72.0") |
| 144 | + @Disabled("no S3 fixture through v0.75.0") |
85 | 145 | @ParameterizedTest |
86 | 146 | @ValueSource(strings = {"masked.vortex", "patched.vortex", "variant.vortex"}) |
87 | | - void scan_unimplementedEncoding_decodesAllRows(String fixture) throws Exception { |
| 147 | + void scan_unpublishedFixture_decodesAllRows(String fixture) throws Exception { |
88 | 148 | // Given |
89 | 149 | URI uri = BASE.resolve(fixture); |
90 | 150 |
|
|
0 commit comments