diff --git a/.github/workflows/raincloud-conformance.yml b/.github/workflows/raincloud-conformance.yml new file mode 100644 index 00000000..1acdea37 --- /dev/null +++ b/.github/workflows/raincloud-conformance.yml @@ -0,0 +1,64 @@ +name: Raincloud conformance + +# Real-world cross-implementation conformance (issue #205): hydrates the Raincloud +# corpus (Vortex files written by the Python bindings + Parquet oracles) and runs +# RaincloudConformanceIntegrationTest against everything hydrated. Scheduled, not on +# the PR path: hydration depends on upstream dataset URLs that rot independently of +# our code, so a failure here is a finding to triage, not a broken build. + +on: + schedule: + - cron: '0 5 * * 1' # Mondays 05:00 UTC + workflow_dispatch: + inputs: + max-mb: + description: 'Per-slug artifact size cap in MB (0 = uncapped)' + default: '200' + +jobs: + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up Azul Zulu JDK 25 + uses: actions/setup-java@v5 + with: + distribution: zulu + java-version: '25' + + - name: Cache Maven repository + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + # keyed on the matrix + pinned tag: a corpus change rebuilds, otherwise + # weekly runs reuse artifacts instead of re-fetching upstream sources + - name: Cache Raincloud corpus + uses: actions/cache@v6 + with: + path: ~/.cache/raincloud + key: raincloud-corpus-${{ hashFiles('scripts/hydrate-raincloud-corpus.sh', 'integration/src/test/resources/raincloud/expected-status.csv') }} + + - name: Hydrate corpus + run: scripts/hydrate-raincloud-corpus.sh --max-mb "${{ github.event.inputs.max-mb || '200' }}" + + - name: Run conformance suite + run: ./mvnw verify -pl integration -am -Dit.test=RaincloudConformanceIntegrationTest + + - name: Publish per-slug results to job summary + if: always() + run: | + { + echo '## Raincloud conformance' + echo '```' + grep -hE "Tests run|slug" integration/target/failsafe-reports/*.txt || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index 6368bea8..914ee95e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CSV export renders unsigned integer columns (U8–U64) with their unsigned values — high-half values previously printed as two's-complement negatives (uci-wine `magnesium` U8 132 exported as -124), silent corruption found by the Raincloud conformance suite. ([#208](https://github.com/dfa1/vortex-java/issues/208)) - `fastlanes.rle` decodes F64/F32 value pools (`LazyRleDoubleArray`, `LazyRleFloatArray`) — files from the Python bindings RLE-encode double columns with long constant runs, which previously failed to scan with `unsupported ptype F64`. ([#209](https://github.com/dfa1/vortex-java/issues/209)) +- Lazy dict decode now covers I8/U8/I16/U16 value columns (`DictByteArray`, `DictShortArray`) — files from the Python bindings dict-encode narrow-integer columns, which previously failed to scan with `unsupported ptype for lazy dict`. ([#206](https://github.com/dfa1/vortex-java/issues/206)) +- CSV export handles all-null (`DType.Null`) columns as empty fields instead of throwing `unsupported array type: NullArray`. ([#211](https://github.com/dfa1/vortex-java/issues/211)) +- Null rows no longer silently decode as values: wrapper decoders now propagate the row validity that files from the Python bindings carry deep in the encoding tree. Three representations were dropped — a trailing validity child on `fastlanes.bitpacked` (reached through `vortex.alp`/`vortex.zigzag`/`fastlanes.for`, which delegate validity to their encoded child per the Rust `ValidityChild` contract), dict pools with invalid slots, and dict codes with their own validity — across both the eager `vortex.dict` decoder and the lazy dict layout path. Found on real data: penguins and kepler exported invented values (`32.1`, `0.0`) for thousands of null cells. ([#210](https://github.com/dfa1/vortex-java/issues/210)) + +### Added + +- Real-world conformance suite against the [Raincloud](https://github.com/spiraldb/raincloud) corpus: 247 public datasets whose Vortex files are written by the Python bindings, each validated value-for-value against its Parquet sibling. `scripts/hydrate-raincloud-corpus.sh` hydrates any subset (cache → mirror → local build), `RaincloudConformanceIntegrationTest` tests whatever is hydrated against the checked-in per-dataset status matrix, and a weekly workflow runs a size-capped sweep. First triage found four reader gaps on real data, including one silent-corruption bug (unsigned integers rendered as signed). ([#205](https://github.com/dfa1/vortex-java/issues/205)) ## [0.12.0] — 2026-07-04 diff --git a/CLAUDE.md b/CLAUDE.md index 5ef60e2c..1f9ae7f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,6 +63,8 @@ Trunk-based. PRs fine but always squash or rebase — no merge commits. Keep com ./mvnw verify -pl integration -am # integration (failsafe, NOT surefire) ./mvnw verify -pl integration -am -Dit.test="RustWritesJavaReadsIntegrationTest#method" ./bench JavaVsJniReadBenchmark.javaReadVolume # benchmark — always ClassName.methodName filter +scripts/hydrate-raincloud-corpus.sh --max-mb 200 # hydrate real-world conformance corpus (#205), + # then verify -Dit.test=RaincloudConformanceIntegrationTest ``` Regenerate after editing `.fbs`/`.proto` (both generators are in-house, no external tools): diff --git a/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java b/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java index c366728b..6a48e88b 100644 --- a/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java +++ b/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java @@ -13,6 +13,7 @@ import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.NullArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.reader.VortexReader; import io.github.dfa1.vortex.reader.ScanIterator; @@ -159,6 +160,9 @@ private static String cellValue(Array arr, long rowIdx) { // Nullable columns decode as MaskedArray: null rows export as an empty field, valid // rows defer to the inner values array. case MaskedArray ma -> ma.isValid(rowIdx) ? cellValue(ma.inner(), rowIdx) : ""; + // All-null columns (DType.Null) hold only a row count: every cell is an empty + // field, same rule as a MaskedArray null row. + case NullArray ignored -> ""; default -> throw new VortexException( "unsupported array type for CSV export: " + arr.getClass().getSimpleName()); }; diff --git a/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java b/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java index b5672a56..fc7bd650 100644 --- a/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java +++ b/csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java @@ -5,7 +5,9 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.writer.VortexWriter; import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.NullEncodingEncoder; import io.github.dfa1.vortex.writer.encode.NullableData; +import io.github.dfa1.vortex.writer.encode.PrimitiveEncodingEncoder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -71,6 +73,32 @@ void exportsToWriter(@TempDir Path tmp) throws Exception { assertThat(lines[2]).isEqualTo("2.7"); } + @Test + void exportsAllNullColumnAsEmptyFields(@TempDir Path tmp) throws Exception { + // Given a file mixing a value column with an all-null (DType.Null) column — + // the shape that made real-world exports throw (#211); the long[] carrier for + // the null column only supplies the row count + Path vortex = tmp.resolve("data.vortex"); + DType.Struct schema = new DType.Struct( + List.of(ColumnName.of("id"), ColumnName.of("empty")), + List.of(DType.I64, DType.NULL), + false); + try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + // explicit encoder list: the default cascading set has no DType.Null codec + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults(), + List.of(new NullEncodingEncoder(), new PrimitiveEncodingEncoder()))) { + writer.writeChunk(Map.of(ColumnName.of("id"), new long[]{1L, 2L}, ColumnName.of("empty"), new long[2])); + } + Path csv = tmp.resolve("out.csv"); + + // When + CsvExporter.exportCsv(vortex, csv); + + // Then — null cells are empty fields, same as MaskedArray null rows + List result = Files.readAllLines(csv); + assertThat(result).containsExactly("id,empty", "1,", "2,"); + } + @Test void suppressesHeaderWhenConfigured(@TempDir Path tmp) throws Exception { // Given diff --git a/docs/compatibility.md b/docs/compatibility.md index ee13e7c1..783e1136 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -43,6 +43,36 @@ resolves only the standalone decoders in `reader`; no encoder class is loaded. | Duplicate struct field names | Rust writer rejects ("StructLayout must have unique field names"); Rust reader tolerates foreign files (first-match access) | ⚠️ Deliberate divergence on read: Java rejects such files with `VortexException("duplicate field name in file schema")` instead of tolerating them — the name-keyed `Chunk` API cannot represent both columns, and silent column loss is worse than a loud failure on a file the reference writer refuses to produce. Java's writer mirrors the Rust writer's rejection. | | Blank / control-character field names | Wire-legal; the Rust writer produces `""` and whitespace-only names. NUL (`U+0000`) additionally aborts the Rust toolchain: Arrow FFI schema export hits a panic-cannot-unwind in `arrow-rs` (`ffi_stream::get_schema`) and SIGABRTs the process (measured against vortex-jni 0.75.0) | ⚠️ Deliberate strictness BOTH ways: vortex-java's writer refuses blank and control-character field names (`IllegalArgumentException`), and its reader rejects files carrying them (`VortexException` naming the producing pipeline as the likely bug) — the JSON-`""`-key principle: wire-legal is a floor, not a policy. Printable names of any shape (`$`-runs, spaces inside, emoji) are legal and round-trip intact both directions (measured; pinned by `ColumnNameEdgeCasesIntegrationTest`). | +## Real-world conformance: the Raincloud corpus + +Cross-implementation conformance against [Raincloud](https://github.com/spiraldb/raincloud) +(tracked in [#205](https://github.com/dfa1/vortex-java/issues/205)): 247 curated public +datasets whose `.vortex` files are written by the Vortex **Python** bindings +(`vortex-data 0.69.0`, pinned via Raincloud `v0.2.1`) with a Parquet sibling built from the +same Arrow table. `RaincloudConformanceIntegrationTest` reads each hydrated `.vortex` with +vortex-java, exports CSV, and diffs it against the Parquet sibling read by hardwood (the +oracle) — a zero-diff proves every value survived the Python-write / Java-read boundary. + +```bash +scripts/hydrate-raincloud-corpus.sh --max-mb 200 # cache → mirror → local build +./mvnw verify -pl integration -am -Dit.test=RaincloudConformanceIntegrationTest +``` + +Per-slug status lives in `integration/src/test/resources/raincloud/expected-status.csv` +(`ok` must pass; `gap:` must still fail, so a fix flips the entry in the same change; +`untriaged` runs and reports without failing the build). A scheduled workflow +(`raincloud-conformance.yml`) hydrates a size-capped subset weekly. Current triage — +18 `ok`, 2 known gaps: nested struct columns in scan +([#207](https://github.com/dfa1/vortex-java/issues/207)), narrow dict-offset ptypes in +string dicts ([#215](https://github.com/dfa1/vortex-java/issues/215)); +227 slugs untriaged. Fixed so far by this suite: lazy dict U8/U16 values +([#206](https://github.com/dfa1/vortex-java/issues/206)), unsigned integers rendered signed +([#208](https://github.com/dfa1/vortex-java/issues/208) — silent corruption), RLE over F64 +([#209](https://github.com/dfa1/vortex-java/issues/209)), row validity dropped by wrapper +decoders ([#210](https://github.com/dfa1/vortex-java/issues/210) — silent corruption), +all-null columns in CSV export +([#211](https://github.com/dfa1/vortex-java/issues/211)). + ## Encodings | Encoding ID | Decoder | Encoder | Decode | Encode | Notes | diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java new file mode 100644 index 00000000..29c7ce5a --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java @@ -0,0 +1,235 @@ +package io.github.dfa1.vortex.integration; + +import de.siegmar.fastcsv.writer.CsvWriter; +import dev.hardwood.InputFile; +import dev.hardwood.metadata.RepetitionType; +import dev.hardwood.reader.ParquetFileReader; +import dev.hardwood.reader.RowReader; +import dev.hardwood.schema.ColumnSchema; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.csv.CsvExporter; +import io.github.dfa1.vortex.csv.ExportOptions; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.opentest4j.TestAbortedException; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/// Real-world cross-implementation conformance over the Raincloud corpus +/// ([issue #205](https://github.com/dfa1/vortex-java/issues/205)). +/// +/// Each corpus dataset is a `.vortex` file written by the Vortex Python bindings with +/// its `.parquet` sibling built from the same Arrow table. The parquet file is the +/// oracle: hardwood reads it row-by-row and formats cells with the exact rules of +/// `CsvExporter.cellValue` (null → empty field, `Double.toString`, …), writing through +/// the same fastcsv writer so quoting is identical. A zero-diff against vortex-java's +/// CSV export proves every value survived the Python-write / Java-read boundary. +/// +/// Skipped (visibly, via assumption) when the corpus is not hydrated — run +/// `scripts/hydrate-raincloud-corpus.sh` first, or point `RAINCLOUD_CORPUS_MANIFEST` +/// at a manifest TSV (`slugvortex-pathparquet-path` per line). +/// +/// Expected per-slug status lives in `src/test/resources/raincloud/expected-status.csv`. +/// Known gaps assert that the failure still occurs, so fixing the reader forces the +/// matrix entry to flip to `ok` in the same change. +class RaincloudConformanceIntegrationTest { + + private static final Path DEFAULT_MANIFEST = + Path.of(System.getProperty("user.home"), ".cache", "raincloud", "corpus-manifest.tsv"); + + @Test + void corpusIsHydrated() { + // Given / When / Then — visible skip marker when the corpus is absent; the + // factory below yields zero tests in that case, which would otherwise pass silently + assumeTrue(Files.exists(manifestPath()), + "raincloud corpus not hydrated — run scripts/hydrate-raincloud-corpus.sh"); + } + + @TestFactory + Stream conformancePerSlug() throws IOException { + Path manifest = manifestPath(); + if (!Files.exists(manifest)) { + return Stream.empty(); + } + Map expected = readExpectedStatus(); + return Files.readAllLines(manifest).stream() + .filter(line -> !line.isBlank()) + .map(line -> line.split("\t")) + .map(parts -> DynamicTest.dynamicTest(parts[0], () -> { + String slug = parts[0]; + Path vortex = Path.of(parts[1]); + Path parquet = Path.of(parts[2]); + // slugs missing from the matrix are treated as untriaged, not failing + String status = expected.getOrDefault(slug, "untriaged"); + switch (status) { + case "ok" -> assertMatchesParquetOracle(vortex, parquet); + case "untriaged" -> reportUntriaged(vortex, parquet); + default -> assertStillFails(vortex, parquet, status); + } + })); + } + + private static void assertMatchesParquetOracle(Path vortex, Path parquet) throws IOException { + // Given + List oracleLines = oracleLines(parquet); + + // When + List result = exportVortex(vortex); + + // Then + assertLinesMatch(result, oracleLines); + } + + /// Runs the full conformance check but reports the outcome as an aborted test + /// either way: only triaged matrix entries may count as green or red. The abort + /// message says which way to flip the entry. + private static void reportUntriaged(Path vortex, Path parquet) { + try { + assertMatchesParquetOracle(vortex, parquet); + } catch (TestAbortedException e) { + throw e; // oracle limitation, not a conformance outcome + } catch (AssertionError | Exception e) { + throw new TestAbortedException( + "untriaged slug fails — classify as gap: in expected-status.csv: " + e); + } + throw new TestAbortedException("untriaged slug passes — flip its matrix entry to ok"); + } + + private static void assertStillFails(Path vortex, Path parquet, String status) { + // Given / When — a decode gap still reproduces when the export itself throws. + // Only VortexException (the contractual untrusted-input failure) and, narrowly, + // IndexOutOfBoundsException count: the latter is itself a bounds-guard bug (#215 + // throws it today) and this arm dies with that fix — a blanket RuntimeException + // catch would green unrelated regressions (NPEs, ...). No oracle needed here + // (the oracle may not even read this parquet, e.g. nested columns). + List result; + try { + result = exportVortex(vortex); + } catch (VortexException | IndexOutOfBoundsException e) { + return; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + // Then — the export now succeeds, so the gap must still be a silent-corruption + // one (e.g. #208): values must still mismatch the oracle. A clean pass means the + // gap was fixed: flip the expected-status.csv entry to ok in the same change + List oracleLines = oracleLines(parquet); + assertThatThrownBy(() -> assertLinesMatch(result, oracleLines)) + .as("known gap %s no longer reproduces — flip its matrix entry to ok", status) + .isInstanceOf(AssertionError.class); + } + + private static void assertLinesMatch(List result, List oracleLines) { + assertThat(result).hasSameSizeAs(oracleLines); + for (int i = 0; i < oracleLines.size(); i++) { + assertThat(result.get(i)).as("line %d", i + 1).isEqualTo(oracleLines.get(i)); + } + } + + private static List exportVortex(Path vortex) throws IOException { + StringWriter out = new StringWriter(); + CsvExporter.exportCsv(vortex, out, ExportOptions.defaults()); + return out.toString().lines().toList(); + } + + /// Reads the parquet sibling through hardwood; an oracle-side failure (nested + /// columns, unsupported physical type) aborts the slug rather than failing it — + /// it says nothing about vortex-java. Deliberate asymmetry: an `ok` slug whose + /// parquet the oracle cannot read stops being verified (visibly, as skipped) — + /// widen the oracle rather than let unverifiable entries fail the build. + private static List oracleLines(Path parquet) { + StringWriter out = new StringWriter(); + try { + writeOracleCsv(parquet, out); + } catch (TestAbortedException e) { + throw e; + } catch (Exception e) { + throw new TestAbortedException("oracle cannot read the parquet sibling: " + e); + } + return out.toString().lines().toList(); + } + + /// Oracle: hardwood reads the parquet sibling and emits CSV through the same + /// fastcsv writer configuration as `CsvExporter`, using its exact cell rules. + private static void writeOracleCsv(Path parquet, StringWriter out) throws IOException { + try (ParquetFileReader pfr = ParquetFileReader.open(InputFile.of(parquet)); + RowReader rows = pfr.rowReader(); + CsvWriter csv = CsvWriter.builder().fieldSeparator(',').build(out)) { + + List cols = pfr.getFileSchema().getColumns(); + csv.writeRecord(cols.stream().map(ColumnSchema::name).toList()); + + String[] row = new String[cols.size()]; + while (rows.hasNext()) { + rows.next(); + for (int c = 0; c < cols.size(); c++) { + row[c] = oracleCell(cols.get(c), rows); + } + csv.writeRecord(row); + } + } + } + + /// Formats a parquet cell with the exact rules of `CsvExporter.cellValue`: + /// null rows export as an empty field, valid rows use the JDK canonical + /// `toString` of the value. + /// + /// @param col the column schema + /// @param rows the row reader positioned at the current row + /// @return the formatted cell string + private static String oracleCell(ColumnSchema col, RowReader rows) { + String name = col.name(); + if (col.repetitionType() == RepetitionType.OPTIONAL && rows.isNull(name)) { + return ""; + } + return switch (col.type()) { + case INT32 -> Integer.toString(rows.getInt(name)); + case INT64 -> Long.toString(rows.getLong(name)); + case FLOAT -> Float.toString(rows.getFloat(name)); + case DOUBLE -> Double.toString(rows.getDouble(name)); + case BOOLEAN -> Boolean.toString(rows.getBoolean(name)); + case BYTE_ARRAY -> rows.getString(name); + // aborts (not fails) the slug: the oracle can't format this physical type + // yet, which is an oracle limitation rather than a vortex-java gap + default -> throw new TestAbortedException( + "oracle cannot format parquet type " + col.type() + " (column: " + name + ")"); + }; + } + + private static Path manifestPath() { + String env = System.getenv("RAINCLOUD_CORPUS_MANIFEST"); + return env != null ? Path.of(env) : DEFAULT_MANIFEST; + } + + private static Map readExpectedStatus() throws IOException { + try (var in = RaincloudConformanceIntegrationTest.class + .getResourceAsStream("/raincloud/expected-status.csv")) { + if (in == null) { + throw new UncheckedIOException(new IOException("expected-status.csv not on test classpath")); + } + Map statuses = new HashMap<>(); + for (String line : new String(in.readAllBytes()).lines().toList()) { + if (line.isBlank() || line.startsWith("#")) { + continue; + } + String[] parts = line.split(",", 2); + statuses.put(parts[0].trim(), parts[1].trim()); + } + return statuses; + } + } +} diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv new file mode 100644 index 00000000..337115a9 --- /dev/null +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -0,0 +1,256 @@ +# Raincloud conformance matrix — one line per vortex-enabled corpus slug: , +# status: ok — vortex-java must read the file and match the parquet oracle exactly +# gap: — known reader gap tracked by issue #; the test asserts the failure +# still occurs, so a fix forces flipping the entry to ok in the same change +# untriaged — not yet run; the test executes it and reports the outcome as an +# aborted (skipped) test instead of failing the build — flip to ok/gap +# as hydration coverage grows +# Corpus: all slugs with a Vortex artifact in the Raincloud catalog (spiraldb/raincloud), +# written by vortex-data 0.69.0. Hydrate any subset with scripts/hydrate-raincloud-corpus.sh. +120-years-of-olympic-history-athletes-and-results,untriaged +ai2-arc,untriaged +airbnb-prices-in-european-cities,untriaged +airbnbopendata,untriaged +amazon-reviews-2023-subscription-boxes,untriaged +ampds-whe,untriaged +anthropic-economic-index,untriaged +anthropic-hh-rlhf-helpful-base,untriaged +anthropic-interviewer,untriaged +aya-collection-templated,untriaged +aya-dataset,untriaged +bank-account-fraud-dataset-neurips-2022,untriaged +basketball,untriaged +behavioral-risk-factor-surveillance-system,untriaged +beir-msmarco,untriaged +bi-arade,untriaged +bi-bimbo,untriaged +bi-citymaxcapita,untriaged +bi-cmsprovider,untriaged +bi-commongovernment,untriaged +bi-corporations,untriaged +bi-eixo,untriaged +bi-euro2016,untriaged +bi-food,untriaged +bi-generico,untriaged +bi-hashtags,untriaged +bi-hatred,untriaged +bi-iglocations1,untriaged +bi-iglocations2,untriaged +bi-iublibrary,untriaged +bi-medicare1,untriaged +bi-medicare2,untriaged +bi-medicare3,untriaged +bi-medpayment1,untriaged +bi-medpayment2,untriaged +bi-mlb,untriaged +bi-motos,untriaged +bi-mulheresmil,untriaged +bi-nyc,untriaged +bi-pancreactomy1,untriaged +bi-pancreactomy2,untriaged +bi-physicians,untriaged +bi-provider,untriaged +bi-realestate1,untriaged +bi-realestate2,untriaged +bi-redfin1,untriaged +bi-redfin2,untriaged +bi-redfin3,untriaged +bi-redfin4,untriaged +bi-rentabilidad,untriaged +bi-romance,untriaged +bi-salariesfrance,untriaged +bi-tablerosistemapenal,untriaged +bi-taxpayer,untriaged +bi-telco,untriaged +bi-trainsuk1,untriaged +bi-trainsuk2,untriaged +bi-uberlandia,untriaged +bi-uscensus,untriaged +bi-wins,untriaged +bi-yalelanguages,untriaged +bigscience-p3-eval-subset,untriaged +building-permit-applications-data,untriaged +c4-en-validation,untriaged +california-housing-prices,untriaged +cardiovascular-diseases-risk-prediction-dataset,untriaged +chess,untriaged +clickbench-hits,untriaged +cnn-dailymail,untriaged +codeparrot-clean-valid,untriaged +cohere-wikipedia-simple-embed,untriaged +coig,untriaged +coronahack-chest-xraydataset,untriaged +cosmopedia-stanford,untriaged +countries-of-the-world,gap:207 +covid-world-vaccination-progress,untriaged +covid19-data-from-john-hopkins-university,untriaged +crimes-in-boston,untriaged +databricks-dolly-15k,untriaged +dbpedia-embeddings,untriaged +diabetes-health-indicators-dataset,untriaged +disease-symptom-description-dataset,untriaged +docmatix-zero-shot,untriaged +electric-motor-temperature,untriaged +emotions-dataset-for-nlp,untriaged +fhv_tripdata_2025,untriaged +fhvhv_tripdata_2025,untriaged +finemath-4plus,untriaged +finepdfs-en-test,untriaged +finetranslations-swedish,untriaged +fineweb-2-swedish,untriaged +fineweb-sample-10bt,untriaged +fitbit,untriaged +formula-1-world-championship-1950-2020,untriaged +frames-benchmark,untriaged +ghcn-daily,untriaged +glass,untriaged +global-fossil-co2-emissions-by-country-2002-2022,untriaged +glove-6b-100d,untriaged +glove-6b-200d,untriaged +glove-6b-50d,untriaged +goodbooks-10k,untriaged +google-cluster-trace-2011-machine-events,untriaged +green_tripdata_2025,untriaged +gsm8k,untriaged +gtsrb-german-traffic-sign,untriaged +hacker-news,untriaged +heart-disease-health-indicators-dataset,untriaged +hellaswag,untriaged +helpsteer2,untriaged +hotel-booking-demand,untriaged +hotpotqa-fullwiki,untriaged +housing-prices-dataset,untriaged +humaneval,untriaged +insurance,ok +international-football-results-from-1872-to-2017,ok +iowa-liquor-sales,untriaged +kepler-exoplanet-search-results,ok +kepler-labelled-time-series-data,untriaged +laion-400m,untriaged +librispeech-test-clean,untriaged +loan-default-dataset,untriaged +lung-cancer,untriaged +marketing-data,untriaged +mbpp,untriaged +medmcqa,untriaged +mlcourse,untriaged +mmlu,untriaged +mmlu-pro,untriaged +mmmlu,untriaged +mmmu,untriaged +mnist,untriaged +movies,untriaged +new-york-city-airbnb-open-data,untriaged +news-headlines-dataset-for-sarcasm-detection,untriaged +nips-papers,untriaged +no-robots,untriaged +nyc-311,untriaged +nyc-parking-tickets,untriaged +nyc-property-sales,untriaged +nypd-complaints,untriaged +oasst1,untriaged +open-food-facts,untriaged +openlibrary-authors,untriaged +openlibrary-editions,untriaged +openlibrary-works,untriaged +openorca,untriaged +openpowerlifting,untriaged +osm-germany-nodes,untriaged +osm-germany-relations,untriaged +osm-germany-ways,untriaged +osmi-mental-health-in-tech-2016,untriaged +osmi-mental-health-in-tech-2017,untriaged +osmi-mental-health-in-tech-2018,untriaged +osmi-mental-health-in-tech-2019,untriaged +osmi-mental-health-in-tech-2020,untriaged +osmi-mental-health-in-tech-2021,untriaged +osmi-mental-health-in-tech-2022,untriaged +osmi-mental-health-in-tech-2023,untriaged +palmer-archipelago-antarctica-penguin-data,ok +peoples-speech-clean-validation,untriaged +personal-key-indicators-of-heart-disease,untriaged +pleias-synth,untriaged +pubmedqa-labeled,untriaged +sf-salaries,untriaged +slimpajama-6b,untriaged +squad-v2,untriaged +stack-overflow-2018-developer-survey,untriaged +stackoverflow-badges,untriaged +stackoverflow-postlinks,untriaged +stackoverflow-posts,untriaged +stackoverflow-tags,untriaged +stackoverflow-users,untriaged +synthetic-text-to-sql,untriaged +temperature-change,untriaged +tinystories,untriaged +titanic-dataset,untriaged +truthfulqa-mc,untriaged +uber-pickups-in-new-york-city,untriaged +uci-abalone,ok +uci-adult,ok +uci-ai4i-2020-predictive-maintenance,untriaged +uci-air-quality,untriaged +uci-auto-mpg,ok +uci-automobile,untriaged +uci-bank-marketing,untriaged +uci-beijing-multi-site-air-quality,untriaged +uci-bike-sharing-dataset,ok +uci-breast-cancer,untriaged +uci-breast-cancer-wisconsin-diagnostic,untriaged +uci-breast-cancer-wisconsin-original,untriaged +uci-car-evaluation,untriaged +uci-cdc-diabetes-health-indicators,untriaged +uci-census-income,untriaged +uci-chronic-kidney-disease,untriaged +uci-concrete-compressive-strength,untriaged +uci-credit-approval,untriaged +uci-default-of-credit-card-clients,untriaged +uci-diabetes,untriaged +uci-diabetes-130-us-hospitals,untriaged +uci-dry-bean-dataset,ok +uci-electricityloaddiagrams20112014,untriaged +uci-energy-efficiency,untriaged +uci-estimation-of-obesity-levels,ok +uci-forest-fires,ok +uci-heart-disease,ok +uci-heart-failure-clinical-records,untriaged +uci-human-activity-recognition-using-smartphones,untriaged +uci-individual-household-electric-power-consumption,untriaged +uci-iris,ok +uci-magic-gamma-telescope,gap:215 +uci-mushroom,ok +uci-online-retail,untriaged +uci-online-retail-ii,untriaged +uci-online-shoppers-purchasing-intention,untriaged +uci-optical-recognition-of-handwritten-digits,untriaged +uci-parkinsons,untriaged +uci-phishing-websites,untriaged +uci-predict-students-dropout-and-academic-success,untriaged +uci-real-estate-valuation-data-set,untriaged +uci-seeds,ok +uci-seoul-bike-sharing-demand,ok +uci-sms-spam-collection,untriaged +uci-spambase,untriaged +uci-statlog-german-credit-data,untriaged +uci-student-performance,untriaged +uci-thyroid-disease,untriaged +uci-wholesale-customers,untriaged +uci-wine,ok +uci-wine-quality,ok +ufcdata,untriaged +uk-price-paid,untriaged +uk-road-safety-accidents-and-vehicles,untriaged +ultrachat-200k,untriaged +ultrafeedback-binarized,untriaged +us-accidents,untriaged +us-airbnb-open-data,untriaged +us-drought-meteorological-data,untriaged +walmart-dataset,untriaged +waxal-dagbani-asr-test,untriaged +wdi,untriaged +websight-v01,untriaged +wikipedia-en,untriaged +world-energy-consumption,untriaged +yellow_tripdata_2025,untriaged +youtube-commons-sample,untriaged +zoo-animal-classification,untriaged diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictByteArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictByteArray.java new file mode 100644 index 00000000..aec09337 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictByteArray.java @@ -0,0 +1,150 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.error.VortexException; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.lang.foreign.ValueLayout; +import java.util.function.LongBinaryOperator; + +/// Dict-encoded [ByteArray] view. ADR 0012 shape. +/// +/// Stores `values` (the dictionary pool) and `codes` (one index per +/// row into `values`). Scalar access resolves on demand: +/// `getByte(i) = values.getByte(codes.getCode(i))`. Per ADR 0012, this +/// preserves zero-copy on dict-encoded categorical columns. +/// +/// The `codes` array is typed as [Array] because the codes ptype +/// varies with dictionary size — U8/U16/U32/U64 backed by +/// [ByteArray]/[ShortArray]/[IntArray]/[LongArray]. +/// +/// @param dtype logical element type (matches `values.dtype()`) +/// @param length total logical row count (matches `codes.length()`) +/// @param values dictionary pool — element at code `c` is `values.getByte(c)` +/// @param codes per-row index into `values`; must be one of +/// [ByteArray], [ShortArray], [IntArray], [LongArray] +public record DictByteArray(DType dtype, long length, ByteArray values, Array codes) implements ByteArray { + + /// Builds a [DictByteArray], validating that `codes` is one of the + /// four narrow-int code array types and that its length matches `length`. + /// + /// @param dtype logical element type + /// @param length total logical row count + /// @param values dictionary pool + /// @param codes per-row code array (must be [ByteArray], [ShortArray], + /// [IntArray], or [LongArray]) + /// @return a new [DictByteArray] + /// @throws VortexException if `codes` is not a supported code-array type or + /// its length does not equal `length` + public static DictByteArray of(DType dtype, long length, ByteArray values, Array codes) { + DictArrays.validateCodes(codes, length); + return new DictByteArray(dtype, length, values, codes); + } + + @Override + public byte getByte(long i) { + return values.getByte(DictArrays.readCode(codes, i)); + } + + /// Materializes by gathering one dictionary value per code into a fresh + /// one-byte-per-element segment. The codes switch is hoisted outside the loop so + /// each branch is a uniform gather over a single code width. + /// + /// @param arena allocator for the output segment + /// @return a read-only segment of `length()` gathered bytes + /// @throws VortexException if `codes` is not a supported code-array type + @Override + public MemorySegment materialize(SegmentAllocator arena) { + long n = length; + MemorySegment dst = arena.allocate(n); + ByteArray vals = values; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, vals.getByte(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, vals.getByte(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, vals.getByte(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, vals.getByte(la.getLong(i))); + } + } + default -> throw new VortexException("DictByteArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return dst.asReadOnly(); + } + + @Override + public void forEachByte(ByteConsumer cons) { + long n = length; + ByteArray vals = values; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getByte(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getByte(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getByte(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getByte(la.getLong(i))); + } + } + default -> throw new VortexException("DictByteArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + } + + @Override + public long fold(long identity, LongBinaryOperator op) { + long n = length; + ByteArray vals = values; + long result = identity; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getByte(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getByte(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getByte(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getByte(la.getLong(i))); + } + } + default -> throw new VortexException("DictByteArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return result; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictShortArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictShortArray.java new file mode 100644 index 00000000..84846e90 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictShortArray.java @@ -0,0 +1,150 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.io.VortexFormat; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.util.function.LongBinaryOperator; + +/// Dict-encoded [ShortArray] view. ADR 0012 shape. +/// +/// Stores `values` (the dictionary pool) and `codes` (one index per +/// row into `values`). Scalar access resolves on demand: +/// `getShort(i) = values.getShort(codes.getCode(i))`. Per ADR 0012, this +/// preserves zero-copy on dict-encoded categorical columns. +/// +/// The `codes` array is typed as [Array] because the codes ptype +/// varies with dictionary size — U8/U16/U32/U64 backed by +/// [ByteArray]/[ShortArray]/[IntArray]/[LongArray]. +/// +/// @param dtype logical element type (matches `values.dtype()`) +/// @param length total logical row count (matches `codes.length()`) +/// @param values dictionary pool — element at code `c` is `values.getShort(c)` +/// @param codes per-row index into `values`; must be one of +/// [ByteArray], [ShortArray], [IntArray], [LongArray] +public record DictShortArray(DType dtype, long length, ShortArray values, Array codes) implements ShortArray { + + /// Builds a [DictShortArray], validating that `codes` is one of the + /// four narrow-int code array types and that its length matches `length`. + /// + /// @param dtype logical element type + /// @param length total logical row count + /// @param values dictionary pool + /// @param codes per-row code array (must be [ByteArray], [ShortArray], + /// [IntArray], or [LongArray]) + /// @return a new [DictShortArray] + /// @throws VortexException if `codes` is not a supported code-array type or + /// its length does not equal `length` + public static DictShortArray of(DType dtype, long length, ShortArray values, Array codes) { + DictArrays.validateCodes(codes, length); + return new DictShortArray(dtype, length, values, codes); + } + + @Override + public short getShort(long i) { + return values.getShort(DictArrays.readCode(codes, i)); + } + + /// Materializes by gathering one dictionary value per code into a fresh + /// little-endian `i16` segment. The codes switch is hoisted outside the loop so + /// each branch is a uniform gather over a single code width. + /// + /// @param arena allocator for the output segment + /// @return a read-only little-endian `i16` segment of gathered values + /// @throws VortexException if `codes` is not a supported code-array type + @Override + public MemorySegment materialize(SegmentAllocator arena) { + long n = length; + MemorySegment dst = arena.allocate(n * 2L, 2); + ShortArray vals = values; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, vals.getShort(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, vals.getShort(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, vals.getShort(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, vals.getShort(la.getLong(i))); + } + } + default -> throw new VortexException("DictShortArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return dst.asReadOnly(); + } + + @Override + public void forEachShort(ShortConsumer cons) { + long n = length; + ShortArray vals = values; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getShort(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getShort(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getShort(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + cons.accept(vals.getShort(la.getLong(i))); + } + } + default -> throw new VortexException("DictShortArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + } + + @Override + public long fold(long identity, LongBinaryOperator op) { + long n = length; + ShortArray vals = values; + long result = identity; + switch (codes) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getShort(Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getShort(Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getShort(Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + result = op.applyAsLong(result, vals.getShort(la.getLong(i))); + } + } + default -> throw new VortexException("DictShortArray: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return result; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java index fbc02523..4a53a282 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java @@ -8,10 +8,12 @@ import io.github.dfa1.vortex.core.proto.ProtoALPMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.LazyAlpDoubleArray; import io.github.dfa1.vortex.reader.array.LazyAlpFloatArray; import io.github.dfa1.vortex.reader.array.LazyConstantDoubleArray; import io.github.dfa1.vortex.reader.array.LazyConstantFloatArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; @@ -55,21 +57,38 @@ public Array decode(DecodeContext ctx) { PType ptype = p.ptype(); long n = ctx.rowCount(); - return switch (ptype) { - case F64 -> decodeF64(ctx, meta, expE, expF, n); - case F32 -> decodeF32(ctx, meta, expE, expF, n); + // Validity mirrors the Rust reference (`ValidityChild`): an ALP array's + // validity IS its encoded child's validity, and the encoded child's dtype + // inherits the ALP dtype's nullability. Decode the child as an Array so a + // nullable primitive surfaces its MaskedArray instead of being flattened to a + // raw segment (which silently dropped nulls — #210). + DType.Primitive encodedDtype = new DType.Primitive( + ptype == PType.F64 ? PType.I64 : PType.I32, p.nullable()); + Array encoded = ctx.decodeChild(0, encodedDtype, n); + BoolArray validity = null; + Array rawEncoded = encoded; + if (encoded instanceof MaskedArray masked) { + rawEncoded = masked.inner(); + validity = masked.validity(); + } + MemorySegment src = ctx.materialize(rawEncoded); + + Array result = switch (ptype) { + case F64 -> decodeF64(ctx, meta, expE, expF, n, src); + case F32 -> decodeF32(ctx, meta, expE, expF, n, src); default -> throw new VortexException(EncodingId.VORTEX_ALP, "unsupported dtype " + ptype); }; + return validity != null ? new MaskedArray(result, validity) : result; } - private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n) { + private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n, + MemorySegment src) { // Decode formula mirrors the Rust reference (`ALPFloat::decode_single`): two-step // `encoded * F10[f] * IF10[e]`. A pre-multiplied `scale = F10[f] * IF10[e]` // gives different IEEE rounding for non-trivial `expF`, breaking round-trip with // the encoder's verify step. double df = F10_F64[expF]; double de = IF10_F64[expE]; - MemorySegment src = ctx.decodeChildSegment(0, DType.I64, n); long srcCap = SegmentBroadcast.capacity(src, 8); if (meta.patches() == null) { @@ -96,10 +115,10 @@ private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int exp return new MaterializedDoubleArray(ctx.dtype(), n, buf.asReadOnly()); } - private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n) { + private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int expE, int expF, long n, + MemorySegment src) { float df = F10_F32[expF]; float de = IF10_F32[expE]; - MemorySegment src = ctx.decodeChildSegment(0, DType.I32, n); long srcCap = SegmentBroadcast.capacity(src, 4); if (meta.patches() == null) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java index 26f495ce..6907ac3d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java @@ -8,6 +8,8 @@ import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedIntArray; import io.github.dfa1.vortex.reader.array.MaterializedLongArray; @@ -59,13 +61,50 @@ public Array decode(DecodeContext ctx) { applyPatches(ctx, meta.patches(), output, ptype.byteSize()); } - return switch (ptype) { + Array values = switch (ptype) { case I64, U64 -> new MaterializedLongArray(ctx.dtype(), rowCount, output); case I32, U32 -> new MaterializedIntArray(ctx.dtype(), rowCount, output); case I16, U16 -> new MaterializedShortArray(ctx.dtype(), rowCount, output); case I8, U8 -> new MaterializedByteArray(ctx.dtype(), rowCount, output); default -> throw new VortexException(EncodingId.FASTLANES_BITPACKED, "unsupported ptype " + ptype); }; + return wrapValidity(ctx, meta, values, rowCount); + } + + /// Row validity mirrors the Rust reference (`BitPacked` vtable `deserialize`): the + /// children are `[patch_indices, patch_values, patch_chunk_offsets?]` (present only + /// with patches) followed by an optional trailing bool validity child. Encodings + /// stacked above bitpacked (ALP, FoR, …) delegate their validity here + /// (`ValidityChild`), so dropping this child silently un-nulls rows (#210). + /// + /// @param ctx decode context + /// @param meta bitpacked metadata (patch shape determines the validity index) + /// @param values the unpacked (and patched) values array + /// @param rowCount logical row count + /// @return `values`, wrapped in a [MaskedArray] when a validity child is present + private static Array wrapValidity(DecodeContext ctx, ProtoBitPackedMetadata meta, Array values, long rowCount) { + int validityIdx = validityChildIndex(meta); + int childCount = ctx.node().children().length; + if (childCount == validityIdx) { + return values; + } + if (childCount != validityIdx + 1) { + throw new VortexException(EncodingId.FASTLANES_BITPACKED, + "expected " + validityIdx + " or " + (validityIdx + 1) + " children, got " + childCount); + } + Array va = ctx.decodeChild(validityIdx, DType.BOOL, rowCount); + if (!(va instanceof BoolArray validity)) { + throw new VortexException(EncodingId.FASTLANES_BITPACKED, + "validity child decoded to unexpected type: " + va.getClass().getSimpleName()); + } + return new MaskedArray(values, validity); + } + + private static int validityChildIndex(ProtoBitPackedMetadata meta) { + if (meta.patches() == null) { + return 0; + } + return meta.patches().chunk_offsets_ptype() != null ? 3 : 2; } private static void fastlanesUnpackToSeg( diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 7dedf576..7789eb82 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -7,6 +7,9 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoDictMetadata; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray; import io.github.dfa1.vortex.reader.array.MaterializedFloatArray; @@ -101,9 +104,28 @@ private static Array decodeRustProto(DecodeContext ctx, MemorySegment metaBuf) { PType valPType = ((DType.Primitive) ctx.dtype()).ptype(); int elemSize = valPType.byteSize(); - DType codesDtype = new DType.Primitive(codePType, false); - MemorySegment codesBuf = ctx.decodeChildSegment(0, codesDtype, rowCount); - MemorySegment valuesBuf = ctx.decodeChildSegment(1, ctx.dtype(), valuesLen); + // Row validity mirrors the Rust reference: a DictArray row is null when its CODE + // is null (codes-side validity child) or when the code points at an invalid pool + // slot (pool-null representation). Both arrive as MaskedArray children and must + // be propagated per row, not flattened away (#210). + DType codesDtype = new DType.Primitive(codePType, ctx.dtype().nullable()); + Array codesArr = ctx.decodeChild(0, codesDtype, rowCount); + BoolArray codesValidity = null; + Array rawCodes = codesArr; + if (codesArr instanceof MaskedArray masked) { + rawCodes = masked.inner(); + codesValidity = masked.validity(); + } + MemorySegment codesBuf = ctx.materialize(rawCodes); + + Array valuesArr = ctx.decodeChild(1, ctx.dtype(), valuesLen); + BoolArray poolValidity = null; + Array rawValues = valuesArr; + if (valuesArr instanceof MaskedArray masked) { + rawValues = masked.inner(); + poolValidity = masked.validity(); + } + MemorySegment valuesBuf = ctx.materialize(rawValues); MemorySegment out = ctx.arena().allocate(rowCount * elemSize); switch (codePType) { @@ -112,7 +134,77 @@ private static Array decodeRustProto(DecodeContext ctx, MemorySegment metaBuf) { case U32 -> expandU32(codesBuf, valuesBuf, out, rowCount, elemSize); default -> throw new VortexException(EncodingId.VORTEX_DICT, "unexpected code type: " + codePType); } - return typedArray(ctx.dtype(), valPType, rowCount, out.asReadOnly()); + Array values = typedArray(ctx.dtype(), valPType, rowCount, out.asReadOnly()); + BoolArray rowValidity = rowValidity(ctx, codesBuf, codePType, codesValidity, poolValidity, rowCount); + return rowValidity == null ? values : new MaskedArray(values, rowValidity); + } + + /// Combines codes-side and pool-side validity into per-row validity: row `i` is + /// valid iff its code is valid and the pool slot the code references is valid. + /// Returns `null` when neither side carries validity (all rows valid), and the + /// codes-side mask unchanged when the pool is all-valid (it is already per-row). + /// Output is bit-packed LSB-first ([MaterializedBoolArray] layout). The broadcast + /// branch mirrors the `expandXxx` loops (undersized codes buffer = ConstantEncoding + /// fan-out) and is split out of the fast path. + /// + /// @param ctx decode context (allocation arena) + /// @param codesBuf decoded raw codes buffer + /// @param codePType unsigned code ptype (U8/U16/U32) + /// @param codesValidity per-row validity from the codes child, or `null` + /// @param poolValidity validity of the values pool, or `null` + /// @param rowCount logical row count + /// @return a bit-packed row validity array of `rowCount` bits, or `null` when all valid + private static BoolArray rowValidity(DecodeContext ctx, MemorySegment codesBuf, PType codePType, + BoolArray codesValidity, BoolArray poolValidity, long rowCount) { + if (poolValidity == null) { + return codesValidity; + } + MemorySegment bits = ctx.arena().allocate((rowCount + 7) >>> 3); + long codesCap = SegmentBroadcast.capacity(codesBuf, codePType.byteSize()); + if (codesCap >= rowCount) { + for (long i = 0; i < rowCount; i++) { + boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) + && poolValid(poolValidity, readCode(codesBuf, i, codePType)); + if (valid) { + setBit(bits, i); + } + } + } else { + for (long i = 0; i < rowCount; i++) { + boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) + && poolValid(poolValidity, readCode(codesBuf, i % codesCap, codePType)); + if (valid) { + setBit(bits, i); + } + } + } + return new MaterializedBoolArray(DType.BOOL, rowCount, bits.asReadOnly()); + } + + /// Untrusted-input guard: a malformed file may carry codes outside the pool, and the + /// validity bitmap lookup must fail as [VortexException], never as a raw JDK + /// IndexOutOfBoundsException. + private static boolean poolValid(BoolArray poolValidity, long code) { + if (code >= poolValidity.length()) { + throw new VortexException(EncodingId.VORTEX_DICT, + "code " + code + " out of range for pool validity of length " + poolValidity.length()); + } + return poolValidity.getBoolean(code); + } + + private static long readCode(MemorySegment codes, long i, PType codePType) { + return switch (codePType) { + case U8 -> Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i)); + case U16 -> Short.toUnsignedLong(codes.getAtIndex(VortexFormat.LE_SHORT, i)); + case U32 -> Integer.toUnsignedLong(codes.getAtIndex(VortexFormat.LE_INT, i)); + default -> throw new VortexException(EncodingId.VORTEX_DICT, "unexpected code type: " + codePType); + }; + } + + private static void setBit(MemorySegment bits, long i) { + long byteIdx = i >>> 3; + byte cur = bits.get(ValueLayout.JAVA_BYTE, byteIdx); + bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) (cur | (1 << (i & 7)))); } private static Array decodeUtf8DictLegacy(DecodeContext ctx, MemorySegment meta) { @@ -140,15 +232,32 @@ private static Array decodeUtf8DictProto(DecodeContext ctx, MemorySegment metaBu long dictSize = meta.values_len(); long n = ctx.rowCount(); - DType codesDtype = new DType.Primitive(codePType, false); - MemorySegment codesBuf = ctx.decodeChildSegment(0, codesDtype, n); + // Same two null representations as the primitive path (#210): codes-side + // validity and/or an invalid pool slot referenced by null rows. + DType codesDtype = new DType.Primitive(codePType, ctx.dtype().nullable()); + Array codesArr = ctx.decodeChild(0, codesDtype, n); + BoolArray codesValidity = null; + Array rawCodes = codesArr; + if (codesArr instanceof MaskedArray masked) { + rawCodes = masked.inner(); + codesValidity = masked.validity(); + } + MemorySegment codesBuf = ctx.materialize(rawCodes); - VarBinArray valuesArr = (VarBinArray) ctx.decodeChild(1, ctx.dtype(), dictSize); + Array valuesDecoded = ctx.decodeChild(1, ctx.dtype(), dictSize); + BoolArray poolValidity = null; + if (valuesDecoded instanceof MaskedArray masked) { + valuesDecoded = masked.inner(); + poolValidity = masked.validity(); + } + VarBinArray valuesArr = (VarBinArray) valuesDecoded; VarBinArray.OffsetMode dictValues = VarBinArray.toOffsetMode(valuesArr, ctx.arena()); - return VarBinArray.ofDict(ctx.dtype(), n, + BoolArray rowValidity = rowValidity(ctx, codesBuf, codePType, codesValidity, poolValidity, n); + Array dict = VarBinArray.ofDict(ctx.dtype(), n, dictValues.bytesSegment(), dictValues.offsetsSegment(), PType.I64, codesBuf, codePType); + return rowValidity == null ? dict : new MaskedArray(dict, rowValidity); } static void expandU8(MemorySegment codes, MemorySegment values, MemorySegment out, long rowCount, int elemSize) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java index fe05b257..f39fbaab 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java @@ -6,6 +6,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.LazyConstantByteArray; import io.github.dfa1.vortex.reader.array.LazyConstantIntArray; import io.github.dfa1.vortex.reader.array.LazyConstantLongArray; @@ -14,6 +15,7 @@ import io.github.dfa1.vortex.reader.array.LazyZigZagIntArray; import io.github.dfa1.vortex.reader.array.LazyZigZagLongArray; import io.github.dfa1.vortex.reader.array.LazyZigZagShortArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -35,10 +37,25 @@ public Array decode(DecodeContext ctx) { PType unsigned = toUnsigned(signed); long n = ctx.rowCount(); - MemorySegment src = ctx.decodeChildSegment(0, new DType.Primitive(unsigned, false), n); + // Validity mirrors the Rust reference (`ValidityChild`): a zigzag array's + // validity IS its encoded child's, and the child dtype inherits the nullability. + // Decode as an Array so a masked child is split rather than flattened (#210). + Array encoded = ctx.decodeChild(0, new DType.Primitive(unsigned, p.nullable()), n); + BoolArray validity = null; + Array rawEncoded = encoded; + if (encoded instanceof MaskedArray masked) { + rawEncoded = masked.inner(); + validity = masked.validity(); + } + MemorySegment src = ctx.materialize(rawEncoded); int elemBytes = signed.byteSize(); long srcCap = SegmentBroadcast.capacity(src, elemBytes); + Array result = decodeUnwrapped(ctx, signed, n, src, srcCap); + return validity != null ? new MaskedArray(result, validity) : result; + } + + private static Array decodeUnwrapped(DecodeContext ctx, PType signed, long n, MemorySegment src, long srcCap) { if (srcCap < n) { // Broadcast: single encoded value maps to all n rows — decode once, return constant return switch (signed) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index 4c2608f7..c9b13390 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -10,15 +10,21 @@ import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DictByteArray; import io.github.dfa1.vortex.reader.array.DictDoubleArray; import io.github.dfa1.vortex.reader.array.DictFloatArray; import io.github.dfa1.vortex.reader.array.DictIntArray; import io.github.dfa1.vortex.reader.array.DictLongArray; +import io.github.dfa1.vortex.reader.array.DictShortArray; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import java.lang.foreign.MemorySegment; @@ -83,7 +89,7 @@ public Array decode(LayoutDecodeContext ctx, Layout dictLayout, DType dtype) { // mmap-bounded. Validate by inspecting the underlying segment without forcing // materialization of non-segment-backed codes (lazy variants). validateDictCodesCapacity(codes, codesPType, n); - return buildLazyDictPrimitive(pDtype, n, values, codes); + return buildLazyDictPrimitive(pDtype, n, values, codes, arena); } // Non-Utf8, non-Primitive dict — e.g. extension types backed by VarBin. Fall through // to the existing string expansion for compatibility. @@ -116,27 +122,85 @@ private static void validateDictCodesCapacity(Array codes, PType codesPType, lon } } - /// Builds the matching `DictXxxArray` for a primitive dictionary, unwrapping - /// any [MaskedArray] layer on either side — dictionary lookups are keyed by code - /// so value-side validity is meaningless at this layer. + /// Builds the matching `DictXxxArray` for a primitive dictionary. + /// + /// Row validity mirrors the Rust reference (#210): a dict row is null when its CODE + /// is null (codes child arrives as a [MaskedArray]) or when the code points at an + /// invalid pool slot (nullable values pool arrives as a [MaskedArray]). Both masks + /// must be propagated to per-row validity — dropping either silently un-nulls rows. /// /// @param dtype primitive logical type of dict values /// @param n total logical row count /// @param values dictionary values /// @param codes per-row codes into `values` - /// @return a lazy `DictXxxArray` matching the value ptype - private static Array buildLazyDictPrimitive(DType.Primitive dtype, long n, Array values, Array codes) { + /// @param arena allocator for the gathered row-validity bitmap + /// @return a lazy `DictXxxArray` matching the value ptype, wrapped in a + /// [MaskedArray] when either side carries validity + private static Array buildLazyDictPrimitive(DType.Primitive dtype, long n, Array values, Array codes, + SegmentAllocator arena) { + BoolArray poolValidity = values instanceof MaskedArray mv ? mv.validity() : null; Array valuesData = values instanceof MaskedArray mv ? mv.inner() : values; + BoolArray codesValidity = codes instanceof MaskedArray mc ? mc.validity() : null; Array codesData = codes instanceof MaskedArray mc ? mc.inner() : codes; PType ptype = dtype.ptype(); - return switch (ptype) { + Array dict = switch (ptype) { case I64, U64 -> DictLongArray.of(dtype, n, (LongArray) valuesData, codesData); case I32, U32 -> DictIntArray.of(dtype, n, (IntArray) valuesData, codesData); + case I16, U16 -> DictShortArray.of(dtype, n, (ShortArray) valuesData, codesData); + case I8, U8 -> DictByteArray.of(dtype, n, (ByteArray) valuesData, codesData); case F64 -> DictDoubleArray.of(dtype, n, (DoubleArray) valuesData, codesData); case F32 -> DictFloatArray.of(dtype, n, (FloatArray) valuesData, codesData); + // F16 has no Array subtype yet default -> throw new VortexException(EncodingId.VORTEX_DICT, "layout: unsupported ptype for lazy dict: " + ptype); }; + if (poolValidity == null && codesValidity == null) { + return dict; + } + if (poolValidity == null) { + return new MaskedArray(dict, codesValidity); + } + return new MaskedArray(dict, gatherRowValidity(codesData, codesValidity, poolValidity, n, arena)); + } + + /// Combines codes-side and pool-side validity per row: row `i` is valid iff its code + /// is valid and the pool slot the code references is valid. Bit-packed LSB-first + /// ([MaterializedBoolArray] layout). + /// + /// @param codes raw per-row codes + /// @param codesValidity per-row validity from the codes side, or `null` + /// @param poolValidity validity of the values pool + /// @param n logical row count + /// @param arena allocator for the bitmap + /// @return a bit-packed row validity array of `n` bits + private static BoolArray gatherRowValidity(Array codes, BoolArray codesValidity, BoolArray poolValidity, + long n, SegmentAllocator arena) { + MemorySegment bits = arena.allocate((n + 7) >>> 3); + for (long i = 0; i < n; i++) { + long code = switch (codes) { + case ByteArray ba -> Byte.toUnsignedLong(ba.getByte(i)); + case ShortArray sa -> Short.toUnsignedLong(sa.getShort(i)); + case IntArray ia -> Integer.toUnsignedLong(ia.getInt(i)); + case LongArray la -> la.getLong(i); + default -> throw new VortexException(EncodingId.VORTEX_DICT, + "layout: invalid codes type: " + codes.getClass().getSimpleName()); + }; + // Untrusted-input guard: codes outside the pool must fail as VortexException, + // never as a raw JDK IndexOutOfBoundsException. + if (code >= poolValidity.length()) { + throw new VortexException(EncodingId.VORTEX_DICT, + "layout: code " + code + " out of range for pool validity of length " + + poolValidity.length()); + } + boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) + && poolValidity.getBoolean(code); + if (valid) { + long byteIdx = i >>> 3; + byte cur = bits.get(ValueLayout.JAVA_BYTE, byteIdx); + bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) (cur | (1 << (i & 7)))); + } + } + return new MaterializedBoolArray(DType.BOOL, n, bits.asReadOnly()); } private static PType readDictLayoutCodesPType(MemorySegment rawMeta) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java index 70561fc1..c1ab3650 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java @@ -455,6 +455,248 @@ void bulkOps_invalidCodesType_throw() { } } + @Nested + class DictByte { + + @Test + void u8CodesDispatch() { + try (Arena arena = Arena.ofConfined()) { + // Given a byte pool and U8 codes selecting out of order + ByteArray values = byteArray(arena, (byte) 10, (byte) 20, (byte) 30); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 2); + DictByteArray sut = DictByteArray.of(U8, 2, values, codes); + + // When/Then + assertThat(sut.getByte(0)).isEqualTo((byte) 10); + assertThat(sut.getByte(1)).isEqualTo((byte) 30); + } + } + + @Test + void u8DtypeGetIntWidensUnsigned() { + try (Arena arena = Arena.ofConfined()) { + // Given a U8 pool holding 132 (0x84 — negative as a signed byte). This is the + // real-world shape from the Raincloud corpus (uci-wine magnesium): dict-encoded + // U8 values above 127 must widen unsigned through the ByteArray.getInt default. + ByteArray values = byteArray(arena, (byte) 132, (byte) 7); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 1); + DictByteArray sut = DictByteArray.of(U8, 2, values, codes); + + // When + int result = sut.getInt(0); + + // Then — unsigned 132, not -124; the raw storage byte stays signed + assertThat(result).isEqualTo(132); + assertThat(sut.getByte(0)).isEqualTo((byte) -124); + } + } + + @Test + void forEachByteMaterializesInOrder() { + try (Arena arena = Arena.ofConfined()) { + // Given + ByteArray values = byteArray(arena, (byte) 10, (byte) 20); + ByteArray codes = byteArray(arena, (byte) 1, (byte) 0, (byte) 1); + DictByteArray sut = DictByteArray.of(U8, 3, values, codes); + + // When + var result = new ArrayList(); + sut.forEachByte(result::add); + + // Then + assertThat(result).containsExactly((byte) 20, (byte) 10, (byte) 20); + } + } + + @Test + void foldThroughCodes() { + try (Arena arena = Arena.ofConfined()) { + // Given codes = [0, 1, 1] -> values = [1, 5, 5] -> sum 11 + ByteArray values = byteArray(arena, (byte) 1, (byte) 5); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 1, (byte) 1); + DictByteArray sut = DictByteArray.of(U8, 3, values, codes); + + // When + long result = sut.fold(0L, Long::sum); + + // Then + assertThat(result).isEqualTo(11L); + } + } + + @Test + void allCodeTypes_getForEachFoldMaterialize() { + try (Arena arena = Arena.ofConfined()) { + // Given — same selection [2,0,1] expressed as U8/U16/U32/U64 codes + ByteArray values = byteArray(arena, (byte) 100, (byte) 101, (byte) 102); + byte[] expected = {102, 100, 101}; + List codeVariants = List.of( + byteArray(arena, (byte) 2, (byte) 0, (byte) 1), + shortArray(arena, U16, (short) 2, (short) 0, (short) 1), + intArray(arena, U32, 2, 0, 1), + longArray(arena, U64, 2L, 0L, 1L)); + + for (Array codes : codeVariants) { + DictByteArray sut = DictByteArray.of(U8, 3, values, codes); + String label = codes.getClass().getSimpleName(); + + // getter + for (int i = 0; i < 3; i++) { + assertThat(sut.getByte(i)).as(label).isEqualTo(expected[i]); + } + // forEach + var seen = new ArrayList(); + sut.forEachByte(seen::add); + assertThat(seen).as(label).containsExactly((byte) 102, (byte) 100, (byte) 101); + // fold + assertThat(sut.fold(0L, Long::sum)).as(label).isEqualTo(303L); + // materialize + MemorySegment m = sut.materialize(arena); + for (int i = 0; i < 3; i++) { + assertThat(m.get(ValueLayout.JAVA_BYTE, i)).as(label).isEqualTo(expected[i]); + } + } + } + } + + @Test + void bulkOps_invalidCodesType_throw() { + // Given — a record built directly (bypassing of()) with a non-int codes array + try (Arena arena = Arena.ofConfined()) { + ByteArray values = byteArray(arena, (byte) 1); + DictByteArray sut = new DictByteArray(U8, 1, values, floatArray(arena, 0f)); + + // When / Then — every bulk path hits the defensive default arm + assertThatThrownBy(() -> sut.materialize(arena)) + .isInstanceOf(VortexException.class).hasMessageContaining("invalid codes"); + assertThatThrownBy(() -> sut.forEachByte(v -> { })) + .isInstanceOf(VortexException.class); + assertThatThrownBy(() -> sut.fold(0L, Long::sum)) + .isInstanceOf(VortexException.class); + } + } + } + + @Nested + class DictShort { + + @Test + void u8CodesDispatch() { + try (Arena arena = Arena.ofConfined()) { + // Given + ShortArray values = shortArray(arena, U16, (short) 100, (short) 200, (short) 300); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 2); + DictShortArray sut = DictShortArray.of(U16, 2, values, codes); + + // When/Then + assertThat(sut.getShort(0)).isEqualTo((short) 100); + assertThat(sut.getShort(1)).isEqualTo((short) 300); + } + } + + @Test + void u16DtypeGetIntWidensUnsigned() { + try (Arena arena = Arena.ofConfined()) { + // Given a U16 pool holding 40000 (negative as a signed short) — the U16 + // sibling of the uci-wine U8 corner: values above 32767 must widen unsigned. + ShortArray values = shortArray(arena, U16, (short) 40000, (short) 7); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 1); + DictShortArray sut = DictShortArray.of(U16, 2, values, codes); + + // When + int result = sut.getInt(0); + + // Then + assertThat(result).isEqualTo(40000); + } + } + + @Test + void forEachShortMaterializesInOrder() { + try (Arena arena = Arena.ofConfined()) { + // Given + ShortArray values = shortArray(arena, U16, (short) 10, (short) 20); + ByteArray codes = byteArray(arena, (byte) 1, (byte) 0, (byte) 1); + DictShortArray sut = DictShortArray.of(U16, 3, values, codes); + + // When + var result = new ArrayList(); + sut.forEachShort(result::add); + + // Then + assertThat(result).containsExactly((short) 20, (short) 10, (short) 20); + } + } + + @Test + void foldThroughCodes() { + try (Arena arena = Arena.ofConfined()) { + // Given codes = [0, 1, 1] -> values = [1, 5, 5] -> sum 11 + ShortArray values = shortArray(arena, U16, (short) 1, (short) 5); + ByteArray codes = byteArray(arena, (byte) 0, (byte) 1, (byte) 1); + DictShortArray sut = DictShortArray.of(U16, 3, values, codes); + + // When + long result = sut.fold(0L, Long::sum); + + // Then + assertThat(result).isEqualTo(11L); + } + } + + @Test + void allCodeTypes_getForEachFoldMaterialize() { + try (Arena arena = Arena.ofConfined()) { + // Given — same selection [2,0,1] expressed as U8/U16/U32/U64 codes + ShortArray values = shortArray(arena, U16, (short) 100, (short) 200, (short) 300); + short[] expected = {300, 100, 200}; + List codeVariants = List.of( + byteArray(arena, (byte) 2, (byte) 0, (byte) 1), + shortArray(arena, U16, (short) 2, (short) 0, (short) 1), + intArray(arena, U32, 2, 0, 1), + longArray(arena, U64, 2L, 0L, 1L)); + + for (Array codes : codeVariants) { + DictShortArray sut = DictShortArray.of(U16, 3, values, codes); + String label = codes.getClass().getSimpleName(); + + // getter + for (int i = 0; i < 3; i++) { + assertThat(sut.getShort(i)).as(label).isEqualTo(expected[i]); + } + // forEach + var seen = new ArrayList(); + sut.forEachShort(seen::add); + assertThat(seen).as(label).containsExactly((short) 300, (short) 100, (short) 200); + // fold + assertThat(sut.fold(0L, Long::sum)).as(label).isEqualTo(600L); + // materialize + MemorySegment m = sut.materialize(arena); + for (int i = 0; i < 3; i++) { + assertThat(m.getAtIndex(VortexFormat.LE_SHORT, i)).as(label).isEqualTo(expected[i]); + } + } + } + } + + @Test + void bulkOps_invalidCodesType_throw() { + // Given — a record built directly (bypassing of()) with a non-int codes array + try (Arena arena = Arena.ofConfined()) { + ShortArray values = shortArray(arena, U16, (short) 1); + DictShortArray sut = new DictShortArray(U16, 1, values, floatArray(arena, 0f)); + + // When / Then — every bulk path hits the defensive default arm + assertThatThrownBy(() -> sut.materialize(arena)) + .isInstanceOf(VortexException.class).hasMessageContaining("invalid codes"); + assertThatThrownBy(() -> sut.forEachShort(v -> { })) + .isInstanceOf(VortexException.class); + assertThatThrownBy(() -> sut.fold(0L, Long::sum)) + .isInstanceOf(VortexException.class); + } + } + } + @Nested class MaskedValues { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java index 7419d15a..9237af35 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java @@ -5,8 +5,11 @@ import io.github.dfa1.vortex.core.proto.ProtoALPMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FloatArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -21,7 +24,8 @@ class AlpEncodingDecoderTest { private static final AlpEncodingDecoder SUT = new AlpEncodingDecoder(); - private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(SUT, new PrimitiveEncodingDecoder()); + private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( + SUT, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); private static final DType F64 = DType.F64; private static final DType F32 = DType.F32; @@ -161,4 +165,90 @@ void decode_patches_nonUnsignedIndexPtype_throws() { // When / Then assertThatThrownBy(() -> SUT.decode(ctx)).hasMessageContaining("non-unsigned patch index ptype"); } + + /// An ALP array's validity IS its encoded child's (`ValidityChild`, #210): a nullable + /// encoded primitive child surfaces as a [MaskedArray], and that mask must ride through both the + /// per-row lazy path and the single-value constant-broadcast path rather than being flattened. + @Nested + class MaskedChildPropagation { + + @Test + void maskedEncodedChild_lazyPath_propagatesNullsAndDecodesValues() { + // Given — encoded I64 [100,200,300] with row 1 null; exp_e=2 scales by 0.01 + ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, + new ArrayNode[]{validityNode}, new int[]{0}); + byte[] meta = new ProtoALPMetadata(2, 0, null).encode(); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), + new ArrayNode[]{enc}, new int[0]); + MemorySegment[] segs = {leLongs(100L, 200L, 300L), boolBitmap(true, false, true)}; + DecodeContext ctx = new DecodeContext(node, F64, 3, segs, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — nulls survive and valid rows decode through the lazy ALP path + MaskedArray masked = (MaskedArray) result; + assertThat(masked.isValid(0)).isTrue(); + assertThat(masked.isValid(1)).isFalse(); + assertThat(masked.isValid(2)).isTrue(); + DoubleArray inner = (DoubleArray) masked.inner(); + assertThat(inner.getDouble(0)).isCloseTo(1.0, within(1e-9)); + assertThat(inner.getDouble(2)).isCloseTo(3.0, within(1e-9)); + } + + @Test + void maskedEncodedChild_constantBroadcast_preservesValidity() { + // Given — a single encoded value broadcast to 3 rows (capacity < n, no patches) routes + // through the LazyConstant arm; the validity mask must survive that path too. + ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, + new ArrayNode[]{validityNode}, new int[]{0}); + byte[] meta = new ProtoALPMetadata(2, 0, null).encode(); // *0.01 + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), + new ArrayNode[]{enc}, new int[0]); + MemorySegment[] segs = {leLongs(100L), boolBitmap(true, false, true)}; + DecodeContext ctx = new DecodeContext(node, F64, 3, segs, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — every row is the broadcast constant 1.0, and row 1 stays null + MaskedArray masked = (MaskedArray) result; + assertThat(masked.isValid(0)).isTrue(); + assertThat(masked.isValid(1)).isFalse(); + assertThat(masked.isValid(2)).isTrue(); + DoubleArray inner = (DoubleArray) masked.inner(); + for (int i = 0; i < 3; i++) { + assertThat(inner.getDouble(i)).as("row %d", i).isCloseTo(1.0, within(1e-9)); + } + } + + @Test + void plainEncodedChild_returnsPlainArray() { + // Given — a non-nullable encoded child (no validity): no-regression path must NOT mask + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + byte[] meta = new ProtoALPMetadata(2, 0, null).encode(); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), + new ArrayNode[]{enc}, new int[0]); + DecodeContext ctx = new DecodeContext(node, F64, 2, + new MemorySegment[]{leLongs(100L, 200L)}, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then + assertThat(result).isInstanceOf(DoubleArray.class).isNotInstanceOf(MaskedArray.class); + } + + private MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } + } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java new file mode 100644 index 00000000..e29a5e82 --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java @@ -0,0 +1,211 @@ +package io.github.dfa1.vortex.reader.decode; + +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; +import io.github.dfa1.vortex.core.proto.ProtoPType; +import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; +import io.github.dfa1.vortex.encoding.TestSegments; +import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/// Pins the validity-child propagation that [BitpackedEncodingDecoder] performs for the +/// `ValidityChild` shape (#210): a trailing bool validity child that stacked +/// encodings (ALP, FoR) delegate down to bitpacked, and whose position in the child vector +/// depends on the patch shape. Dropping it silently un-nulls rows. +/// +/// Every fixture uses `bit_width=0`, the real constant-residual shape seen when bitpacked is +/// nested under FoR/RLE (penguins `bill_length` decodes alp -> for -> bitpacked with the +/// residual collapsed to zero width). That makes the unpacked base deterministically all-zero, +/// so the tests isolate validity-index selection and mask wrapping from the packed-bit unpack. +/// Patch children then write real non-zero values, proving the patch path and validity index +/// coexist. +class BitpackedEncodingDecoderTest { + + private static final BitpackedEncodingDecoder SUT = new BitpackedEncodingDecoder(); + private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( + SUT, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); + + private static final EncodingId BITPACKED = EncodingId.FASTLANES_BITPACKED; + private static final EncodingId PRIMITIVE = EncodingId.VORTEX_PRIMITIVE; + private static final EncodingId BOOL = EncodingId.VORTEX_BOOL; + private static final DType I32 = new DType.Primitive(io.github.dfa1.vortex.core.model.PType.I32, true); + + @Test + void encodingId_isFastlanesBitpacked() { + // Given / When / Then + assertThat(SUT.encodingId()).isEqualTo(EncodingId.FASTLANES_BITPACKED); + } + + @Nested + class ValidityChildIndex { + + @Test + void noPatches_trailingValidityAtIndexZero_masksRows() { + // Given — no patches, so the validity child is the first (index 0) child. This is the + // penguins bill_length shape: bitpacked residual (bit_width 0 -> all zero) carrying only + // a validity child. Rows 0,2,4 valid; 1,3,5 null. + byte[] meta = new ProtoBitPackedMetadata(0, 0, null).encode(); + MemorySegment validity = boolBitmap(true, false, true, false, true, false); + ArrayNode validityNode = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[]{validityNode}, new int[]{0}); + DecodeContext ctx = new DecodeContext(node, I32, 6, + new MemorySegment[]{empty(), validity}, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — a MaskedArray whose validity mirrors the child; base values all zero + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false, true, false); + IntArray inner = (IntArray) masked.inner(); + for (int i = 0; i < 6; i++) { + assertThat(inner.getInt(i)).as("row %d", i).isZero(); + } + } + + @Test + void patchesWithoutChunkOffsets_validityAtIndexTwo_masksRows() { + // Given — patches present but no chunk offsets: children are [indices, values, validity], + // so validity sits at index 2. One patch overwrites row 2 with 700; rows 1,4 null. + ProtoPatchesMetadata pm = new ProtoPatchesMetadata(1L, 0L, ProtoPType.U8, null, null, null); + byte[] meta = new ProtoBitPackedMetadata(0, 0, pm).encode(); + + ArrayNode indices = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode values = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode validityNode = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{3}); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[]{indices, values, validityNode}, new int[]{0}); + + MemorySegment[] segs = { + empty(), // 0: packed (unread, bit_width 0) + MemorySegment.ofArray(new byte[]{2}), // 1: patch index -> row 2 + TestSegments.leInts(700), // 2: patch value + boolBitmap(true, false, true, true, false) // 3: validity + }; + DecodeContext ctx = new DecodeContext(node, I32, 5, segs, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — validity at index 2 survives; the patch wrote row 2 + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, true, false); + IntArray inner = (IntArray) masked.inner(); + assertThat(inner.getInt(2)).isEqualTo(700); + assertThat(inner.getInt(0)).isZero(); + assertThat(inner.getInt(4)).isZero(); + } + + @Test + void patchesWithChunkOffsets_validityAtIndexThree_masksRows() { + // Given — patches carrying chunk offsets: children are [indices, values, chunkOffsets, + // validity], pushing validity to index 3. The chunk-offsets child is never decoded by the + // patch path, so it is a placeholder. One patch overwrites row 1 with 900; rows 3,4 null. + ProtoPatchesMetadata pm = new ProtoPatchesMetadata(1L, 0L, ProtoPType.U8, 1L, ProtoPType.U32, 0L); + byte[] meta = new ProtoBitPackedMetadata(0, 0, pm).encode(); + + ArrayNode indices = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode values = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode chunkOffsets = new ArrayNode(PRIMITIVE, null, new ArrayNode[0], new int[]{0}); // placeholder + ArrayNode validityNode = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{3}); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[]{indices, values, chunkOffsets, validityNode}, new int[]{0}); + + MemorySegment[] segs = { + empty(), + MemorySegment.ofArray(new byte[]{1}), // patch index -> row 1 + TestSegments.leInts(900), // patch value + boolBitmap(true, true, true, false, false) + }; + DecodeContext ctx = new DecodeContext(node, I32, 5, segs, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then — validity at index 3 survives; the patch wrote row 1 + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, true, true, false, false); + IntArray inner = (IntArray) masked.inner(); + assertThat(inner.getInt(1)).isEqualTo(900); + assertThat(inner.getInt(0)).isZero(); + } + + @Test + void noValidityChild_returnsPlainArray() { + // Given — no patches and zero children: nothing to mask, so the decoder returns the bare + // values array, never a MaskedArray (the no-regression path for non-nullable columns). + byte[] meta = new ProtoBitPackedMetadata(0, 0, null).encode(); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[0], new int[]{0}); + DecodeContext ctx = new DecodeContext(node, I32, 4, + new MemorySegment[]{empty()}, REGISTRY, Arena.ofAuto()); + + // When + Array result = SUT.decode(ctx); + + // Then + assertThat(result).isInstanceOf(IntArray.class).isNotInstanceOf(MaskedArray.class); + } + + @Test + void unexpectedChildCount_throwsWithExpectedCounts() { + // Given — no patches (expected 0 or 1 children) but two children are present: an + // ambiguous shape a trusted encoder never emits, so decode fails loudly. + byte[] meta = new ProtoBitPackedMetadata(0, 0, null).encode(); + ArrayNode a = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode b = new ArrayNode(BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta), + new ArrayNode[]{a, b}, new int[]{0}); + DecodeContext ctx = new DecodeContext(node, I32, 4, + new MemorySegment[]{empty(), boolBitmap(true, true, true, true)}, REGISTRY, Arena.ofAuto()); + + // When / Then + assertThatThrownBy(() -> SUT.decode(ctx)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("expected 0 or 1 children") + .hasMessageContaining("got 2"); + } + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + private static MaskedArray assertMasked(Array result) { + assertThat(result).isInstanceOf(MaskedArray.class); + return (MaskedArray) result; + } + + private static void assertValidity(MaskedArray masked, boolean... expected) { + for (int i = 0; i < expected.length; i++) { + assertThat(masked.isValid(i)).as("valid row %d", i).isEqualTo(expected[i]); + } + } + + /// Builds an LSB-first packed validity bitmap (`MaterializedBoolArray` layout) where bit `i` + /// is set when `valid[i]` is true. + private static MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } + + private static MemorySegment empty() { + return MemorySegment.ofArray(new byte[0]); + } +} diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java index bdab8a31..85096013 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java @@ -14,6 +14,7 @@ import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.ShortArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import org.junit.jupiter.api.Nested; @@ -35,7 +36,7 @@ class DictEncodingDecoderTest { private static final DictEncodingDecoder SUT = new DictEncodingDecoder(); private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( - SUT, new PrimitiveEncodingDecoder(), new VarBinEncodingDecoder()); + SUT, new PrimitiveEncodingDecoder(), new VarBinEncodingDecoder(), new BoolEncodingDecoder()); @Test void encodingId_isVortexDict() { @@ -416,6 +417,145 @@ void protoLayout_emptyMetadata_throws() { } } + /// Per-row validity for the Rust proto primitive path (`decodeRustProto`, #210). A DictArray + /// row is null when its CODE is null (codes-side validity child) or when the code points at an + /// invalid pool slot (pool-null representation); both arrive as [MaskedArray] children and must + /// combine per row. Shapes mirror real Kepler columns: `koi_gmag` is pool-null, `koi_smet_err2` + /// is codes-side. + @Nested + class RowValidity { + + @Test + void poolNull_masksRowsPointingAtInvalidSlot() { + // Given — pool [10,20,30] with slot 1 invalid (koi_gmag shape); codes point rows 1 and 3 + // at that dead slot, so those rows are null while their expanded value is still present. + ArrayNode codesNode = primitiveNode(0); // plain codes: no codes-side nulls + ArrayNode valuesNode = maskedPrimitiveNode(1, 2); // masked pool: slot validity + MemorySegment[] segs = { + u8Codes(0, 1, 2, 1), + TestSegments.leInts(10, 20, 30), + boolBitmap(true, false, true) // slot 1 invalid + }; + + // When + Array result = decodeDict(DType.I32, PType.U8, 3, 4, segs, codesNode, valuesNode); + + // Then + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void codesNull_masksRowsWithNullCode() { + // Given — pool [100,200] all valid; the codes child carries its own validity + // (koi_smet_err2 shape). Row nulls must mirror the codes mask exactly. + ArrayNode codesNode = maskedPrimitiveNode(0, 1); // masked codes: rows 1,3 null + ArrayNode valuesNode = primitiveNode(2); // plain pool: all valid + MemorySegment[] segs = { + u8Codes(0, 1, 0, 1), + boolBitmap(true, false, true, false), + TestSegments.leInts(100, 200) + }; + + // When + Array result = decodeDict(DType.I32, PType.U8, 2, 4, segs, codesNode, valuesNode); + + // Then + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 100, 200, 100, 200); + } + + @Test + void bothSides_combineWithAndSemantics() { + // Given — codes-side nulls AND a pool-null slot: row i is valid iff its code is valid and + // the pool slot it references is valid. Row 2 is null via its code; rows 1,3 via the pool. + ArrayNode codesNode = maskedPrimitiveNode(0, 1); // codes: row 2 null + ArrayNode valuesNode = maskedPrimitiveNode(2, 3); // pool: slot 1 invalid + MemorySegment[] segs = { + u8Codes(0, 1, 2, 1), + boolBitmap(true, true, false, true), // codes valid except row 2 + TestSegments.leInts(10, 20, 30), + boolBitmap(true, false, true) // pool slot 1 invalid + }; + + // When + Array result = decodeDict(DType.I32, PType.U8, 3, 4, segs, codesNode, valuesNode); + + // Then — only row 0 survives both masks + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, false, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void codeOutOfRangeWithPoolValidity_throwsVortexException() { + // Given — a single-element pool (so expandU8 broadcasts, never reading out of bounds) with + // pool validity present, and an untrusted code 5 that overruns the validity bitmap. The + // poolValid guard must convert the overrun into a VortexException, not an IOOBE. + ArrayNode codesNode = primitiveNode(0); + ArrayNode valuesNode = maskedPrimitiveNode(1, 2); + MemorySegment[] segs = { + u8Codes(0, 5), + TestSegments.leInts(10), + boolBitmap(true) // pool length 1 + }; + + // When / Then + assertThatThrownBy(() -> decodeDict(DType.I32, PType.U8, 1, 2, segs, codesNode, valuesNode)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("out of range for pool validity"); + } + + private Array decodeDict(DType dtype, PType codePType, int valuesLen, long rowCount, + MemorySegment[] segs, ArrayNode codesNode, ArrayNode valuesNode) { + MemorySegment meta = MemorySegment.ofArray( + new ProtoDictMetadata(valuesLen, protoPType(codePType), null, null).encode()); + ArrayNode dictNode = new ArrayNode(EncodingId.VORTEX_DICT, meta, + new ArrayNode[]{codesNode, valuesNode}, new int[]{}); + DecodeContext ctx = new DecodeContext(dictNode, dtype, rowCount, segs, REGISTRY, Arena.ofAuto()); + return SUT.decode(ctx); + } + + private MaskedArray assertMasked(Array result) { + assertThat(result).isInstanceOf(MaskedArray.class); + return (MaskedArray) result; + } + + private void assertValidity(MaskedArray masked, boolean... expected) { + for (int i = 0; i < expected.length; i++) { + assertThat(masked.isValid(i)).as("valid row %d", i).isEqualTo(expected[i]); + } + } + + private void assertIntValues(MaskedArray masked, int... expected) { + IntArray inner = (IntArray) masked.inner(); + for (int i = 0; i < expected.length; i++) { + assertThat(inner.getInt(i)).as("row %d", i).isEqualTo(expected[i]); + } + } + + /// A primitive node whose single validity child (a `vortex.bool` bitmap) makes + /// [PrimitiveEncodingDecoder] surface it as a [MaskedArray]. + private ArrayNode maskedPrimitiveNode(int dataBufIndex, int validityBufIndex) { + ArrayNode validity = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], + new int[]{validityBufIndex}); + return new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[]{validity}, + new int[]{dataBufIndex}); + } + + private MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } + } + // ── parameter sources ────────────────────────────────────────────────────── static Stream codeAndValueTypes() { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java index af4b5da5..cce4dd3a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java @@ -10,7 +10,9 @@ import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.ShortArray; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -23,7 +25,8 @@ class ZigZagEncodingDecoderTest { private static final ZigZagEncodingDecoder SUT = new ZigZagEncodingDecoder(); - private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(SUT, new PrimitiveEncodingDecoder()); + private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( + SUT, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); // --- zigzag encode helpers (mirror of the decoder's (u >>> 1) ^ -(u & 1)) --- @@ -211,4 +214,62 @@ void decode_nonPrimitiveDtype_throws() { .isInstanceOf(VortexException.class) .hasMessageContaining("expected primitive dtype"); } + + /// A zigzag array's validity IS its encoded child's (`ValidityChild`, #210): when the + /// encoded primitive child is nullable it arrives as a [MaskedArray], and that mask must ride + /// through to the decoded result rather than being flattened to a raw segment. + @Nested + class MaskedChildPropagation { + + @Test + void maskedEncodedChild_propagatesNullsAndDecodesValues() { + // Given — encoded I32 child [5,-3,7] with a validity child marking row 1 null + MemorySegment encoded = encodedInts(5, -3, 7); + MemorySegment validity = boolBitmap(true, false, true); + + // When + Array result = decodeMasked(PType.I32, 3, encoded, validity); + + // Then — nulls survive and valid rows decode correctly + MaskedArray masked = (MaskedArray) result; + assertThat(masked.isValid(0)).isTrue(); + assertThat(masked.isValid(1)).isFalse(); + assertThat(masked.isValid(2)).isTrue(); + IntArray inner = (IntArray) masked.inner(); + assertThat(inner.getInt(0)).isEqualTo(5); + assertThat(inner.getInt(1)).isEqualTo(-3); + assertThat(inner.getInt(2)).isEqualTo(7); + } + + @Test + void plainEncodedChild_returnsPlainArray() { + // Given — a non-nullable encoded child (no validity): the no-regression path must NOT + // wrap the result in a MaskedArray. + Array result = decode(PType.I32, 3, encodedInts(5, -3, 7)); + + // Then + assertThat(result).isInstanceOf(IntArray.class).isNotInstanceOf(MaskedArray.class); + } + + private Array decodeMasked(PType signed, long n, MemorySegment encoded, MemorySegment validity) { + DType dtype = new DType.Primitive(signed, true); + ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); + ArrayNode child = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, + new ArrayNode[]{validityNode}, new int[]{0}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZIGZAG, null, new ArrayNode[]{child}, new int[]{}); + DecodeContext ctx = new DecodeContext(node, dtype, n, + new MemorySegment[]{encoded, validity}, REGISTRY, Arena.ofAuto()); + return SUT.decode(ctx); + } + + private MemorySegment boolBitmap(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return MemorySegment.ofArray(bytes); + } + } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java new file mode 100644 index 00000000..2cd117f4 --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java @@ -0,0 +1,187 @@ +package io.github.dfa1.vortex.reader.layout; + +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.LayoutId; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.encoding.TestSegments; +import io.github.dfa1.vortex.reader.SegmentSpec; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; +import io.github.dfa1.vortex.reader.array.MaterializedByteArray; +import io.github.dfa1.vortex.reader.array.MaterializedIntArray; +import org.junit.jupiter.api.Test; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/// Pins the pool/codes row-validity gather in [DictLayoutDecoder.buildLazyDictPrimitive] (#210). +/// +/// Driven through the public [DictLayoutDecoder#decode(LayoutDecodeContext, Layout, DType)] seam +/// with a stub [LayoutDecodeContext] that hands the decoder pre-built values and codes arrays — the +/// closest constructible seam, since the private gather takes already-decoded children and building +/// a real on-disk dict layout fixture would be disproportionate. Shapes mirror real Kepler columns: +/// `koi_gmag` (pool-null) and `koi_smet_err2` (codes-side). +class DictLayoutDecoderTest { + + private static final DType I32 = new DType.Primitive(PType.I32, true); + + @Test + void poolNull_masksRowsPointingAtInvalidSlot() { + // Given — pool [10,20,30] with slot 1 invalid; codes route rows 1,3 at the dead slot + Array values = new MaskedArray(intPool(10, 20, 30), boolArray(true, false, true)); + Array codes = byteCodes(0, 1, 2, 1); + + // When + Array result = decode(4, I32, values, codes); + + // Then + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void codesNull_masksRowsWithNullCode() { + // Given — an all-valid pool but a codes child carrying its own validity (rows 1,3 null) + Array values = intPool(100, 200); + Array codes = new MaskedArray(byteCodes(0, 1, 0, 1), boolArray(true, false, true, false)); + + // When + Array result = decode(4, I32, values, codes); + + // Then — the codes mask passes straight through unchanged + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 100, 200, 100, 200); + } + + @Test + void bothSides_combineWithAndSemantics() { + // Given — codes-side null on row 2 AND a pool-null slot 1: a row is valid iff both hold + Array values = new MaskedArray(intPool(10, 20, 30), boolArray(true, false, true)); + Array codes = new MaskedArray(byteCodes(0, 1, 2, 1), boolArray(true, true, false, true)); + + // When + Array result = decode(4, I32, values, codes); + + // Then — only row 0 survives both masks + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, false, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void codeOutOfRangeWithPoolValidity_throwsVortexException() { + // Given — pool validity of length 1 and an untrusted code 5 overrunning it; the gather guard + // must raise a VortexException, not a raw JDK IndexOutOfBoundsException + Array values = new MaskedArray(intPool(10), boolArray(true)); + Array codes = byteCodes(0, 5); + + // When / Then + assertThatThrownBy(() -> decode(2, I32, values, codes)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("out of range for pool validity"); + } + + @Test + void noValidityEitherSide_returnsPlainDictArray() { + // Given — all-valid pool and plain codes: nothing to mask (no-regression path) + Array values = intPool(10, 20, 30); + Array codes = byteCodes(0, 1, 2); + + // When + Array result = decode(3, I32, values, codes); + + // Then + assertThat(result).isInstanceOf(IntArray.class).isNotInstanceOf(MaskedArray.class); + } + + // ── harness ───────────────────────────────────────────────────────────────── + + private static Array decode(long n, DType dtype, Array values, Array codes) { + // metadata null -> codesPType defaults to U8, matching the byte codes built below + Layout valuesLayout = new Layout(LayoutId.FLAT, values.length(), null, List.of(), List.of()); + Layout codesLayout = new Layout(LayoutId.FLAT, n, null, List.of(), List.of()); + Layout dictLayout = new Layout(LayoutId.DICT, n, null, + List.of(valuesLayout, codesLayout), List.of()); + LayoutDecodeContext ctx = new StubContext(valuesLayout, values, codesLayout, codes); + return new DictLayoutDecoder().decode(ctx, dictLayout, dtype); + } + + private static IntArray intPool(int... values) { + return new MaterializedIntArray(I32, values.length, TestSegments.leInts(values)); + } + + private static Array byteCodes(int... codes) { + byte[] bytes = new byte[codes.length]; + for (int i = 0; i < codes.length; i++) { + bytes[i] = (byte) codes[i]; + } + return new MaterializedByteArray(new DType.Primitive(PType.U8, false), codes.length, + MemorySegment.ofArray(bytes)); + } + + private static BoolArray boolArray(boolean... valid) { + byte[] bytes = new byte[(valid.length + 7) / 8]; + for (int i = 0; i < valid.length; i++) { + if (valid[i]) { + bytes[i >>> 3] |= (byte) (1 << (i & 7)); + } + } + return new MaterializedBoolArray(DType.BOOL, valid.length, MemorySegment.ofArray(bytes)); + } + + private static MaskedArray assertMasked(Array result) { + assertThat(result).isInstanceOf(MaskedArray.class); + return (MaskedArray) result; + } + + private static void assertValidity(MaskedArray masked, boolean... expected) { + for (int i = 0; i < expected.length; i++) { + assertThat(masked.isValid(i)).as("valid row %d", i).isEqualTo(expected[i]); + } + } + + private static void assertIntValues(MaskedArray masked, int... expected) { + IntArray inner = (IntArray) masked.inner(); + for (int i = 0; i < expected.length; i++) { + assertThat(inner.getInt(i)).as("row %d", i).isEqualTo(expected[i]); + } + } + + /// Stub context that hands the decoder pre-built children keyed by layout identity, and + /// allocates the gathered validity bitmap from a real arena. Segment I/O is never reached on + /// the primitive dict path, so those methods are unsupported. + private record StubContext(Layout valuesLayout, Array values, Layout codesLayout, Array codes) + implements LayoutDecodeContext { + + @Override + public Array decodeChild(Layout child, DType dtype) { + return child == valuesLayout ? values : codes; + } + + @Override + public Array decodeSegment(SegmentSpec spec, DType dtype, long rowCount) { + throw new UnsupportedOperationException("not used on the primitive dict path"); + } + + @Override + public SegmentSpec segmentSpec(int index) { + throw new UnsupportedOperationException("not used on the primitive dict path"); + } + + @Override + public SegmentAllocator arena() { + return Arena.ofAuto(); + } + } +} diff --git a/scripts/hydrate-raincloud-corpus.sh b/scripts/hydrate-raincloud-corpus.sh new file mode 100755 index 00000000..5b11e76c --- /dev/null +++ b/scripts/hydrate-raincloud-corpus.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Hydrates the Raincloud conformance corpus (issue #205) for +# RaincloudConformanceIntegrationTest. +# +# Installs the pinned Raincloud release into a private venv, resolves each corpus +# slug via the raincloud loader (cache -> mirror -> local build), and writes the +# manifest TSV the integration test discovers: +# \t\t +# +# Usage: +# scripts/hydrate-raincloud-corpus.sh [--max-mb N] [slug ...] +# +# With no slugs, hydrates every entry of the conformance matrix +# (integration/src/test/resources/raincloud/expected-status.csv) whose combined +# artifact size fits --max-mb (default 200; use --max-mb 0 for no size cap — the +# full corpus is hundreds of GB and includes multi-hour builds). +# +# Per-slug failures (rotted upstream URL, missing Kaggle/HF credentials, size cap) +# skip the slug and keep going: the test only sees what hydrated. Set +# RAINCLOUD_MIRROR to turn builds into downloads. +set -euo pipefail + +RAINCLOUD_TAG="v0.2.1" +CACHE_ROOT="${RAINCLOUD_HOME:-$HOME/.cache/raincloud}" +MANIFEST="${RAINCLOUD_CORPUS_MANIFEST:-$CACHE_ROOT/corpus-manifest.tsv}" +VENV="$CACHE_ROOT/conformance-venv-$RAINCLOUD_TAG" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +MATRIX="$REPO_ROOT/integration/src/test/resources/raincloud/expected-status.csv" + +MAX_MB=200 +if [[ "${1:-}" == "--max-mb" ]]; then + if [[ -z "${2:-}" ]]; then + echo "error: --max-mb needs a value" >&2 + exit 2 + fi + MAX_MB="$2" + shift 2 +fi +# after the flag: any remaining --max-mb is misplaced, not a slug +for arg in "$@"; do + if [[ "$arg" == --* ]]; then + echo "error: flags must come before slugs: $arg" >&2 + exit 2 + fi +done + +if [[ ! -d "$VENV" ]]; then + echo "creating venv for raincloud@$RAINCLOUD_TAG" + python3 -m venv "$VENV" + "$VENV/bin/pip" --quiet install "raincloud[build] @ git+https://github.com/spiraldb/raincloud@$RAINCLOUD_TAG" +fi + +if [[ $# -gt 0 ]]; then + SLUGS=("$@") +else + # not mapfile: macOS ships bash 3.2 + SLUGS=() + while IFS= read -r slug; do + SLUGS+=("$slug") + done < <(grep -v '^#' "$MATRIX" | cut -d, -f1) +fi + +# bash 3.2 + set -u: expanding an empty array is an unbound-variable error +if [ ${#SLUGS[@]} -eq 0 ]; then + echo "error: no slugs to hydrate (empty matrix?)" >&2 + exit 2 +fi + +mkdir -p "$(dirname "$MANIFEST")" +printf '%s\n' "${SLUGS[@]}" | MAX_MB="$MAX_MB" MANIFEST="$MANIFEST" "$VENV/bin/python" - <<'PY' +import json +import os +import sys +from importlib import resources + +import raincloud + +max_bytes = int(os.environ["MAX_MB"]) * 1024 * 1024 +snapshot = json.loads(resources.files("raincloud").joinpath("_data/snapshot.json").read_text()) +sizes = { + slug: entry.get("parquet_bytes", 0) + entry.get("vortex_bytes", 0) + for slug, entry in snapshot["slugs"].items() +} + +hydrated, skipped = 0, 0 +with open(os.environ["MANIFEST"], "w") as manifest: + for slug in (line.strip() for line in sys.stdin if line.strip()): + size = sizes.get(slug, 0) + if max_bytes and size > max_bytes: + print(f"skip {slug}: {size / 1e6:.0f} MB exceeds --max-mb") + skipped += 1 + continue + try: + vortex = raincloud.load(slug, format="vortex").path() + parquet = raincloud.load(slug, format="parquet").path() + except Exception as e: # rotted URL, missing credentials, build failure + print(f"skip {slug}: {type(e).__name__}: {e}") + skipped += 1 + continue + manifest.write(f"{slug}\t{vortex}\t{parquet}\n") + hydrated += 1 + print(f"ok {slug}") +print(f"\nhydrated={hydrated} skipped={skipped} manifest={os.environ['MANIFEST']}") +PY