fix: nullable vortex.bool columns decoded as false instead of null

dfa1 · claude · dfa1 · commit 39c1d0847a1f · 2026-07-10T20:52:28.000+02:00
BoolEncodingDecoder ignored validity children entirely. When a vortex.bool
node has one child, that child is a bit-packed validity bitmask (false = null
row) — the same pattern used by PrimitiveEncodingDecoder. Null rows were
silently decoded as 'false', causing silent corruption in any corpus file with
a nullable boolean column (e.g. bi-eixo st_extemporaneo_matricula).

Also refactors RaincloudConformanceIntegrationTest to stream oracle and vortex
export in parallel via virtual threads + PipedWriter/PipedReader, comparing
line-by-line without temp files. This eliminates the OOM for large slugs
(bi-bimbo, 74 M rows) and roughly halves wall time by overlapping I/O.

Conformance matrix: bi-bimbo, bi-eixo, bi-medicare3, bi-medpayment1 → ok.

Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -37,6 +37,8 @@ input (`ConstantEncodingDecoder`, `AlpRdEncodingDecoder`, `SparseEncodingDecoder
 - `DateTimePartsArrays.readLong` now zero-extends unsigned children (`U8`/`U16`/`U32`) instead of sign-extending them; previously a `U16` seconds-within-day value ≥ 32 768 was widened as a negative short, producing a 65 536-second error in the reconstructed timestamp (#252, bi-euro2016 corpus).
 - `RaincloudConformanceIntegrationTest` now writes both the vortex-java and Parquet oracle CSVs to temp files and streams them line-by-line for comparison; previously it materialized both into heap `String` objects, causing OOM for large corpus files (≥157 MB) that were previously hidden by throwing before reaching the corrupted line. ([#254](https://github.com/dfa1/vortex-java/issues/254))
 - `RaincloudConformanceIntegrationTest` oracle now formats `INT32`/`INT64` Parquet columns with a `UINT_32`/`UINT_64` logical-type annotation using `Integer.toUnsignedString`/`Long.toUnsignedString`, matching the unsigned representation that Vortex stores as `U32`/`U64`; previously the Parquet signed interpretation diverged for values above `Integer.MAX_VALUE` (e.g. `total_boosters` in the covid-world-vaccination-progress corpus). ([#253](https://github.com/dfa1/vortex-java/issues/253))
+- `RaincloudConformanceIntegrationTest` now runs the Parquet oracle and vortex-java export in parallel via virtual threads connected by `PipedWriter`/`PipedReader`, comparing line-by-line without temp files; previously both CSV streams were written to disk and then compared, causing OOM for large corpus slugs (e.g. bi-bimbo with 74 M rows) and doubling wall time. (internal)
+- `BoolEncodingDecoder` now handles nullable `vortex.bool` columns: when the encoding node has one child, it is decoded as a validity bitmask and the values array is wrapped in a `MaskedArray`; previously the validity child was silently ignored and null rows were decoded as `false`, causing silent corruption in any corpus file with a nullable boolean column (e.g. bi-eixo `st_extemporaneo_matricula`). (internal)
 
 ## [Unreleased]
 
diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java
@@ -17,10 +17,13 @@
 
 import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.PipedReader;
+import java.io.PipedWriter;
 import java.io.UncheckedIOException;
 import java.io.Writer;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -86,20 +89,81 @@ Stream<DynamicTest> conformancePerSlug() throws IOException {
     }
 
     private static void assertMatchesParquetOracle(Path vortex, Path parquet) throws IOException {
-        // Given — write both CSVs to temp files; never hold multi-GB strings in heap
-        Path oracleCsv = Files.createTempFile("oracle-", ".csv");
-        Path vortexCsv = Files.createTempFile("vortex-", ".csv");
-        try {
-            writeOracle(parquet, oracleCsv);
+        // Given — both producers run in parallel via virtual threads; compare line-by-line
+        // without temp files, cutting wall time roughly in half for large slugs.
+        AtomicReference<Throwable> oracleError = new AtomicReference<>();
+        AtomicReference<Throwable> vortexError = new AtomicReference<>();
+        PipedWriter oracleSink = new PipedWriter();
+        PipedWriter vortexSink = new PipedWriter();
+        try (BufferedReader oracleReader = new BufferedReader(new PipedReader(oracleSink, 1 << 17));
+             BufferedReader vortexReader = new BufferedReader(new PipedReader(vortexSink, 1 << 17))) {
 
-            // When
-            CsvExporter.exportCsv(vortex, vortexCsv, ExportOptions.defaults());
+            Thread oracleThread = Thread.ofVirtual().start(() -> {
+                try (oracleSink) {
+                    writeOracleCsv(parquet, oracleSink);
+                } catch (Throwable t) {
+                    oracleError.set(t);
+                }
+            });
+            Thread vortexThread = Thread.ofVirtual().start(() -> {
+                try (vortexSink) {
+                    CsvExporter.exportCsv(vortex, vortexSink, ExportOptions.defaults());
+                } catch (Throwable t) {
+                    vortexError.set(t);
+                }
+            });
 
-            // Then
-            assertFilesMatch(vortexCsv, oracleCsv);
-        } finally {
-            Files.deleteIfExists(oracleCsv);
-            Files.deleteIfExists(vortexCsv);
+            // When / Then
+            Throwable mainError = null;
+            try {
+                assertFilesMatch(vortexReader, oracleReader);
+            } catch (Throwable t) {
+                mainError = t;
+            }
+
+            try {
+                oracleThread.join();
+                vortexThread.join();
+            } catch (InterruptedException ie) {
+                Thread.currentThread().interrupt();
+            }
+
+            // Propagate in priority order: mismatch > decode gap > oracle abort
+            Throwable oe = oracleError.get();
+            Throwable ve = vortexError.get();
+            if (mainError instanceof AssertionError e) {
+                throw e;
+            }
+            if (ve instanceof VortexException e) {
+                throw e;
+            }
+            if (ve instanceof IndexOutOfBoundsException e) {
+                throw e;
+            }
+            if (oe instanceof TestAbortedException e) {
+                throw e;
+            }
+            if (oe != null) {
+                throw new TestAbortedException("oracle cannot read the parquet sibling: " + oe);
+            }
+            if (ve instanceof IOException e) {
+                throw e;
+            }
+            if (ve instanceof RuntimeException e) {
+                throw e;
+            }
+            if (ve != null) {
+                throw new RuntimeException(ve);
+            }
+            if (mainError instanceof IOException e) {
+                throw e;
+            }
+            if (mainError instanceof RuntimeException e) {
+                throw e;
+            }
+            if (mainError != null) {
+                throw new RuntimeException(mainError);
+            }
         }
     }
 
@@ -159,24 +223,32 @@ private static void assertStillFails(Path vortex, Path parquet, String status) t
         }
     }
 
-    /// Stream-compares two CSV files line by line without loading either into heap.
+    /// Stream-compares two CSV sources line by line without loading either into heap.
     ///
-    /// @param result the vortex-java export
-    /// @param oracle the parquet oracle export
-    /// @throws AssertionError  if any line differs or the files have different lengths
-    /// @throws IOException     if either file cannot be read
+    /// @param result the vortex-java export reader
+    /// @param oracle the parquet oracle reader
+    /// @throws AssertionError if any line differs or the sources have different lengths
+    /// @throws IOException    if either reader throws
+    private static void assertFilesMatch(BufferedReader result, BufferedReader oracle) throws IOException {
+        long lineNum = 0;
+        String rLine;
+        while ((rLine = result.readLine()) != null) {
+            lineNum++;
+            String oLine = oracle.readLine();
+            assertThat(oLine).as("oracle ended before vortex output at line %d", lineNum).isNotNull();
+            assertThat(rLine).as("line %d", lineNum).isEqualTo(oLine);
+        }
+        assertThat(oracle.readLine()).as("vortex output ended before oracle at line %d", lineNum + 1).isNull();
+    }
+
+    /// @param result the vortex-java export file
+    /// @param oracle the parquet oracle file
+    /// @throws AssertionError if any line differs or the files have different lengths
+    /// @throws IOException    if either file cannot be read
     private static void assertFilesMatch(Path result, Path oracle) throws IOException {
         try (BufferedReader r = Files.newBufferedReader(result);
              BufferedReader o = Files.newBufferedReader(oracle)) {
-            long lineNum = 0;
-            String rLine;
-            while ((rLine = r.readLine()) != null) {
-                lineNum++;
-                String oLine = o.readLine();
-                assertThat(oLine).as("oracle ended before vortex output at line %d", lineNum).isNotNull();
-                assertThat(rLine).as("line %d", lineNum).isEqualTo(oLine);
-            }
-            assertThat(o.readLine()).as("vortex output ended before oracle at line %d", lineNum + 1).isNull();
+            assertFilesMatch(r, o);
         }
     }
 
diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv
@@ -23,12 +23,12 @@ basketball,untriaged
 behavioral-risk-factor-surveillance-system,ok
 beir-msmarco,untriaged
 bi-arade,ok
-bi-bimbo,untriaged
+bi-bimbo,ok
 bi-citymaxcapita,ok
 bi-cmsprovider,untriaged
 bi-commongovernment,untriaged
 bi-corporations,ok
-bi-eixo,untriaged
+bi-eixo,ok
 bi-euro2016,ok
 bi-food,ok
 bi-generico,untriaged
@@ -39,8 +39,8 @@ bi-iglocations2,untriaged
 bi-iublibrary,ok
 bi-medicare1,untriaged
 bi-medicare2,untriaged
-bi-medicare3,untriaged
-bi-medpayment1,untriaged
+bi-medicare3,ok
+bi-medpayment1,ok
 bi-medpayment2,untriaged
 bi-mlb,untriaged
 bi-motos,untriaged
diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BoolEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BoolEncodingDecoder.java
@@ -1,12 +1,18 @@
 package io.github.dfa1.vortex.reader.decode;
 
+import io.github.dfa1.vortex.core.error.VortexException;
+import io.github.dfa1.vortex.core.model.DType;
 import io.github.dfa1.vortex.core.model.EncodingId;
 import io.github.dfa1.vortex.reader.array.Array;
+import io.github.dfa1.vortex.reader.array.BoolArray;
+import io.github.dfa1.vortex.reader.array.MaskedArray;
 import io.github.dfa1.vortex.reader.array.MaterializedBoolArray;
 
 /// Read-only decoder for `vortex.bool` (bit-packed boolean arrays, LSB first).
 ///
-/// Read-only decoder for bit-packed boolean arrays.
+/// When the encoding node has one child, that child is the validity bitmask:
+/// a [BoolArray] where `false` marks null rows. The values array is wrapped in
+/// a [MaskedArray] so callers see nulls as invalid rows.
 public final class BoolEncodingDecoder implements EncodingDecoder {
 
     @Override
@@ -16,6 +22,16 @@ public EncodingId encodingId() {
 
     @Override
     public Array decode(DecodeContext ctx) {
-        return new MaterializedBoolArray(ctx.dtype(), ctx.rowCount(), ctx.buffer(0));
+        long n = ctx.rowCount();
+        Array values = new MaterializedBoolArray(ctx.dtype(), n, ctx.buffer(0));
+        if (ctx.node().children().length == 1) {
+            Array va = ctx.decodeChild(0, DType.BOOL, n);
+            if (!(va instanceof BoolArray validity)) {
+                throw new VortexException(EncodingId.VORTEX_BOOL,
+                        "validity child decoded to unexpected type: " + va.getClass().getSimpleName());
+            }
+            return new MaskedArray(values, validity);
+        }
+        return values;
     }
 }
diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BoolEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BoolEncodingDecoderTest.java
@@ -0,0 +1,127 @@
+package io.github.dfa1.vortex.reader.decode;
+
+import io.github.dfa1.vortex.core.model.EncodingId;
+import io.github.dfa1.vortex.encoding.DTypes;
+import io.github.dfa1.vortex.reader.ReadRegistry;
+import io.github.dfa1.vortex.reader.array.BoolArray;
+import io.github.dfa1.vortex.reader.array.MaskedArray;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class BoolEncodingDecoderTest {
+
+    // Bit-packs a boolean[] LSB-first into a byte[]: bit i is at byte[i/8] bit (i%8).
+    private static MemorySegment packBits(boolean[] values) {
+        byte[] bytes = new byte[(values.length + 7) / 8];
+        for (int i = 0; i < values.length; i++) {
+            if (values[i]) {
+                bytes[i / 8] |= (byte) (1 << (i % 8));
+            }
+        }
+        return MemorySegment.ofArray(bytes);
+    }
+
+    // Builds a context with no validity child (non-nullable bool).
+    private static DecodeContext nonNullableCtx(boolean[] values) {
+        MemorySegment bits = packBits(values);
+        ArrayNode node = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{0});
+        ReadRegistry registry = TestRegistry.ofDecoders(new BoolEncodingDecoder());
+        return new DecodeContext(node, DTypes.BOOL, values.length, new MemorySegment[]{bits}, registry, Arena.ofAuto());
+    }
+
+    // Builds a context with a vortex.bool validity child (nullable bool).
+    // The validity child is itself bit-packed; false = null row.
+    private static DecodeContext nullableCtx(boolean[] values, boolean[] valid) {
+        MemorySegment bits = packBits(values);
+        MemorySegment validBits = packBits(valid);
+        ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1});
+        ArrayNode node = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[]{validityNode}, new int[]{0});
+        ReadRegistry registry = TestRegistry.ofDecoders(new BoolEncodingDecoder());
+        return new DecodeContext(node, DTypes.BOOL_N, values.length,
+                new MemorySegment[]{bits, validBits}, registry, Arena.ofAuto());
+    }
+
+    static Stream<Arguments> nonNullableCases() {
+        return Stream.of(
+                Arguments.of("empty", new boolean[]{}),
+                Arguments.of("all false", new boolean[]{false, false, false}),
+                Arguments.of("all true", new boolean[]{true, true, true}),
+                Arguments.of("mixed", new boolean[]{true, false, true, true, false}),
+                // Crosses a byte boundary: bit 7 is in byte 0, bit 8 is in byte 1.
+                Arguments.of("byte boundary", new boolean[]{false, false, false, false, false, false, false, true, false})
+        );
+    }
+
+    @ParameterizedTest(name = "{0}")
+    @MethodSource("nonNullableCases")
+    void decode_nonNullable_returnsBoolArray(String name, boolean[] values) {
+        // Given
+        DecodeContext ctx = nonNullableCtx(values);
+        var sut = new BoolEncodingDecoder();
+
+        // When
+        var result = sut.decode(ctx);
+
+        // Then — non-nullable path returns a plain BoolArray, not a MaskedArray
+        assertThat(result).isInstanceOf(BoolArray.class);
+        assertThat(result).isNotInstanceOf(MaskedArray.class);
+        assertThat(result.length()).isEqualTo(values.length);
+        BoolArray boolArr = (BoolArray) result;
+        for (int i = 0; i < values.length; i++) {
+            assertThat(boolArr.getBoolean(i)).as("index %d", i).isEqualTo(values[i]);
+        }
+    }
+
+    @Test
+    void decode_nullable_returnsMaskedArray_withNullsHidingUnderlying() {
+        // Given — rows 0,2 are valid; row 1 is null. The value bits at null positions are
+        // irrelevant, but a non-zero bit is stored there to confirm the decoder ignores them.
+        boolean[] values = {true, true, false};
+        boolean[] valid = {true, false, true};
+        DecodeContext ctx = nullableCtx(values, valid);
+        var sut = new BoolEncodingDecoder();
+
+        // When
+        var result = sut.decode(ctx);
+
+        // Then — nullable path wraps in MaskedArray; null rows report isValid=false.
+        assertThat(result).isInstanceOf(MaskedArray.class);
+        MaskedArray masked = (MaskedArray) result;
+        assertThat(masked.length()).isEqualTo(3);
+        assertThat(masked.isValid(0)).isTrue();
+        assertThat(masked.isValid(1)).isFalse();
+        assertThat(masked.isValid(2)).isTrue();
+        // Valid rows deliver the correct boolean value via the inner BoolArray.
+        BoolArray inner = (BoolArray) masked.inner();
+        assertThat(inner.getBoolean(0)).isTrue();
+        assertThat(inner.getBoolean(2)).isFalse();
+    }
+
+    @Test
+    void decode_nullable_allNulls_allRowsInvalid() {
+        // Given — every validity bit is false; values buffer content does not matter
+        boolean[] values = {false, false, false};
+        boolean[] valid = {false, false, false};
+        DecodeContext ctx = nullableCtx(values, valid);
+        var sut = new BoolEncodingDecoder();
+
+        // When
+        var result = sut.decode(ctx);
+
+        // Then
+        assertThat(result).isInstanceOf(MaskedArray.class);
+        MaskedArray masked = (MaskedArray) result;
+        for (int i = 0; i < 3; i++) {
+            assertThat(masked.isValid(i)).as("row %d must be null", i).isFalse();
+        }
+    }
+}
