test: gate-running interop test for null-fill Sparse and null-run RunEnd (#233)

Four JNI-written fixtures verify that the #225/#226 validity fixes hold on
every gate run, not just the weekly Raincloud conformance sweep:

- jniNullFillSparse_f64_nullRowsDecodeNull: ~98% null column, JNI chooses
  vortex.sparse with null fill; every unpatched row must decode as null.
- jniNullFillSparse_i64_nullRowsDecodeNull: same shape for i64 (nuclear_share_energy).
- jniNullRunRunEnd_i16_nullRowsDecodeNull: i16 null/value runs, JNI picks
  vortex.runend; null runs must decode as null, not the FoR base.
- jniNullRunRunEnd_i64_nullRowsDecodeNull: same for i64.

The encoding assertions are hard-coded (JNI version is pinned); a future JNI
bump that changes the choice should refresh the fixture.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: dfa1

integration/src/test/java/io/github/dfa1/vortex/integration/NullSparseRunEndInteropIntegrationTest.java

@@ -0,0 +1,254 @@
+package io.github.dfa1.vortex.integration;
+
+import dev.vortex.api.Session;
+import dev.vortex.api.VortexWriter;
+import dev.vortex.arrow.ArrowAllocation;
+import dev.vortex.jni.NativeLoader;
+import io.github.dfa1.vortex.inspect.InspectorTree;
+import io.github.dfa1.vortex.reader.ReadRegistry;
+import io.github.dfa1.vortex.reader.ScanOptions;
+import io.github.dfa1.vortex.reader.VortexReader;
+import io.github.dfa1.vortex.reader.array.Array;
+import io.github.dfa1.vortex.reader.array.DoubleArray;
+import io.github.dfa1.vortex.reader.array.LongArray;
+import io.github.dfa1.vortex.reader.array.MaskedArray;
+import io.github.dfa1.vortex.reader.array.ShortArray;
+import org.apache.arrow.c.ArrowArray;
+import org.apache.arrow.c.ArrowSchema;
+import org.apache.arrow.c.Data;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+import java.util.function.ObjIntConsumer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/// Gate-running interop cover for null-fill Sparse (#226) and null-run RunEnd (#225) validity.
+///
+/// The `RunEnd`/`Sparse` validity fixes are exercised by the weekly Raincloud conformance
+/// corpus only; this test reproduces both shapes on every `verify` run from a JNI-written
+/// (Rust reference) file, so a validity regression reddens the per-PR gate rather than
+/// surfacing a week later.
+///
+/// The bundled `vortex-jni` compressor is version-pinned, so its encoding choice for a crafted
+/// input is deterministic — verified by probing: a mostly-null column with a small fraction of
+/// scattered non-`ALP`-able values compresses to `vortex.sparse` with a null fill scalar, and a
+/// column of distinct value runs interleaved with null runs compresses to `vortex.runend` with
+/// null run-values. Both assertions below therefore hard-check the chosen encoding; a future JNI
+/// bump that changes the choice is a deliberate change that should refresh this fixture.
+class NullSparseRunEndInteropIntegrationTest {
+
+    private static final Session SESSION = Session.create();
+    private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator();
+    private static final int ROWS = 5000;
+
+    static {
+        NativeLoader.loadJni();
+    }
+
+    @Test
+    void jniNullFillSparse_f64_nullRowsDecodeNull(@TempDir Path tmp) throws IOException {
+        // Given — a nullable f64 column that is ~98% null with scattered random (non-ALP-able)
+        // values. This is the world-energy `biofuel_cons_change_pct` shape: the JNI compressor
+        // picks vortex.sparse with a null fill, so every unpatched row must decode as null (#226).
+        Schema schema = new Schema(List.of(
+                Field.nullable("v", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))));
+        Random rng = new Random(226);
+        Double[] expected = new Double[ROWS];
+        Path file = tmp.resolve("null_fill_sparse_f64.vtx");
+        writeJni(file, schema, (root, i) -> {
+            Float8Vector vec = (Float8Vector) root.getVector("v");
+            if (i % 50 == 0) {
+                double value = rng.nextDouble() * 1e9;
+                expected[i] = value;
+                vec.setSafe(i, value);
+            } else {
+                vec.setNull(i);
+            }
+        });
+
+        // When
+        List<Double> result = readF64(file);
+
+        // Then — encoding is null-fill sparse, and every row round-trips value-for-value
+        // (null in ⇒ null out, value in ⇒ value out).
+        assertThat(usedEncodings(file)).contains("vortex.sparse");
+        assertThat(result).containsExactly(expected);
+    }
+
+    @Test
+    void jniNullFillSparse_i64_nullRowsDecodeNull(@TempDir Path tmp) throws IOException {
+        // Given — the nuclear_share_energy shape: a nullable i64 column, ~99% null with a few
+        // scattered random values; the JNI compressor picks vortex.sparse with a null fill.
+        Schema schema = new Schema(List.of(Field.nullable("v", new ArrowType.Int(64, true))));
+        Random rng = new Random(2266);
+        Long[] expected = new Long[ROWS];
+        Path file = tmp.resolve("null_fill_sparse_i64.vtx");
+        writeJni(file, schema, (root, i) -> {
+            BigIntVector vec = (BigIntVector) root.getVector("v");
+            if (i % 97 == 0) {
+                long value = rng.nextLong();
+                expected[i] = value;
+                vec.setSafe(i, value);
+            } else {
+                vec.setNull(i);
+            }
+        });
+
+        // When
+        List<Long> result = readI64(file);
+
+        // Then
+        assertThat(usedEncodings(file)).contains("vortex.sparse");
+        assertThat(result).containsExactly(expected);
+    }
+
+    @Test
+    void jniNullRunRunEnd_i16_nullRowsDecodeNull(@TempDir Path tmp) throws IOException {
+        // Given — the uci-online-retail `customerid` u16? shape: runs of a distinct value per
+        // block interleaved with whole null runs. The JNI compressor picks vortex.runend; before
+        // #225 a null run expanded to the run's filler value instead of null.
+        Schema schema = new Schema(List.of(Field.nullable("v", new ArrowType.Int(16, true))));
+        Short[] expected = new Short[ROWS];
+        Path file = tmp.resolve("null_run_runend_i16.vtx");
+        writeJni(file, schema, (root, i) -> {
+            SmallIntVector vec = (SmallIntVector) root.getVector("v");
+            int block = i / 100;
+            if (block % 4 == 0) {
+                vec.setNull(i);
+            } else {
+                short value = (short) (1000 + block);
+                expected[i] = value;
+                vec.setSafe(i, value);
+            }
+        });
+
+        // When
+        List<Short> result = readI16(file);
+
+        // Then — encoding is run-end, and null runs decode as null (not the run filler).
+        assertThat(usedEncodings(file)).contains("vortex.runend");
+        assertThat(result).containsExactly(expected);
+    }
+
+    @Test
+    void jniNullRunRunEnd_i64_nullRowsDecodeNull(@TempDir Path tmp) throws IOException {
+        // Given — a nullable i64 column of distinct value runs interleaved with null runs; the
+        // JNI compressor picks vortex.runend with nullable run-values (#225).
+        Schema schema = new Schema(List.of(Field.nullable("v", new ArrowType.Int(64, true))));
+        Long[] expected = new Long[ROWS];
+        Path file = tmp.resolve("null_run_runend_i64.vtx");
+        writeJni(file, schema, (root, i) -> {
+            BigIntVector vec = (BigIntVector) root.getVector("v");
+            int block = i / 200;
+            if (block % 3 == 0) {
+                vec.setNull(i);
+            } else {
+                long value = 100_000L + block;
+                expected[i] = value;
+                vec.setSafe(i, value);
+            }
+        });
+
+        // When
+        List<Long> result = readI64(file);
+
+        // Then
+        assertThat(usedEncodings(file)).contains("vortex.runend");
+        assertThat(result).containsExactly(expected);
+    }
+
+    // ── JNI write helper ──────────────────────────────────────────────────────
+
+    private static void writeJni(Path file, Schema schema, ObjIntConsumer<VectorSchemaRoot> fill) throws IOException {
+        String uri = file.toAbsolutePath().toUri().toString();
+        try (VortexWriter writer = VortexWriter.create(SESSION, uri, schema, new HashMap<>(), ALLOCATOR);
+             VectorSchemaRoot root = VectorSchemaRoot.create(schema, ALLOCATOR)) {
+            for (var vec : root.getFieldVectors()) {
+                vec.setInitialCapacity(ROWS);
+                vec.allocateNew();
+            }
+            for (int i = 0; i < ROWS; i++) {
+                fill.accept(root, i);
+            }
+            root.setRowCount(ROWS);
+            try (ArrowArray arr = ArrowArray.allocateNew(ALLOCATOR);
+                 ArrowSchema as = ArrowSchema.allocateNew(ALLOCATOR)) {
+                Data.exportVectorSchemaRoot(ALLOCATOR, root, null, arr, as);
+                writer.writeBatch(arr.memoryAddress(), as.memoryAddress());
+            }
+        }
+    }
+
+    // ── Java read helpers ─────────────────────────────────────────────────────
+
+    private static List<String> usedEncodings(Path file) throws IOException {
+        try (VortexReader reader = VortexReader.open(file, ReadRegistry.loadAll())) {
+            return new ArrayList<>(InspectorTree.build(reader).usedEncodings());
+        }
+    }
+
+    private static List<Double> readF64(Path file) throws IOException {
+        var out = new ArrayList<Double>();
+        forEachValue(file, (masked, inner, i) -> out.add(masked.isValid(i) ? ((DoubleArray) inner).getDouble(i) : null));
+        return out;
+    }
+
+    private static List<Long> readI64(Path file) throws IOException {
+        var out = new ArrayList<Long>();
+        forEachValue(file, (masked, inner, i) -> out.add(masked.isValid(i) ? ((LongArray) inner).getLong(i) : null));
+        return out;
+    }
+
+    private static List<Short> readI16(Path file) throws IOException {
+        var out = new ArrayList<Short>();
+        forEachValue(file, (masked, inner, i) -> out.add(masked.isValid(i) ? ((ShortArray) inner).getShort(i) : null));
+        return out;
+    }
+
+    /// Scans the single-column `file` and hands each row's validity view, the unwrapped inner
+    /// payload [Array], and the row index to `consumer`. A nullable column decodes to a
+    /// [MaskedArray], so this centralizes the unwrap the value readers share.
+    ///
+    /// @param file     the Vortex file to scan
+    /// @param consumer receiver of `(maskedColumn, innerPayload, rowIndex)` per row
+    /// @throws IOException if the file cannot be opened or scanned
+    private static void forEachValue(Path file, RowConsumer consumer) throws IOException {
+        try (VortexReader reader = VortexReader.open(file, ReadRegistry.loadAll());
+             var iter = reader.scan(ScanOptions.all())) {
+            iter.forEachRemaining(chunk -> {
+                MaskedArray masked = chunk.column("v");
+                Array inner = masked.inner();
+                for (long i = 0; i < masked.length(); i++) {
+                    consumer.accept(masked, inner, i);
+                }
+            });
+        }
+    }
+
+    /// Per-row callback used by [#forEachValue(Path, RowConsumer)].
+    @FunctionalInterface
+    private interface RowConsumer {
+        /// Consumes one decoded row.
+        ///
+        /// @param masked the nullable column as a [MaskedArray] for validity checks
+        /// @param inner  the unwrapped non-nullable payload [Array]
+        /// @param index  the row index within the current chunk
+        void accept(MaskedArray masked, Array inner, long index);
+    }
+}