perf: constant-encode all-valid/invalid validity

MaskedEncodingEncoder wrote a full n/8-byte raw bitmap for a
chunk's validity even when every row shared the same validity.
Now detects that case and emits vortex.constant instead, which
the reader already decodes generically (no reader change needed).

Also documents in TODO.md the larger follow-up gap found while
investigating vortex-jni parity: DType.Bool has no RLE/Sparse
encoding at all, which is the dominant remaining cost for
validity bitmaps with mixed (not all-same) nulls.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: dfa1

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Nullable low-cardinality columns now use the shared global dictionary across chunks instead of re-emitting a per-chunk dictionary. The writer previously excluded any nullable column from global-dict candidacy, so a nullable low-cardinality string/numeric column (extremely common in real data — most Raincloud categorical columns have nulls) re-wrote its dictionary in every chunk. Nullable columns now run the same cardinality/ratio gate against their non-null values and emit one shared `vortex.dict` layout with per-chunk masked codes; the reader combines codes-side and pool-side validity per row. On a 200k-row, 10-chunk nullable low-cardinality Utf8 column this shrinks the file 54% (232 KB → 107 KB) and closes the gap to the Rust reference from ~4× to ~1.5×. (internal)
+- `MaskedEncodingEncoder` now encodes an all-valid or all-invalid validity bitmap as `vortex.constant` instead of a raw per-row bitmap. A nullable column chunk with zero nulls previously still paid a full `n/8`-byte bitmap for no information; an all-valid or all-invalid chunk now costs a few bytes regardless of row count. (internal)
 
 ## [0.12.2] — 2026-07-12
 

TODO.md

@@ -12,6 +12,14 @@
 - [ ] Performance tests must be peer-reviewed
 - [ ] Run performance tests on other machines (I have access only to Apple M5)
 - [ ] **Vector API adoption** — see [ADR-0005](adr/0005-vector-api-adoption.md).
+- [ ] **`DType.Bool` has no RLE/Sparse/Constant encoding** — `RunEndEncodingEncoder`,
+  `RleEncodingEncoder`, `SparseEncodingEncoder`, and `ConstantEncodingEncoder` all `accepts()`
+  only `DType.Primitive`; the sole Bool-capable encoder, `BoolEncodingEncoder`, always emits a raw
+  1-bit/row bitmap. This caps how small a nullable column's `vortex.masked` validity child can get:
+  measured at ~75% of the remaining vortex-jni gap (25 KB of 33 KB) on a 200k-row/10-chunk nullable
+  low-cardinality Utf8 benchmark (`FileSizeComparisonIntegrationTest#nullableLowCardinalityUtf8_multiChunk_globalDict_javaVsJni`,
+  commit 5fe8b544). Needs end-to-end Bool support (write + read) for at least one of RLE/Sparse —
+  comparable effort to the nullable-global-dict fix, not a small tweak.
 
 ## Security
 

writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java

@@ -4,6 +4,7 @@
 import io.github.dfa1.vortex.core.error.VortexException;
 
 import io.github.dfa1.vortex.core.model.EncodingId;
+import io.github.dfa1.vortex.core.proto.ProtoScalarValue;
 
 import java.lang.foreign.MemorySegment;
 import java.util.ArrayList;
@@ -43,7 +44,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
         }
         DType nonNullable = dtype.withNullable(false);
         EncodeResult valuesResult = encodeValues(nonNullable, values, ctx);
-        EncodeResult validityResult = new BoolEncodingEncoder().encode(DType.BOOL, validity, ctx);
+        EncodeResult validityResult = encodeValidity(validity, ctx);
 
         int valuesBufCount = valuesResult.buffers().size();
         EncodeNode validityNode = EncodeNode.remapBufferIndices(validityResult.rootNode(), valuesBufCount);
@@ -82,6 +83,35 @@ private static EncodeResult encodeValues(DType nonNullable, Object values, Encod
         return pickInner(nonNullable).encode(nonNullable, values, ctx);
     }
 
+    /// Encodes a masked column's validity bitmap, using `vortex.constant` instead of a raw bitmap
+    /// when every row shares the same validity (the whole chunk is all-valid or all-invalid) — a
+    /// common case that otherwise costs a full `n/8`-byte bitmap for no information.
+    ///
+    /// @param validity per-row validity bitmap
+    /// @param ctx      the encode context
+    /// @return the encoded validity child
+    private static EncodeResult encodeValidity(boolean[] validity, EncodeContext ctx) {
+        if (isConstantValidity(validity)) {
+            boolean value = validity.length == 0 || validity[0];
+            ProtoScalarValue scalar = ProtoScalarValue.ofBoolValue(value);
+            return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, MemorySegment.ofArray(scalar.encode()));
+        }
+        return new BoolEncodingEncoder().encode(DType.BOOL, validity, ctx);
+    }
+
+    private static boolean isConstantValidity(boolean[] validity) {
+        if (validity.length == 0) {
+            return true;
+        }
+        boolean first = validity[0];
+        for (boolean b : validity) {
+            if (b != first) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     /// Returns `values` unchanged, except a `String[]` with null elements is copied with each null
     /// replaced by the empty string so cascade encoders (Dict, FSST) can call `getBytes()` safely.
     ///

writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java

@@ -12,6 +12,7 @@
 import io.github.dfa1.vortex.reader.ReadRegistry;
 import io.github.dfa1.vortex.reader.decode.TestRegistry;
 import io.github.dfa1.vortex.reader.decode.BoolEncodingDecoder;
+import io.github.dfa1.vortex.reader.decode.ConstantEncodingDecoder;
 import io.github.dfa1.vortex.reader.decode.MaskedEncodingDecoder;
 import io.github.dfa1.vortex.reader.decode.PrimitiveEncodingDecoder;
 import org.junit.jupiter.api.Test;
@@ -29,7 +30,10 @@ class MaskedEncodingEncoderTest {
     private static final MaskedEncodingDecoder DECODER = new MaskedEncodingDecoder();
     private static final PrimitiveEncodingEncoder PRIM_ENCODER = new PrimitiveEncodingEncoder();
     private static final BoolEncodingEncoder BOOL_ENCODER = new BoolEncodingEncoder();
+    private static final MaskedEncodingEncoder SUT = new MaskedEncodingEncoder();
     private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(DECODER, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder());
+    private static final ReadRegistry REGISTRY_WITH_CONSTANT = TestRegistry.ofDecoders(
+            DECODER, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder(), new ConstantEncodingDecoder());
 
     private static EncodeResult maskedResult(int[] values, boolean[] validity) {
         DType i32 = DType.I32;
@@ -176,4 +180,62 @@ void threeChildrenThrows() {
                 .isInstanceOf(VortexException.class)
                 .hasMessageContaining("expected 1 or 2 children");
     }
+
+    @Test
+    void allValidColumn_encodesValidityAsConstant() {
+        // Given
+        DType i32Nullable = new DType.Primitive(PType.I32, true);
+        NullableData data = new NullableData(new int[]{1, 2, 3}, new boolean[]{true, true, true});
+
+        // When
+        EncodeResult result = SUT.encode(i32Nullable, data, EncodeTestHelper.testCtx());
+
+        // Then — the validity child is a vortex.constant scalar, not a raw bitmap
+        assertThat(result.rootNode().children()[1].encodingId()).isEqualTo(EncodingId.VORTEX_CONSTANT);
+
+        // And it still round-trips to an all-valid MaskedArray
+        Array decoded = DECODER.decode(DecodeTestHelper.toDecodeContext(result, 3L, i32Nullable, REGISTRY_WITH_CONSTANT));
+        MaskedArray masked = (MaskedArray) decoded;
+        assertThat(masked.isValid(0)).isTrue();
+        assertThat(masked.isValid(1)).isTrue();
+        assertThat(masked.isValid(2)).isTrue();
+    }
+
+    @Test
+    void allInvalidColumn_encodesValidityAsConstant() {
+        // Given
+        DType i32Nullable = new DType.Primitive(PType.I32, true);
+        NullableData data = new NullableData(new int[]{0, 0, 0}, new boolean[]{false, false, false});
+
+        // When
+        EncodeResult result = SUT.encode(i32Nullable, data, EncodeTestHelper.testCtx());
+
+        // Then
+        assertThat(result.rootNode().children()[1].encodingId()).isEqualTo(EncodingId.VORTEX_CONSTANT);
+
+        Array decoded = DECODER.decode(DecodeTestHelper.toDecodeContext(result, 3L, i32Nullable, REGISTRY_WITH_CONSTANT));
+        MaskedArray masked = (MaskedArray) decoded;
+        assertThat(masked.isValid(0)).isFalse();
+        assertThat(masked.isValid(1)).isFalse();
+        assertThat(masked.isValid(2)).isFalse();
+    }
+
+    @Test
+    void mixedValidity_stillEncodesAsRawBitmap() {
+        // Given — a regression guard: mixed validity must not be misdetected as constant
+        DType i32Nullable = new DType.Primitive(PType.I32, true);
+        NullableData data = new NullableData(new int[]{1, 2, 3}, new boolean[]{true, false, true});
+
+        // When
+        EncodeResult result = SUT.encode(i32Nullable, data, EncodeTestHelper.testCtx());
+
+        // Then
+        assertThat(result.rootNode().children()[1].encodingId()).isEqualTo(EncodingId.VORTEX_BOOL);
+
+        Array decoded = DECODER.decode(DecodeTestHelper.toDecodeContext(result, 3L, i32Nullable, REGISTRY));
+        MaskedArray masked = (MaskedArray) decoded;
+        assertThat(masked.isValid(0)).isTrue();
+        assertThat(masked.isValid(1)).isFalse();
+        assertThat(masked.isValid(2)).isTrue();
+    }
 }