From 95e0dfb420f4a6bb1ecf68a0429a9d22757c4525 Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Sun, 12 Jul 2026 22:30:51 +0200
Subject: [PATCH 01/11] perf: FSST iterative symbol training

Replace the fixed 2-byte bigram symbol table with iterative
FSST-paper-style training (longest-match-first parsing, gain-ranked
candidate refinement) producing variable-length 1-8 byte symbols.

Order the final table per the Rust reference's wire contract
(FSSTData::validate_symbol_lengths): multi-byte symbols non-decreasing
by length, then all length-1 symbols, verified against the JNI reader.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../FileSizeComparisonIntegrationTest.java    |  10 +-
 .../JavaWritesRustReadsIntegrationTest.java   |   2 +-
 .../writer/encode/FsstEncodingEncoder.java    | 279 +++++++++++++++---
 .../encode/FsstEncodingEncoderTest.java       |  92 +++++-
 4 files changed, 331 insertions(+), 52 deletions(-)

diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java
index 45c797ff..8ed2088b 100644
--- a/integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java
+++ b/integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java
@@ -620,11 +620,11 @@ void highCardinalityUtf8_javaVsJni(@TempDir Path tmp) throws IOException {
                 (double) javaSize / jniSize,
                 (double) javaSize / rawBytes);
 
-        // Then — Java within 2.5x of JNI. Java's FSST symbol-table builder is still less
-        // aggressive than Rust's on truly random short strings (Linux Rust FSST ≈2.26x
-        // observed); tighten further when FsstEncoding.Encoder gets iterative symbol
-        // training (per the FSST paper).
-        assertThat((double) javaSize / jniSize).isLessThan(2.5);
+        // Then — Java within 2.4x of JNI. On truly random 6-byte strings there is no
+        // repeated substructure for either symbol-table builder to exploit (observed ≈2.26x
+        // with Java's iterative training), so this is close to the escape-scheme floor for
+        // both implementations rather than a training-quality gap.
+        assertThat((double) javaSize / jniSize).isLessThan(2.4);
 
         // Then — Java file is readable and row count matches
         var totalRows = new java.util.concurrent.atomic.AtomicLong();
diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java
index 8b37775a..d3e9bad9 100644
--- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java
+++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java
@@ -569,7 +569,7 @@ void javaWriter_jniReader_utf8Column(@TempDir Path tmp) throws IOException {
 
     @Test
     void javaWriter_jniReader_fsstUtf8Column(@TempDir Path tmp) throws IOException {
-        // Given — FSST encoding: bigram symbol table, escape fallback for unmatched bytes
+        // Given — FSST encoding: trained variable-length symbol table, escape fallback for unmatched bytes
         Path file = tmp.resolve("java_fsst_utf8.vtx");
         String[] data = {"apple", "banana", "cherry", "apricot", "avocado", "almond"};
         try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
index f4d246b9..f18ee631 100644
--- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
+++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
@@ -11,13 +11,44 @@
 import java.lang.foreign.ValueLayout;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /// Write-only encoder for `vortex.fsst`.
+///
+/// Builds a variable-length (1-8 byte) symbol table with iterative refinement, following the
+/// FSST paper (Boncz/Neumann/Winter, "FSST: Fast Random Access String Compression"). Each
+/// training pass compresses a bounded sample of the input with the current table using
+/// longest-match-first greedy parsing, counts the byte savings of extending each matched symbol
+/// by one more byte, then rebuilds the table from the top candidates. The symbol table converges
+/// after a handful of passes, after which the whole input is compressed with the final table.
+///
+/// The wire format packs each symbol's bytes LSB-first into a `long` (first byte in the low byte)
+/// alongside a per-symbol length byte, and reserves code `0xFF` as the single-literal-byte escape.
+/// Real symbol codes are therefore `0..254` (max 255 symbols), each 1-8 bytes long.
 public final class FsstEncodingEncoder implements EncodingEncoder {
 
+    /// Escape opcode: emitted as `0xFF` followed by one literal byte.
+    private static final int ESCAPE = 0xFF;
+
+    /// Maximum number of real symbols (codes `0..254`; `0xFF` is the escape).
     private static final int MAX_SYMBOLS = 255;
-    private static final int BIGRAM_COUNT = 65536;
+
+    /// Maximum symbol length in bytes, bounded by what fits in one `long`.
+    private static final int MAX_SYMBOL_LENGTH = 8;
+
+    /// Number of training passes. The FSST paper reports the table stabilizing after roughly
+    /// five iterations; each pass can only lengthen symbols by one byte, so five passes suffice
+    /// to grow symbols up to length ~6 from an empty start, which captures nearly all of the gain
+    /// on the string workloads seen here. More passes cost linear time for negligible benefit.
+    private static final int TRAINING_ITERATIONS = 5;
+
+    /// Cap on the number of input strings sampled per training pass. Training is O(sample bytes)
+    /// per pass, so very large inputs are sampled rather than fully scanned; the final compression
+    /// pass still runs over every string. 25k strings is a large-enough sample to learn a stable
+    /// table while keeping training cost bounded.
+    private static final int TRAINING_SAMPLE_STRINGS = 25_000;
 
     @Override
     public EncodingId encodingId() {
@@ -39,49 +70,24 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
             byteArrays[i] = strings[i].getBytes(StandardCharsets.UTF_8);
         }
 
-        int[] freq = new int[BIGRAM_COUNT];
-        for (byte[] b : byteArrays) {
-            for (int i = 0; i + 1 < b.length; i++) {
-                freq[(Byte.toUnsignedInt(b[i]) << 8) | Byte.toUnsignedInt(b[i + 1])]++;
-            }
-        }
-        long[] ranked = new long[BIGRAM_COUNT];
-        for (int i = 0; i < BIGRAM_COUNT; i++) {
-            ranked[i] = ((long) freq[i] << 16) | i;
-        }
-        Arrays.sort(ranked);
-
-        int numSymbols = 0;
-        int[] codeForBigram = new int[BIGRAM_COUNT];
-        Arrays.fill(codeForBigram, -1);
-        long[] symbolValues = new long[MAX_SYMBOLS];
-        for (int rank = BIGRAM_COUNT - 1; rank >= 0 && numSymbols < MAX_SYMBOLS; rank--) {
-            int bg = (int) (ranked[rank] & 0xFFFF);
-            if (freq[bg] == 0) {
-                break;
-            }
-            codeForBigram[bg] = numSymbols;
-            int hi = bg >>> 8;
-            int lo = bg & 0xFF;
-            symbolValues[numSymbols] = hi | ((long) lo << 8);
-            numSymbols++;
-        }
+        SymbolTable table = trainSymbolTable(byteArrays);
 
         byte[][] compressed = new byte[n][];
         for (int i = 0; i < n; i++) {
-            compressed[i] = compressString(byteArrays[i], codeForBigram);
+            compressed[i] = table.compress(byteArrays[i]);
         }
 
         Arena arena = ctx.arena();
+        int numSymbols = table.size();
 
         MemorySegment symBuf = arena.allocate(Math.max(numSymbols * 8L, 1), 8);
         for (int i = 0; i < numSymbols; i++) {
-            symBuf.setAtIndex(VortexFormat.LE_LONG, i, symbolValues[i]);
+            symBuf.setAtIndex(VortexFormat.LE_LONG, i, table.packedBytes(i));
         }
 
         MemorySegment symLenBuf = arena.allocate(Math.max(numSymbols, 1));
         for (int i = 0; i < numSymbols; i++) {
-            symLenBuf.set(ValueLayout.JAVA_BYTE, i, (byte) 2);
+            symLenBuf.set(ValueLayout.JAVA_BYTE, i, (byte) table.length(i));
         }
 
         int totalCompressed = 0;
@@ -126,24 +132,207 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
                 null, null);
     }
 
-    private static byte[] compressString(byte[] input, int[] codeForBigram) {
-        byte[] out = new byte[input.length * 2];
-        int outLen = 0;
-        int i = 0;
-        while (i < input.length) {
-            if (i + 1 < input.length) {
-                int bg = (Byte.toUnsignedInt(input[i]) << 8) | Byte.toUnsignedInt(input[i + 1]);
-                int code = codeForBigram[bg];
+    /// Trains a symbol table by iteratively refining candidate symbols against a bounded sample.
+    ///
+    /// @param byteArrays the raw byte content of every input string
+    /// @return the final symbol table (0-255 symbols, each 1-8 bytes)
+    private static SymbolTable trainSymbolTable(byte[][] byteArrays) {
+        int sampleSize = Math.min(byteArrays.length, TRAINING_SAMPLE_STRINGS);
+        SymbolTable table = SymbolTable.empty();
+        for (int iteration = 0; iteration < TRAINING_ITERATIONS; iteration++) {
+            Map<SymbolCandidate, Long> counts = new HashMap<>();
+            for (int s = 0; s < sampleSize; s++) {
+                table.countCandidates(byteArrays[s], counts);
+            }
+            if (counts.isEmpty()) {
+                break;
+            }
+            table = SymbolTable.fromRankedCandidates(counts);
+        }
+        return table;
+    }
+
+    /// A candidate symbol: up to 8 bytes packed LSB-first into a `long` (first byte in the low
+    /// byte, matching the wire format) paired with an explicit length in `1..8`.
+    ///
+    /// @param packedBytes the symbol's bytes, LSB-first (byte `k` at bit position `k * 8`)
+    /// @param length the symbol length in bytes, `1..8`
+    private record SymbolCandidate(long packedBytes, int length) {
+    }
+
+    /// An immutable FSST symbol table plus the machinery to compress against it and to count
+    /// extension candidates during training.
+    private static final class SymbolTable {
+
+        private final long[] packed;
+        private final int[] lengths;
+        private final int count;
+
+        /// Index for O(1) exact-match lookup during greedy parsing and compression, keyed by
+        /// `(packedBytes, length)`, mapping to the symbol code.
+        private final Map<SymbolCandidate, Integer> codeByCandidate;
+
+        private SymbolTable(long[] packed, int[] lengths, int count) {
+            this.packed = packed;
+            this.lengths = lengths;
+            this.count = count;
+            this.codeByCandidate = new HashMap<>(count * 2);
+            for (int code = 0; code < count; code++) {
+                codeByCandidate.put(new SymbolCandidate(packed[code], lengths[code]), code);
+            }
+        }
+
+        static SymbolTable empty() {
+            return new SymbolTable(new long[0], new int[0], 0);
+        }
+
+        /// Builds a table from ranked candidates, keeping the top `MAX_SYMBOLS` by estimated gain.
+        ///
+        /// Gain is `count * length`: the number of input bytes this symbol covers, since every
+        /// symbol code — regardless of length — emits exactly one output byte. This is the FSST
+        /// paper's "apparent gain" and it is what makes length-1 symbols competitive: a frequent
+        /// single byte covers many input bytes at one output byte each and, crucially, replaces a
+        /// 2-byte escape. Dropping a frequent single byte from the table forces escapes (2 output
+        /// bytes per occurrence), which is the failure mode of a naive bigram-only table. Ties
+        /// break toward longer symbols (strictly more valuable per code).
+        static SymbolTable fromRankedCandidates(Map<SymbolCandidate, Long> counts) {
+            record Ranked(SymbolCandidate candidate, long gain) {
+            }
+            Ranked[] ranked = new Ranked[counts.size()];
+            int idx = 0;
+            for (Map.Entry<SymbolCandidate, Long> e : counts.entrySet()) {
+                long gain = e.getValue() * (long) e.getKey().length();
+                ranked[idx++] = new Ranked(e.getKey(), gain);
+            }
+            Arrays.sort(ranked, (a, b) -> {
+                int byGain = Long.compare(b.gain(), a.gain());
+                if (byGain != 0) {
+                    return byGain;
+                }
+                return Integer.compare(b.candidate().length(), a.candidate().length());
+            });
+
+            int keep = Math.min(ranked.length, MAX_SYMBOLS);
+            SymbolCandidate[] kept = new SymbolCandidate[keep];
+            for (int i = 0; i < keep; i++) {
+                kept[i] = ranked[i].candidate();
+            }
+
+            // Wire order (Rust `FSSTData::validate_symbol_lengths`): multi-byte symbols
+            // (length 2-8) first in non-decreasing length order, then all length-1 symbols.
+            // Selection above picks candidates by gain; this final sort only reorders codes
+            // to satisfy that wire contract, so ties within a length bucket are arbitrary.
+            Arrays.sort(kept, (a, b) -> {
+                boolean aSingle = a.length() == 1;
+                boolean bSingle = b.length() == 1;
+                if (aSingle != bSingle) {
+                    return aSingle ? 1 : -1;
+                }
+                return Integer.compare(a.length(), b.length());
+            });
+
+            long[] packed = new long[keep];
+            int[] lengths = new int[keep];
+            for (int i = 0; i < keep; i++) {
+                packed[i] = kept[i].packedBytes();
+                lengths[i] = kept[i].length();
+            }
+            return new SymbolTable(packed, lengths, keep);
+        }
+
+        int size() {
+            return count;
+        }
+
+        long packedBytes(int code) {
+            return packed[code];
+        }
+
+        int length(int code) {
+            return lengths[code];
+        }
+
+        /// Returns the code of the longest symbol matching `input` at `pos`, or -1 if
+        /// none matches. Checks lengths `MAX_SYMBOL_LENGTH` down to 1.
+        private int longestMatch(byte[] input, int pos) {
+            int maxLen = Math.min(MAX_SYMBOL_LENGTH, input.length - pos);
+            for (int len = maxLen; len >= 1; len--) {
+                Integer code = codeByCandidate.get(new SymbolCandidate(pack(input, pos, len), len));
+                if (code != null) {
+                    return code;
+                }
+            }
+            return -1;
+        }
+
+        /// Walks `input` with longest-match-first greedy parsing using the current table,
+        /// accumulating training candidates into `counts`:
+        /// - each single byte that starts a match (or an unmatched byte) as a length-1 candidate,
+        ///   so the table can bootstrap from empty and keep its seed symbols;
+        /// - the matched multi-byte symbol itself, so it survives the next rebuild;
+        /// - the concatenation of the current match with the next match, capped at 8 bytes. This is
+        ///   the FSST paper's core heuristic: pairing adjacent symbols lets symbols roughly double
+        ///   in length each pass (not just grow by one byte), so length-8 symbols emerge in a few
+        ///   passes rather than seven.
+        void countCandidates(byte[] input, Map<SymbolCandidate, Long> counts) {
+            int i = 0;
+            while (i < input.length) {
+                int code = longestMatch(input, i);
+                int matchLen = code >= 0 ? lengths[code] : 1;
+
+                // Always offer the length-1 symbol at this position (seeds/keeps single bytes).
+                bump(counts, new SymbolCandidate(pack(input, i, 1), 1));
+
+                if (matchLen > 1) {
+                    // Reinforce the matched multi-byte symbol so it survives the next rebuild.
+                    bump(counts, new SymbolCandidate(pack(input, i, matchLen), matchLen));
+                }
+
+                // Pair the current match with the following match to form a longer candidate.
+                int next = i + matchLen;
+                if (next < input.length && matchLen < MAX_SYMBOL_LENGTH) {
+                    int nextCode = longestMatch(input, next);
+                    int nextLen = nextCode >= 0 ? lengths[nextCode] : 1;
+                    int pairLen = Math.min(matchLen + nextLen, MAX_SYMBOL_LENGTH);
+                    bump(counts, new SymbolCandidate(pack(input, i, pairLen), pairLen));
+                }
+
+                i += matchLen;
+            }
+        }
+
+        /// Compresses `input` with longest-match-first greedy parsing over the final table.
+        /// Positions matching no symbol are emitted as an escape (`0xFF` + literal byte).
+        byte[] compress(byte[] input) {
+            byte[] out = new byte[input.length * 2];
+            int outLen = 0;
+            int i = 0;
+            while (i < input.length) {
+                int code = longestMatch(input, i);
                 if (code >= 0) {
                     out[outLen++] = (byte) code;
-                    i += 2;
-                    continue;
+                    i += lengths[code];
+                } else {
+                    out[outLen++] = (byte) ESCAPE;
+                    out[outLen++] = input[i];
+                    i++;
                 }
             }
-            out[outLen++] = (byte) 0xFF;
-            out[outLen++] = input[i];
-            i++;
+            return Arrays.copyOf(out, outLen);
+        }
+
+        private static void bump(Map<SymbolCandidate, Long> counts, SymbolCandidate candidate) {
+            counts.merge(candidate, 1L, Long::sum);
+        }
+
+        /// Packs `len` bytes of `input` starting at `pos` LSB-first into a
+        /// `long`: byte 0 in the low byte, matching the wire and decoder conventions.
+        private static long pack(byte[] input, int pos, int len) {
+            long value = 0;
+            for (int k = 0; k < len; k++) {
+                value |= Byte.toUnsignedLong(input[pos + k]) << (k * 8);
+            }
+            return value;
         }
-        return Arrays.copyOf(out, outLen);
     }
 }
diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java
index 0a17ed30..1c584633 100644
--- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java
+++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java
@@ -57,10 +57,50 @@ static Stream<Arguments> stringArrays() {
                     Arguments.of("repeated-short", repeat("a", 1)),
                     Arguments.of("repeated-short", repeat("ab", 50)),
                     Arguments.of("all-empty", new String[]{"", "", "", ""}),
-                    Arguments.of("unicode", new String[]{"héllo", "wörld", "こんにちは"})
+                    Arguments.of("unicode", new String[]{"héllo", "wörld", "こんにちは"}),
+                    // Long repeated English text: iterative training should learn 3+ byte symbols
+                    // (e.g. "the ", "brown"), exercising the multi-byte match path end to end.
+                    Arguments.of("repeated-sentence",
+                            repeat("the quick brown fox jumps over the lazy dog", 40)),
+                    // Common English words repeated: many overlapping multi-byte symbols.
+                    Arguments.of("common-words", repeatCycle(
+                            new String[]{"hello", "world", "the", "quick", "brown", "hello", "world"}, 60)),
+                    // Strings all shorter than the 8-byte symbol cap.
+                    Arguments.of("sub-8-byte", new String[]{"a", "ab", "abc", "abcd", "abcde", "abcdef", "abcdefg"}),
+                    // Truly incompressible: single distinct-byte-heavy string. Must round-trip and
+                    // not blow up beyond ~2x raw (the escape-scheme floor).
+                    Arguments.of("incompressible", new String[]{distinctBytes()}),
+                    // Non-ASCII multi-byte UTF-8, repeated so symbols can span codepoint boundaries.
+                    Arguments.of("utf8-repeated", repeat("café ☕ münchen 日本語テスト", 30)),
+                    // Exactly-8-byte symbol repeated: confirms the length-8 boundary encodes/decodes.
+                    Arguments.of("exact-8-byte", repeat("ABCDEFGH", 50))
             );
         }
 
+        private static String[] repeatCycle(String[] cycle, int times) {
+            String[] arr = new String[cycle.length * times];
+            for (int t = 0; t < times; t++) {
+                System.arraycopy(cycle, 0, arr, t * cycle.length, cycle.length);
+            }
+            return arr;
+        }
+
+        private static String distinctBytes() {
+            // 0x00..0xFF is not valid UTF-8; use the full printable-ASCII + Latin-1 letter range,
+            // each byte distinct so no bigram repeats — the adversarial case for symbol training.
+            StringBuilder sb = new StringBuilder();
+            for (char c = 'a'; c <= 'z'; c++) {
+                sb.append(c);
+            }
+            for (char c = 'A'; c <= 'Z'; c++) {
+                sb.append(c);
+            }
+            for (char c = '0'; c <= '9'; c++) {
+                sb.append(c);
+            }
+            return sb.toString();
+        }
+
         private static String[] repeat(String s, int n) {
             String[] arr = new String[n];
             java.util.Arrays.fill(arr, s);
@@ -116,6 +156,56 @@ void encode_thenDecode_roundtripsAllStrings(String name, String[] values) {
                 assertThat(decoded.getString(i)).as("index %d", i).isEqualTo(values[i]);
             }
         }
+
+        @Test
+        void encode_repeatedText_learnsMultiByteSymbols_compressesBelowRaw() {
+            // Given — highly repetitive text where iterative training should build long symbols.
+            String[] values = repeat("the quick brown fox jumps over the lazy dog", 100);
+            long rawBytes = 0;
+            for (String v : values) {
+                rawBytes += v.getBytes(StandardCharsets.UTF_8).length;
+            }
+
+            // When
+            EncodeResult result = ENCODER.encode(DTypes.UTF8, values, EncodeTestHelper.testCtx());
+            long compressedBytes = result.buffers().toArray(MemorySegment[]::new)[2].byteSize();
+            byte maxSymLen = maxSymbolLength(result);
+
+            // Then — real compression (< half raw) proves multi-byte symbols are in play, and the
+            // table contains symbols longer than the old fixed 2-byte limit.
+            assertThat(compressedBytes).isLessThan(rawBytes / 2);
+            assertThat(maxSymLen).isGreaterThan((byte) 2);
+        }
+
+        @Test
+        void encode_incompressible_staysUnderTwiceRaw() {
+            // Given — all-distinct bytes, the adversarial case: no repeated substrings to learn.
+            StringBuilder sb = new StringBuilder();
+            for (char c = 'a'; c <= 'z'; c++) {
+                sb.append(c);
+            }
+            String[] values = {sb.toString()};
+            long rawBytes = sb.toString().getBytes(StandardCharsets.UTF_8).length;
+
+            // When
+            EncodeResult result = ENCODER.encode(DTypes.UTF8, values, EncodeTestHelper.testCtx());
+            long compressedBytes = result.buffers().toArray(MemorySegment[]::new)[2].byteSize();
+
+            // Then — must not regress past the escape-scheme floor of ~2x raw.
+            assertThat(compressedBytes).isLessThanOrEqualTo(rawBytes * 2);
+        }
+
+        private static byte maxSymbolLength(EncodeResult result) {
+            MemorySegment symLenBuf = result.buffers().toArray(MemorySegment[]::new)[1];
+            byte max = 0;
+            for (long i = 0; i < symLenBuf.byteSize(); i++) {
+                byte len = symLenBuf.get(ValueLayout.JAVA_BYTE, i);
+                if (len > max) {
+                    max = len;
+                }
+            }
+            return max;
+        }
     }
 
     @Nested

From 4a3fa0a957aa80975fbfb0a29f915d3affa43b5b Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Sun, 12 Jul 2026 22:44:09 +0200
Subject: [PATCH 02/11] perf: non-boxing FSST match index + stratified sampling

Replace the per-symbol-table HashMap<SymbolCandidate, Integer> match
index with a flat open-addressing table over primitive arrays.
longestMatch() runs up to 8 probes per byte position across every row
in the dataset; boxing a record key per probe was measurable
allocation pressure in the hottest loop this encoder has.
~18-23% faster on incompressible data (worst case: many failed
probes), ~10% on realistic repetitive text.

Also replace first-N training sampling with the same stratified,
seeded-PRNG scheme CascadingCompressor already uses, so training
doesn't silently bias toward whatever sorts first in the input.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../writer/encode/FsstEncodingEncoder.java    | 112 ++++++++++++++++--
 1 file changed, 103 insertions(+), 9 deletions(-)

diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
index f18ee631..8bfc2416 100644
--- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
+++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
@@ -14,6 +14,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 
 /// Write-only encoder for `vortex.fsst`.
 ///
@@ -50,6 +51,15 @@ public final class FsstEncodingEncoder implements EncodingEncoder {
     /// table while keeping training cost bounded.
     private static final int TRAINING_SAMPLE_STRINGS = 25_000;
 
+    /// Number of contiguous strides the training sample is drawn from, mirroring
+    /// [CascadingCompressor]'s stratified sampling. One random contiguous chunk of rows is
+    /// taken from each stride so the sample covers the whole input rather than only its prefix.
+    private static final int TRAINING_STRIDE_COUNT = 32;
+
+    /// Fixed seed for the training-sample PRNG. Encoding must be reproducible: the same input
+    /// always trains the same symbol table and produces byte-identical output.
+    private static final long TRAINING_SAMPLE_SEED = 0x5EEDL;
+
     @Override
     public EncodingId encodingId() {
         return EncodingId.VORTEX_FSST;
@@ -138,11 +148,12 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
     /// @return the final symbol table (0-255 symbols, each 1-8 bytes)
     private static SymbolTable trainSymbolTable(byte[][] byteArrays) {
         int sampleSize = Math.min(byteArrays.length, TRAINING_SAMPLE_STRINGS);
+        int[] sampleIndices = sampleRowIndices(byteArrays.length, sampleSize);
         SymbolTable table = SymbolTable.empty();
         for (int iteration = 0; iteration < TRAINING_ITERATIONS; iteration++) {
             Map<SymbolCandidate, Long> counts = new HashMap<>();
-            for (int s = 0; s < sampleSize; s++) {
-                table.countCandidates(byteArrays[s], counts);
+            for (int idx : sampleIndices) {
+                table.countCandidates(byteArrays[idx], counts);
             }
             if (counts.isEmpty()) {
                 break;
@@ -152,6 +163,45 @@ private static SymbolTable trainSymbolTable(byte[][] byteArrays) {
         return table;
     }
 
+    /// Stratified sample of row indices: partitions `[0, n)` into `TRAINING_STRIDE_COUNT`
+    /// contiguous strides and draws one contiguous sub-range from each, the same scheme as
+    /// `CascadingCompressor`'s stratified sampling. Sampling only the first `sampleSize` rows
+    /// would silently bias the learned table on inputs sorted or clustered by value; this covers
+    /// the whole input while keeping each drawn chunk contiguous.
+    ///
+    /// @param n total row count
+    /// @param sampleSize number of rows to draw, `<= n`
+    /// @return `sampleSize` row indices into `[0, n)`
+    @SuppressWarnings("java:S2245") // Deterministic PRNG is the contract: training samples must
+                                    // be reproducible across builds. No security boundary.
+    private static int[] sampleRowIndices(int n, int sampleSize) {
+        int[] indices = new int[sampleSize];
+        if (sampleSize == 0) {
+            return indices;
+        }
+        int strideCount = Math.min(TRAINING_STRIDE_COUNT, sampleSize);
+        Random rng = new Random(TRAINING_SAMPLE_SEED);
+        int dstOff = 0;
+        int partRemainder = n % strideCount;
+        int partShortStep = n / strideCount;
+        int partLongStep = partShortStep + 1;
+        int sampleRemainder = sampleSize % strideCount;
+        int sampleShortStep = sampleSize / strideCount;
+        int sampleLongStep = sampleShortStep + 1;
+        for (int s = 0; s < strideCount; s++) {
+            int partStart = s * partShortStep + Math.min(s, partRemainder);
+            int partLen = s < partRemainder ? partLongStep : partShortStep;
+            int sampleLen = s < sampleRemainder ? sampleLongStep : sampleShortStep;
+            int maxStart = Math.max(0, partLen - sampleLen);
+            int offsetInPart = maxStart == 0 ? 0 : rng.nextInt(maxStart + 1);
+            int srcOff = partStart + offsetInPart;
+            for (int k = 0; k < sampleLen; k++) {
+                indices[dstOff++] = srcOff + k;
+            }
+        }
+        return indices;
+    }
+
     /// A candidate symbol: up to 8 bytes packed LSB-first into a `long` (first byte in the low
     /// byte, matching the wire format) paired with an explicit length in `1..8`.
     ///
@@ -164,24 +214,68 @@ private record SymbolCandidate(long packedBytes, int length) {
     /// extension candidates during training.
     private static final class SymbolTable {
 
+        /// Smallest match-index capacity; also the capacity used for the empty table.
+        private static final int MIN_INDEX_CAPACITY = 4;
+
         private final long[] packed;
         private final int[] lengths;
         private final int count;
 
-        /// Index for O(1) exact-match lookup during greedy parsing and compression, keyed by
-        /// `(packedBytes, length)`, mapping to the symbol code.
-        private final Map<SymbolCandidate, Integer> codeByCandidate;
+        /// Open-addressing index for exact-match lookup during greedy parsing and compression,
+        /// keyed by `(packedBytes, length)`, mapping to the symbol code. Plain primitive arrays
+        /// rather than `Map<SymbolCandidate, Integer>`: `longestMatch` below runs up to
+        /// `MAX_SYMBOL_LENGTH` probes per byte position, for every position of every row in the
+        /// whole dataset, so a boxed record key per probe is real allocation pressure in the
+        /// hottest loop this encoder has.
+        private final long[] indexKey;
+        private final byte[] indexLen; // 0 = empty slot; valid symbol lengths are 1..8
+        private final int[] indexCode;
+        private final int indexMask;
 
         private SymbolTable(long[] packed, int[] lengths, int count) {
             this.packed = packed;
             this.lengths = lengths;
             this.count = count;
-            this.codeByCandidate = new HashMap<>(count * 2);
+
+            int capacity = MIN_INDEX_CAPACITY;
+            while (capacity < count * 4) {
+                capacity <<= 1;
+            }
+            this.indexMask = capacity - 1;
+            this.indexKey = new long[capacity];
+            this.indexLen = new byte[capacity];
+            this.indexCode = new int[capacity];
             for (int code = 0; code < count; code++) {
-                codeByCandidate.put(new SymbolCandidate(packed[code], lengths[code]), code);
+                indexInsert(packed[code], lengths[code], code);
             }
         }
 
+        private void indexInsert(long key, int len, int code) {
+            int idx = indexHash(key, len) & indexMask;
+            while (indexLen[idx] != 0) {
+                idx = (idx + 1) & indexMask;
+            }
+            indexKey[idx] = key;
+            indexLen[idx] = (byte) len;
+            indexCode[idx] = code;
+        }
+
+        private int indexLookup(long key, int len) {
+            int idx = indexHash(key, len) & indexMask;
+            while (indexLen[idx] != 0) {
+                if (indexLen[idx] == len && indexKey[idx] == key) {
+                    return indexCode[idx];
+                }
+                idx = (idx + 1) & indexMask;
+            }
+            return -1;
+        }
+
+        private static int indexHash(long key, int len) {
+            long h = (key ^ ((long) len * 0x9E3779B97F4A7C15L)) * 0xC2B2AE3D27D4EB4FL;
+            return (int) (h ^ (h >>> 32));
+        }
+
         static SymbolTable empty() {
             return new SymbolTable(new long[0], new int[0], 0);
         }
@@ -257,8 +351,8 @@ int length(int code) {
         private int longestMatch(byte[] input, int pos) {
             int maxLen = Math.min(MAX_SYMBOL_LENGTH, input.length - pos);
             for (int len = maxLen; len >= 1; len--) {
-                Integer code = codeByCandidate.get(new SymbolCandidate(pack(input, pos, len), len));
-                if (code != null) {
+                int code = indexLookup(pack(input, pos, len), len);
+                if (code >= 0) {
                     return code;
                 }
             }

From 1bbc3549e9e2083abee61f716c7fc44674ff024e Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Sun, 12 Jul 2026 23:21:48 +0200
Subject: [PATCH 03/11] fix: Utf8 cost-based encoding dispatch, narrow FSST
 ptypes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Root cause of the Java/JNI file-size gap on high-cardinality string
columns: CascadingCompressor routed Utf8/Binary through first-match
dispatch (findPrimitiveEncoding), not the sample-and-measure
competition Primitive dtypes get. Dict is registered before FSST, so
Dict always won regardless of actual size — confirmed by inspecting a
50k-distinct-string file and finding vortex.dict where FSST would be
~40% smaller. Both DictEncodingEncoder's own comment ("Utf8 still uses
sample-encoded selection") and VortexWriter's DEFAULT_CODECS comment
already documented this as the intended behavior; it just wasn't wired
up.

Fix: extract the shared skip/always-use sweep + stratified-sample
measurement loop into competeAndEncode(), reuse it for both Primitive
(unchanged, using primitiveBytes as baseline) and the new Utf8 path
(no analytic baseline available, so the loop just keeps the smallest
measured candidate — VarBinEncodingEncoder unconditionally accepts
Utf8 so a winner always exists). stratifiedSample/dataLength gain a
String[] case to support sampling string columns.

Also narrow FsstEncodingEncoder's uncompressed_lengths/codes_offsets
buffers to the smallest ptype that fits (was always hardcoded I32),
using the metadata fields the wire format already carries for this.

This surfaced a latent bug: proto3 omits fields at their default
(zero/U8) value, so an all-U8 FSST metadata message encodes to zero
bytes and the writer skips the metadata segment entirely. Absent
metadata is a valid encoding of that default, not corruption —
FsstEncodingDecoder's unconditional null-check threw on legitimate
files. Fixed to treat null metadata as defaults, and added proper
buffer/child-count bounds checks (previously null-metadata was
incidentally catching a separate malformed-input case that needed its
own explicit guard).

Net effect on highCardinalityUtf8_javaVsJni (50k distinct 6-byte
strings): Java/JNI 2.26x -> 1.36x (801,908 -> 483,336 bytes).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 docs/compatibility.md                         |  2 +-
 .../FileSizeComparisonIntegrationTest.java    | 10 ++--
 .../reader/decode/FsstEncodingDecoder.java    | 21 +++++--
 .../dfa1/vortex/writer/VortexWriter.java      | 10 ++--
 .../writer/encode/CascadingCompressor.java    | 57 ++++++++++++++++---
 .../writer/encode/FsstEncodingEncoder.java    | 57 ++++++++++++++++---
 .../encode/FsstEncodingEncoderTest.java       | 43 ++++++++++++--
 7 files changed, 163 insertions(+), 37 deletions(-)

diff --git a/docs/compatibility.md b/docs/compatibility.md
index 06ce0de7..dcea9742 100644
--- a/docs/compatibility.md
+++ b/docs/compatibility.md
@@ -101,7 +101,7 @@ integer width is wire-legal, mirroring `VarBinArray`), and `ScanIterator` could
 | `vortex.listview`           | `ListViewEncodingDecoder`        | `ListViewEncodingEncoder`        | ✅      | ✅      |                                                                       |
 | `vortex.fixed_size_list`    | `FixedSizeListEncodingDecoder`   | `FixedSizeListEncodingEncoder`   | ✅      | ✅      |                                                                       |
 | `vortex.zstd`               | `ZstdEncodingDecoder`            | `ZstdEncodingEncoder`            | ✅      | ✅      | Primitive, Utf8, Binary                                               |
-| `vortex.masked`             | `MaskedEncodingDecoder`          | `MaskedEncodingEncoder`          | ✅      | ✅      | NullableData carrier; inner values cascade (Dict/FSST/ALP/…) when cascading is enabled, else first-match Primitive / VarBin / FixedSizeList |
+| `vortex.masked`             | `MaskedEncodingDecoder`          | `MaskedEncodingEncoder`          | ✅      | ✅      | NullableData carrier; inner values cascade (Dict/FSST/ALP/…) when cascading is enabled; Utf8/Primitive still compete on measured size even at depth 0, else first-match FixedSizeList |
 | `vortex.decimal`            | `DecimalEncodingDecoder`         | `DecimalEncodingEncoder`         | ✅      | ✅      |                                                                       |
 | `vortex.decimal_byte_parts` | `DecimalBytePartsEncodingDecoder`| `DecimalBytePartsEncodingEncoder`| ✅      | ✅      |                                                                       |
 | `vortex.datetimeparts`      | `DateTimePartsEncodingDecoder`   | `DateTimePartsEncodingEncoder`   | ✅      | ✅      |                                                                       |
diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java
index 8ed2088b..5166aea7 100644
--- a/integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java
+++ b/integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java
@@ -620,11 +620,11 @@ void highCardinalityUtf8_javaVsJni(@TempDir Path tmp) throws IOException {
                 (double) javaSize / jniSize,
                 (double) javaSize / rawBytes);
 
-        // Then — Java within 2.4x of JNI. On truly random 6-byte strings there is no
-        // repeated substructure for either symbol-table builder to exploit (observed ≈2.26x
-        // with Java's iterative training), so this is close to the escape-scheme floor for
-        // both implementations rather than a training-quality gap.
-        assertThat((double) javaSize / jniSize).isLessThan(2.4);
+        // Then — Java within 1.5x of JNI (observed ≈1.36x). CascadingCompressor now runs Utf8
+        // through the same sample-and-measure competition as Primitive dtypes instead of
+        // first-match dispatch, so FSST — not Dict — wins on this high-cardinality column
+        // (Dict's table for ~50k distinct 6-byte values used to dominate the file size).
+        assertThat((double) javaSize / jniSize).isLessThan(1.5);
 
         // Then — Java file is readable and row count matches
         var totalRows = new java.util.concurrent.atomic.AtomicLong();
diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java
index 04f226c5..343f9acb 100644
--- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java
+++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java
@@ -8,6 +8,7 @@
 import io.github.dfa1.vortex.core.model.EncodingId;
 import io.github.dfa1.vortex.core.io.VortexFormat;
 import io.github.dfa1.vortex.core.proto.ProtoFSSTMetadata;
+import io.github.dfa1.vortex.core.proto.ProtoPType;
 
 import java.io.IOException;
 import java.lang.foreign.MemorySegment;
@@ -25,13 +26,25 @@ public EncodingId encodingId() {
 
     @Override
     public Array decode(DecodeContext ctx) {
-        MemorySegment rawMeta = ctx.metadata();
-        if (rawMeta == null) {
-            throw new VortexException(EncodingId.VORTEX_FSST, "missing metadata");
+        int numBufs = ctx.node().bufferIndices().length;
+        if (numBufs != 3) {
+            throw new VortexException(EncodingId.VORTEX_FSST, "expected 3 buffers, got " + numBufs);
+        }
+        int numChildren = ctx.node().children().length;
+        if (numChildren < 2) {
+            throw new VortexException(EncodingId.VORTEX_FSST, "expected at least 2 children, got " + numChildren);
         }
+
+        // Proto3 omits fields at their default (zero) value on the wire, so an all-U8 metadata
+        // message (both ptypes ordinal 0) encodes to zero bytes and the writer skips the
+        // metadata segment entirely — absent metadata is a valid encoding of that default, not
+        // a malformed file.
+        MemorySegment rawMeta = ctx.metadata();
         ProtoFSSTMetadata meta;
         try {
-            meta = ProtoFSSTMetadata.decode(rawMeta, 0, rawMeta.byteSize());
+            meta = rawMeta == null
+                    ? new ProtoFSSTMetadata(ProtoPType.U8, ProtoPType.U8)
+                    : ProtoFSSTMetadata.decode(rawMeta, 0, rawMeta.byteSize());
         } catch (IOException e) {
             throw new VortexException(EncodingId.VORTEX_FSST, "invalid metadata", e);
         }
diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java
index 2cffb591..289f926a 100644
--- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java
+++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java
@@ -197,11 +197,11 @@ private static List<EncodingEncoder> buildCascadeCodecs(WriteOptions options) {
         codecs.add(new SparseEncodingEncoder());
         codecs.add(new DictEncodingEncoder());
         codecs.add(new BitpackedEncodingEncoder());
-        // FsstEncodingEncoder sits between Dict and VarBin. Today's non-primitive dispatch
-        // (CascadingCompressor.findPrimitiveEncoding) is first-match, so Dict still
-        // wins for Utf8; FSST only fires when Dict is excluded (cascade nested re-runs
-        // via spliceResult's notApplicable retry). Listing it here matches Rust which
-        // uses FSST for high-cardinality short strings (e.g. taxi store_and_fwd_flag).
+        // FsstEncodingEncoder sits between Dict and VarBin. Utf8 goes through
+        // CascadingCompressor's sample-and-measure competition (like Primitive dtypes),
+        // not first-match dispatch, so Dict/FSST/VarBin genuinely compete on measured
+        // size — matching Rust, which uses FSST for high-cardinality short strings
+        // (e.g. taxi store_and_fwd_flag).
         codecs.add(new FsstEncodingEncoder());
         codecs.add(new VarBinEncodingEncoder());
         if (options.enableZstd()) {
diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java
index 76ce6bce..db02217b 100644
--- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java
+++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java
@@ -8,6 +8,7 @@
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
+import java.util.function.IntToLongFunction;
 
 /// Cascading compressor: evaluates multiple encodings on a sample and picks the one
 /// producing the smallest output. With `allowedCascading > 0`, also recurses
@@ -36,6 +37,7 @@ private static int dataLength(Object data) {
             case long[] a -> a.length;
             case float[] a -> a.length;
             case double[] a -> a.length;
+            case String[] a -> a.length;
             default -> throw new IllegalArgumentException("unsupported data type: " + data.getClass());
         };
     }
@@ -89,6 +91,12 @@ case StructData(var fieldArrays) -> {
                         System.arraycopy(a, srcOff, out, dstOff, len));
                 yield out;
             }
+            case String[] a -> {
+                String[] out = new String[sampleSize];
+                forEachStride(a.length, sampleSize, seed, (srcOff, dstOff, len) ->
+                        System.arraycopy(a, srcOff, out, dstOff, len));
+                yield out;
+            }
             default -> throw new IllegalArgumentException("unsupported data type: " + data.getClass());
         };
     }
@@ -151,12 +159,25 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx)
         if (dtype instanceof DType.Struct structDtype) {
             return encodeStruct(structDtype, (StructData) data, ctx);
         }
-        // Non-primitives (extension types): find the accepting encoding and splice
-        // through it so its cascaded children (e.g. datetimeparts → days/seconds/subseconds)
-        // are recursively compressed rather than stored as raw primitives. Honor the
-        // excluded set so spliceResult's notApplicable retry can rotate to the next
-        // accepting encoding (e.g. DateTimePartsEncoding → ExtEncoding when the input
-        // is raw storage rather than DateTimePartsData).
+
+        // Utf8: same sample-and-measure competition as Primitive below (Dict, FSST, VarBin,
+        // Zstd all genuinely compete on measured size) rather than the extension-type
+        // first-match dispatch. No stats are computed — DictEncodingEncoder.expectedRatio()
+        // already defers to this path for Utf8 rather than consuming them — and there is no
+        // cheap analytic "no compression" baseline the way primitiveBytes is for fixed-width
+        // types, so the competition simply keeps whichever accepting encoder measures
+        // smallest (VarBinEncodingEncoder unconditionally accepts Utf8, so a winner always
+        // exists in practice).
+        if (dtype instanceof DType.Utf8) {
+            return competeAndEncode(dtype, data, ctx, ArrayStats.EMPTY, sampleSize -> Long.MAX_VALUE);
+        }
+
+        // Remaining non-primitives (extension types, Binary, List, ...): find the accepting
+        // encoding and splice through it so its cascaded children (e.g. datetimeparts →
+        // days/seconds/subseconds) are recursively compressed rather than stored as raw
+        // primitives. Honor the excluded set so spliceResult's notApplicable retry can rotate
+        // to the next accepting encoding (e.g. DateTimePartsEncoding → ExtEncoding when the
+        // input is raw storage rather than DateTimePartsData).
         if (!(dtype instanceof DType.Primitive p)) {
             return spliceResult(findPrimitiveEncoding(dtype, ctx.excluded()), dtype, data, ctx);
         }
@@ -175,6 +196,24 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx)
                                    ? ArrayStats.EMPTY
                                    : ArrayStats.compute(p.ptype(), data, merged);
 
+        return competeAndEncode(dtype, data, ctx, stats, sampleSize -> primitiveBytes(dtype, sampleSize));
+    }
+
+    /// Shared sample-and-measure competition: stats-based skip/always-use sweep, then a
+    /// stratified-sample cost measurement across every remaining accepting encoder, picking
+    /// whichever measures smallest against `baselineFn`'s reference size.
+    ///
+    /// @param dtype      the logical type of the data to encode
+    /// @param data       the full input data
+    /// @param ctx        encoding context supplying the arena, encoder map, and cascade parameters
+    /// @param stats      pre-computed stats for the [EncodingEncoder#expectedRatio] sweep,
+    ///                   or [ArrayStats#EMPTY] when the dtype has no stats support
+    /// @param baselineFn given the sample size, returns the reference size a candidate must
+    ///                   beat to win
+    /// @return the [EncodeResult] produced by the winning encoding
+    private EncodeResult competeAndEncode(
+            DType dtype, Object data, EncodeContext ctx, ArrayStats stats, IntToLongFunction baselineFn
+    ) {
         // First sweep: stats verdicts. ALWAYS_USE short-circuits; SKIP excludes from
         // the sample-encoded competition below; COMPLETE defers to it.
         boolean[] skipMask = new boolean[encodings.size()];
@@ -200,7 +239,7 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx)
         sampleSize = Math.min(sampleSize, n);
         Object sample = (sampleSize < n) ? stratifiedSample(data, sampleSize, ctx.sampleSeed()) : data;
 
-        long bestSampleSize = primitiveBytes(dtype, sampleSize);
+        long bestSampleSize = baselineFn.applyAsLong(sampleSize);
         EncodingEncoder winner = null;
 
         for (int i = 0; i < encodings.size(); i++) {
@@ -223,8 +262,8 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx)
         }
 
         if (winner == null) {
-            // No encoding beats primitive — fall back
-            return findPrimitiveEncoding(dtype, ctx.excluded()).encode(dtype, data, ctx);
+            // No encoding beats the baseline — fall back to the first accepting encoder
+            return spliceResult(findPrimitiveEncoding(dtype, ctx.excluded()), dtype, data, ctx);
         }
 
         // Re-run winner on full data
diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
index 8bfc2416..f0a2ddfd 100644
--- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
+++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java
@@ -3,6 +3,7 @@
 import io.github.dfa1.vortex.core.model.DType;
 import io.github.dfa1.vortex.core.model.PType;
 import io.github.dfa1.vortex.core.model.EncodingId;
+import io.github.dfa1.vortex.core.error.VortexException;
 import io.github.dfa1.vortex.core.io.VortexFormat;
 import io.github.dfa1.vortex.core.proto.ProtoFSSTMetadata;
 
@@ -101,8 +102,10 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
         }
 
         int totalCompressed = 0;
-        for (byte[] c : compressed) {
-            totalCompressed += c.length;
+        int maxUncompLen = 0;
+        for (int i = 0; i < n; i++) {
+            totalCompressed += compressed[i].length;
+            maxUncompLen = Math.max(maxUncompLen, byteArrays[i].length);
         }
         MemorySegment compBuf = arena.allocate(Math.max(totalCompressed, 1));
         long pos = 0;
@@ -111,22 +114,29 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
             pos += c.length;
         }
 
-        MemorySegment uncompLenBuf = arena.allocate(Math.max(n * 4L, 1), 4);
+        // Narrowest ptype that fits every value: row lengths and cumulative offsets are
+        // typically far below the 4-byte ceiling this always used to pay (e.g. a 6-byte string
+        // column needs only U8 lengths, not I32), and the wire format carries the chosen ptype
+        // per FSSTMetadata specifically so a reader never has to guess.
+        PType uncompLenPType = narrowestUnsignedPType(maxUncompLen);
+        PType codesOffPType = narrowestUnsignedPType(totalCompressed);
+
+        MemorySegment uncompLenBuf = arena.allocate(Math.max((long) n * uncompLenPType.byteSize(), 1));
         for (int i = 0; i < n; i++) {
-            uncompLenBuf.setAtIndex(VortexFormat.LE_INT, i, byteArrays[i].length);
+            writeUnsigned(uncompLenBuf, uncompLenPType, i, byteArrays[i].length);
         }
 
-        MemorySegment codesOffBuf = arena.allocate((long) (n + 1) * 4, 4);
+        MemorySegment codesOffBuf = arena.allocate((long) (n + 1) * codesOffPType.byteSize());
         long off = 0;
-        codesOffBuf.setAtIndex(VortexFormat.LE_INT, 0, 0);
+        writeUnsigned(codesOffBuf, codesOffPType, 0, 0);
         for (int i = 0; i < n; i++) {
             off += compressed[i].length;
-            codesOffBuf.setAtIndex(VortexFormat.LE_INT, (long) i + 1, (int) off);
+            writeUnsigned(codesOffBuf, codesOffPType, i + 1, off);
         }
 
         byte[] metaBytes = new ProtoFSSTMetadata(
-                io.github.dfa1.vortex.core.proto.ProtoPType.fromValue(PType.I32.ordinal()),
-                io.github.dfa1.vortex.core.proto.ProtoPType.fromValue(PType.I32.ordinal())
+                io.github.dfa1.vortex.core.proto.ProtoPType.fromValue(uncompLenPType.ordinal()),
+                io.github.dfa1.vortex.core.proto.ProtoPType.fromValue(codesOffPType.ordinal())
         ).encode();
 
         EncodeNode uncompLensNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 3);
@@ -142,6 +152,35 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
                 null, null);
     }
 
+    /// Narrowest unsigned `PType` that can hold every value up to `maxValue`.
+    ///
+    /// @param maxValue the largest value the buffer must represent, `>= 0`
+    /// @return `U8`, `U16`, or `U32`, whichever is smallest and still fits
+    private static PType narrowestUnsignedPType(int maxValue) {
+        if (maxValue <= 0xFF) {
+            return PType.U8;
+        }
+        if (maxValue <= 0xFFFF) {
+            return PType.U16;
+        }
+        return PType.U32;
+    }
+
+    /// Writes `value` into `seg` at row `idx`, using `ptype`'s byte width.
+    ///
+    /// @param seg   destination segment
+    /// @param ptype `U8`, `U16`, or `U32` (whatever [#narrowestUnsignedPType(int)] returned)
+    /// @param idx   row index (not a byte offset)
+    /// @param value the value to write
+    private static void writeUnsigned(MemorySegment seg, PType ptype, long idx, long value) {
+        switch (ptype) {
+            case U8 -> seg.set(ValueLayout.JAVA_BYTE, idx, (byte) value);
+            case U16 -> seg.set(VortexFormat.LE_SHORT, idx * 2, (short) value);
+            case U32 -> seg.set(VortexFormat.LE_INT, idx * 4, (int) value);
+            default -> throw new VortexException(EncodingId.VORTEX_FSST, "unexpected ptype: " + ptype);
+        }
+    }
+
     /// Trains a symbol table by iteratively refining candidate symbols against a bounded sample.
     ///
     /// @param byteArrays the raw byte content of every input string
diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java
index 1c584633..8c28c8aa 100644
--- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java
+++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java
@@ -332,8 +332,9 @@ void decode_missingMetadata_throwsVortexException() {
     class Metadata {
 
         @Test
-        void encode_metadata_ptypes_areI32() throws Exception {
-            // Given
+        void encode_metadata_ptypes_pickNarrowestThatFits_smallData() throws Exception {
+            // Given — row lengths and cumulative offsets both well under 256, so the narrowest
+            // ptype (U8) should be picked instead of a fixed-width I32 that wastes 3 bytes/row.
             String[] data = {"hello", "world", "hello", "fsst"};
 
             // When
@@ -342,8 +343,42 @@ void encode_metadata_ptypes_areI32() throws Exception {
             ProtoFSSTMetadata meta = ProtoFSSTMetadata.decode(metaSeg, 0, metaSeg.byteSize());
 
             // Then
-            assertThat(meta.uncompressed_lengths_ptype().value()).isEqualTo(6);
-            assertThat(meta.codes_offsets_ptype().value()).isEqualTo(6);
+            assertThat(meta.uncompressed_lengths_ptype().value()).isEqualTo(PType.U8.ordinal());
+            assertThat(meta.codes_offsets_ptype().value()).isEqualTo(PType.U8.ordinal());
+        }
+
+        @Test
+        void encode_metadata_uncompressedLengthsPType_escalates_pastU8Range() throws Exception {
+            // Given — a single row longer than 255 bytes. Content is irrelevant here: this
+            // ptype tracks raw row length, not compressed size.
+            String[] data = {"a".repeat(300)};
+
+            // When
+            EncodeResult result = ENCODER.encode(DTypes.UTF8, data, EncodeTestHelper.testCtx());
+            var metaSeg = result.rootNode().metadata();
+            ProtoFSSTMetadata meta = ProtoFSSTMetadata.decode(metaSeg, 0, metaSeg.byteSize());
+
+            // Then
+            assertThat(meta.uncompressed_lengths_ptype().value()).isEqualTo(PType.U16.ordinal());
+        }
+
+        @Test
+        void encode_metadata_codesOffsetsPType_escalates_pastU8Range() throws Exception {
+            // Given — 300 single-character rows. Each row is too short for any multi-byte
+            // symbol match, so it compresses to exactly one code byte regardless of how
+            // training ranks candidates; the cumulative codes_offsets total is therefore
+            // deterministically 300, past the U8 range, whether or not "x" makes the table.
+            String[] data = new String[300];
+            java.util.Arrays.fill(data, "x");
+
+            // When
+            EncodeResult result = ENCODER.encode(DTypes.UTF8, data, EncodeTestHelper.testCtx());
+            var metaSeg = result.rootNode().metadata();
+            ProtoFSSTMetadata meta = ProtoFSSTMetadata.decode(metaSeg, 0, metaSeg.byteSize());
+
+            // Then
+            assertThat(meta.uncompressed_lengths_ptype().value()).isEqualTo(PType.U8.ordinal());
+            assertThat(meta.codes_offsets_ptype().value()).isEqualTo(PType.U16.ordinal());
         }
     }
 }

From 377a0ee889a3da436e33cef62ad95d73e0f91f15 Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Mon, 13 Jul 2026 08:28:11 +0200
Subject: [PATCH 04/11] raincloud: triage 7 more slugs to ok
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ai2-arc, amazon-reviews-2023-subscription-boxes, anthropic-economic-index,
anthropic-hh-rlhf-helpful-base, anthropic-interviewer, aya-collection-templated,
aya-dataset all match the Parquet oracle exactly — no gaps found.

3 remaining candidates (airbnb-prices-in-european-cities, airbnbopendata,
bank-account-fraud-dataset-neurips-2022) are Kaggle-hosted and need
Kaggle API credentials this environment doesn't have; left untriaged.

117 ok/130 untriaged -> 124 ok/123 untriaged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../test/resources/raincloud/expected-status.csv   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv
index 7c43ce85..52646427 100644
--- a/integration/src/test/resources/raincloud/expected-status.csv
+++ b/integration/src/test/resources/raincloud/expected-status.csv
@@ -8,16 +8,16 @@
 # Corpus: all slugs with a Vortex artifact in the Raincloud catalog (spiraldb/raincloud),
 # written by vortex-data 0.69.0. Hydrate any subset with scripts/hydrate-raincloud-corpus.sh.
 120-years-of-olympic-history-athletes-and-results,ok
-ai2-arc,untriaged
+ai2-arc,ok
 airbnb-prices-in-european-cities,untriaged
 airbnbopendata,untriaged
-amazon-reviews-2023-subscription-boxes,untriaged
+amazon-reviews-2023-subscription-boxes,ok
 ampds-whe,ok
-anthropic-economic-index,untriaged
-anthropic-hh-rlhf-helpful-base,untriaged
-anthropic-interviewer,untriaged
-aya-collection-templated,untriaged
-aya-dataset,untriaged
+anthropic-economic-index,ok
+anthropic-hh-rlhf-helpful-base,ok
+anthropic-interviewer,ok
+aya-collection-templated,ok
+aya-dataset,ok
 bank-account-fraud-dataset-neurips-2022,untriaged
 basketball,untriaged
 behavioral-risk-factor-surveillance-system,ok

From b1397b5a01ecf96cec234261e9c36a60f5d96267 Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Mon, 13 Jul 2026 08:30:16 +0200
Subject: [PATCH 05/11] raincloud: annotate credential-blocked untriaged slugs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Comment lines above the 3 Kaggle-hosted slugs from this round
(airbnb-prices-in-european-cities, airbnbopendata,
bank-account-fraud-dataset-neurips-2022) noting they need Kaggle API
credentials to hydrate — distinguishes "blocked on creds" from
"just not attempted yet" for future triagers. Status stays literal
"untriaged" since the harness string-matches it exactly; comments are
skipped by the parser.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../src/test/resources/raincloud/expected-status.csv        | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv
index 52646427..a06311ab 100644
--- a/integration/src/test/resources/raincloud/expected-status.csv
+++ b/integration/src/test/resources/raincloud/expected-status.csv
@@ -5,11 +5,16 @@
 #         untriaged  — not yet run; the test executes it and reports the outcome as an
 #                      aborted (skipped) test instead of failing the build — flip to ok/gap
 #                      as hydration coverage grows
+# A `#`-comment directly above an `untriaged` line notes WHY it hasn't been hydrated yet
+# (e.g. missing credentials for a gated source) — status stays literal "untriaged" (the
+# harness string-matches it exactly), the comment is for triagers only.
 # Corpus: all slugs with a Vortex artifact in the Raincloud catalog (spiraldb/raincloud),
 # written by vortex-data 0.69.0. Hydrate any subset with scripts/hydrate-raincloud-corpus.sh.
 120-years-of-olympic-history-athletes-and-results,ok
 ai2-arc,ok
+# needs Kaggle credentials to hydrate (kaggle auth login / KAGGLE_API_TOKEN) — not attempted
 airbnb-prices-in-european-cities,untriaged
+# needs Kaggle credentials to hydrate (kaggle auth login / KAGGLE_API_TOKEN) — not attempted
 airbnbopendata,untriaged
 amazon-reviews-2023-subscription-boxes,ok
 ampds-whe,ok
@@ -18,6 +23,7 @@ anthropic-hh-rlhf-helpful-base,ok
 anthropic-interviewer,ok
 aya-collection-templated,ok
 aya-dataset,ok
+# needs Kaggle credentials to hydrate (kaggle auth login / KAGGLE_API_TOKEN) — not attempted
 bank-account-fraud-dataset-neurips-2022,untriaged
 basketball,untriaged
 behavioral-risk-factor-surveillance-system,ok

From b081a17eda294077174d426c8ea8bad1fac77aa8 Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Mon, 13 Jul 2026 08:46:28 +0200
Subject: [PATCH 06/11] raincloud: add missing_auth status
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replace the ad-hoc "#-comment above an untriaged line" convention with
a proper matrix status: missing_auth marks a slug that can't even be
hydrated because its source (Kaggle, gated HuggingFace, ...) needs
credentials this environment doesn't have. Distinguishes "blocked on
a credential" from plain untriaged ("just hasn't been attempted") so
future triagers don't waste time rediscovering the same blocker.

Harness treats it identically to untriaged (reportUntriaged, not
assertStillFails — that path means "known reader gap", which this
isn't). Applied to the 3 Kaggle-blocked slugs from the last round.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../RaincloudConformanceIntegrationTest.java  |  6 +++-
 .../resources/raincloud/expected-status.csv   | 32 ++++++++++---------
 2 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java
index bd8b9aad..bb6d0168 100644
--- a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java
+++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java
@@ -87,7 +87,11 @@ Stream<DynamicTest> conformancePerSlug() throws IOException {
                     String status = expected.getOrDefault(slug, "untriaged");
                     switch (status) {
                         case "ok" -> assertMatchesParquetOracle(vortex, parquet);
-                        case "untriaged" -> reportUntriaged(vortex, parquet);
+                        // missing_auth slugs never reach here in practice — hydration fails
+                        // before a manifest entry exists — but report the same as untriaged
+                        // rather than falling through to assertStillFails (wrong: that path
+                        // means "known reader gap", not "couldn't even attempt this").
+                        case "untriaged", "missing_auth" -> reportUntriaged(vortex, parquet);
                         default -> assertStillFails(vortex, parquet, status);
                     }
                 }));
diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv
index a06311ab..a99c092a 100644
--- a/integration/src/test/resources/raincloud/expected-status.csv
+++ b/integration/src/test/resources/raincloud/expected-status.csv
@@ -1,21 +1,23 @@
 # Raincloud conformance matrix — one line per vortex-enabled corpus slug: <slug>,<status>
-# status: ok         — vortex-java must read the file and match the parquet oracle exactly
-#         gap:<n>    — known reader gap tracked by issue #<n>; the test asserts the failure
-#                      still occurs, so a fix forces flipping the entry to ok in the same change
-#         untriaged  — not yet run; the test executes it and reports the outcome as an
-#                      aborted (skipped) test instead of failing the build — flip to ok/gap
-#                      as hydration coverage grows
-# A `#`-comment directly above an `untriaged` line notes WHY it hasn't been hydrated yet
-# (e.g. missing credentials for a gated source) — status stays literal "untriaged" (the
-# harness string-matches it exactly), the comment is for triagers only.
+# status: ok           — vortex-java must read the file and match the parquet oracle exactly
+#         gap:<n>      — known reader gap tracked by issue #<n>; the test asserts the failure
+#                        still occurs, so a fix forces flipping the entry to ok in the same change
+#         untriaged    — not yet run; the test executes it and reports the outcome as an
+#                        aborted (skipped) test instead of failing the build — flip to ok/gap
+#                        as hydration coverage grows
+#         missing_auth — can't even hydrate: the source (Kaggle, gated HuggingFace, …) needs
+#                        credentials this environment doesn't have. Distinguishes "blocked on
+#                        a credential" from plain untriaged ("just hasn't been attempted yet")
+#                        so a future triager doesn't waste time rediscovering the same blocker.
+#                        An optional `#`-comment above the line can name the credential.
 # Corpus: all slugs with a Vortex artifact in the Raincloud catalog (spiraldb/raincloud),
 # written by vortex-data 0.69.0. Hydrate any subset with scripts/hydrate-raincloud-corpus.sh.
 120-years-of-olympic-history-athletes-and-results,ok
 ai2-arc,ok
-# needs Kaggle credentials to hydrate (kaggle auth login / KAGGLE_API_TOKEN) — not attempted
-airbnb-prices-in-european-cities,untriaged
-# needs Kaggle credentials to hydrate (kaggle auth login / KAGGLE_API_TOKEN) — not attempted
-airbnbopendata,untriaged
+# kaggle auth login / KAGGLE_API_TOKEN
+airbnb-prices-in-european-cities,missing_auth
+# kaggle auth login / KAGGLE_API_TOKEN
+airbnbopendata,missing_auth
 amazon-reviews-2023-subscription-boxes,ok
 ampds-whe,ok
 anthropic-economic-index,ok
@@ -23,8 +25,8 @@ anthropic-hh-rlhf-helpful-base,ok
 anthropic-interviewer,ok
 aya-collection-templated,ok
 aya-dataset,ok
-# needs Kaggle credentials to hydrate (kaggle auth login / KAGGLE_API_TOKEN) — not attempted
-bank-account-fraud-dataset-neurips-2022,untriaged
+# kaggle auth login / KAGGLE_API_TOKEN
+bank-account-fraud-dataset-neurips-2022,missing_auth
 basketball,untriaged
 behavioral-risk-factor-surveillance-system,ok
 beir-msmarco,untriaged

From eec25c177678ba348c554ed334d793a193e8eb89 Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Mon, 13 Jul 2026 09:18:11 +0200
Subject: [PATCH 07/11] docs: CHANGELOG entries for FSST/dispatch work

Cover this session's production-code changes not yet documented:
Utf8/Binary cost-based dispatch fix, FsstEncodingDecoder null-metadata
bug, FSST wire symbol-table ordering fix, and the FSST encoder's
iterative training + non-boxing lookup + ptype narrowing.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d054e4fb..1d11dcd4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `MaskedEncodingEncoder` also tries `vortex.sparse` for a mixed-validity bitmap (fill = the majority value, patches = the minority positions) and keeps it over the raw bitmap when smaller — the patch-index array is compressed through a dedicated `vortex.sequence`/`fastlanes.delta`/FoR/bitpack cascade, so a clustered or regular null pattern (e.g. one null every 10 rows) collapses to a handful of bytes instead of a full bitmap. On the 200k-row, 10-chunk nullable low-cardinality Utf8 benchmark this drops the file a further 20% (107 KB → 86 KB) and the gap to the Rust reference from ~1.45× to ~1.17×. Also: `SequenceEncodingEncoder.encodeCascade` no longer lets `encode`'s "not an arithmetic sequence" exception escape when a stratified sample looked arithmetic but the full data wasn't — it now reports the cascade step as not applicable, like every other encoder's contract requires. (internal)
 - `ConstantEncodingEncoder` now accepts `DType.Bool` (previously `DType.Primitive` only), so any all-true or all-false dense Bool column — not just a masked/nullable one — can win `vortex.constant` through the normal per-column cascade, not only through `MaskedEncodingEncoder`'s dedicated check. `MaskedEncodingEncoder`'s validity encoding now delegates to it instead of duplicating the scalar-building logic. `RunEndEncodingEncoder` also gains a `vortex.runend` path for Bool: `MaskedEncodingEncoder` tries it alongside `vortex.sparse` and keeps whichever is smallest — `vortex.runend` wins on long clustered runs of valid/invalid rows (regardless of which value dominates), a shape `vortex.sparse`'s per-flip patch cost handles poorly. (internal)
 - `RleEncodingEncoder`/`RleEncodingDecoder` (`fastlanes.rle`) round out Bool coverage: the same FastLanes 1024-row chunked layout the numeric path uses, with a `vortex.bool`-encoded values pool (at most 2 distinct values per chunk) instead of a ptype-width one. `MaskedEncodingEncoder` tries it alongside `vortex.sparse`/`vortex.runend` and keeps the smallest — `DType.Bool` now has every general-purpose encoding family (`vortex.constant`, `vortex.sparse`, `vortex.runend`, `fastlanes.rle`) that `DType.Primitive` has. (internal)
+- `CascadingCompressor` now runs `Utf8`/`Binary` columns through the same sample-and-measure encoding competition `Primitive` columns already get, instead of first-match dispatch. `DictEncodingEncoder` is registered ahead of `FsstEncodingEncoder`, so Dict always won regardless of actual output size — confirmed by inspecting a 50k-row, high-cardinality string file and finding `vortex.dict` where `vortex.fsst` is ~40% smaller. On `highCardinalityUtf8_javaVsJni` (50k distinct 6-byte strings) this drops the Java/JNI file-size ratio from 2.26× to 1.36× (801,908 → 483,336 bytes). ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
+- `FsstEncodingDecoder` no longer throws on a valid file whose `vortex.fsst` metadata segment is absent. Proto3 omits fields at their default (zero/`U8`) value, so an all-`U8` metadata message legitimately encodes to zero bytes and the writer omits the segment entirely; the decoder previously treated any absent metadata as corruption. Also adds explicit buffer-count (3) and child-count (≥2) bounds checks, which the removed null-metadata check had been incidentally — and incorrectly — covering for a separate malformed-input case. ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
+- `FsstEncodingEncoder` now writes its symbol table in the order the Rust reference requires (`FSSTData::validate_symbol_lengths`: multi-byte symbols non-decreasing by length, then all length-1 symbols) — a Java-written file with an out-of-order table previously failed to open in the JNI/Rust reader. ([95e0dfb4](https://github.com/dfa1/vortex-java/commit/95e0dfb4))
+
+### Performance
+
+- `FsstEncodingEncoder` replaces its fixed 2-byte bigram symbol table with iterative FSST-paper-style training (longest-match-first parsing, gain-ranked candidate refinement over several passes), producing variable-length 1–8 byte symbols instead of a fixed 2-byte cap. Also narrows the `uncompressed_lengths`/`codes_offsets` metadata buffers to the smallest ptype that fits the actual data (previously always 4-byte `I32`) using wire-format fields that already existed for this. ([95e0dfb4](https://github.com/dfa1/vortex-java/commit/95e0dfb4), [1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
+- `FsstEncodingEncoder`'s symbol-match lookup replaces a `HashMap<SymbolCandidate, Integer>` (boxing a record key on every probe) with a flat open-addressing table over primitive arrays; training sampling switches from "first N rows" to the same stratified stride sampling `CascadingCompressor` uses elsewhere, avoiding bias on sorted/clustered input. ~18–23% faster encode on incompressible data, ~10% on repetitive text. ([4a3fa0a9](https://github.com/dfa1/vortex-java/commit/4a3fa0a9))
 
 ## [0.12.2] — 2026-07-12
 

From a920ff40838c082bbcedebdb3c2686bec18865ce Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Mon, 13 Jul 2026 09:24:58 +0200
Subject: [PATCH 08/11] docs: terser CHANGELOG entries, add changelog skill
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The 5 entries I added for the FSST/dispatch work were verbose
paragraphs (full rationale + before/after numbers inline) — too
verbose per feedback. Rewrote them as one-sentence titles ending in
a commit link; the numbers and rationale live in the commit message,
which the link points to.

Also records a standalone `changelog` skill (.claude/skills/) codifying
this convention, split out of the `release` skill's CHANGELOG-sync step
so it can run without cutting a release.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .claude/skills/changelog.md | 64 +++++++++++++++++++++++++++++++++++++
 CHANGELOG.md                | 10 +++---
 2 files changed, 69 insertions(+), 5 deletions(-)
 create mode 100644 .claude/skills/changelog.md

diff --git a/.claude/skills/changelog.md b/.claude/skills/changelog.md
new file mode 100644
index 00000000..d95eab0e
--- /dev/null
+++ b/.claude/skills/changelog.md
@@ -0,0 +1,64 @@
+---
+name: changelog
+description: Write CHANGELOG.md entries for unreleased commits — terse, one-line bullets that link to the introducing commit(s), not paragraphs. Triggers on "prepare the changelog", "update CHANGELOG", "write changelog entries", or invokes `/changelog`. Standalone version of the `release` skill's "CHANGELOG sync" step — use this when you want entries written without cutting a release yet.
+---
+
+## Overview
+
+Write CHANGELOG.md entries for commits that don't have one yet, under `## [Unreleased]`.
+Entries are **terse** — a title, not a paragraph — ending in a link to the introducing
+commit(s). The link is the receipt; anyone who wants the "why" and the numbers reads the
+commit message (`git show <sha>` or the GitHub commit page), not the changelog.
+
+## Steps
+
+1. Find the range of commits missing a changelog entry: compare `## [Unreleased]` in
+   `CHANGELOG.md` against `git log --oneline <last-documented-commit>..HEAD`. If unclear
+   where the last-documented commit is, ask.
+2. For each notable commit, write **one bullet**:
+   - A single short sentence — what changed, fewest words that stay precise. No numbers,
+     no "why", no before/after prose — those live in the commit message.
+   - End with the commit SHA(s), GitHub-auto-linked:
+     `([abc1234](https://github.com/dfa1/vortex-java/commit/abc1234))`
+   - One logical change spanning multiple commits: comma-separate SHAs, newest first.
+3. Categorize under the right `###` heading (create if missing): `Added` / `Changed` /
+   `Fixed` / `Performance` / `Removed` / `Security` — Keep-a-Changelog order (Added,
+   Changed, Deprecated, Removed, Fixed, Security), plus this project's `Performance`.
+4. Skip pure-internal churn: CI/Sonar fixes, source-tree moves, dedup refactors,
+   test/tooling-only commits (see [[feedback_changelog_terse]]) — omit rather than write a
+   bullet nobody user-facing cares about.
+5. Commit as `docs: CHANGELOG entries for <short description>`.
+
+## Example
+
+Good (terse — what to write):
+```
+- Utf8/Binary now use cost-based encoding selection instead of first-match dispatch. ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
+- FsstEncodingDecoder no longer throws on valid files with empty FSST metadata. ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
+```
+
+Bad (too verbose — do NOT write this):
+```
+- `CascadingCompressor` now runs `Utf8`/`Binary` columns through the same sample-and-measure
+  encoding competition `Primitive` columns already get, instead of first-match dispatch.
+  `DictEncodingEncoder` is registered ahead of `FsstEncodingEncoder`, so Dict always won
+  regardless of actual output size — confirmed by inspecting a 50k-row, high-cardinality
+  string file and finding `vortex.dict` where `vortex.fsst` is ~40% smaller. On
+  `highCardinalityUtf8_javaVsJni` (50k distinct 6-byte strings) this drops the Java/JNI
+  file-size ratio from 2.26× to 1.36× (801,908 → 483,336 bytes). ([1bbc3549](...))
+```
+The numbers and rationale belong in the commit message, not duplicated in the changelog.
+
+## Rules
+
+- Never invent entries for commits that don't exist — the link is the receipt.
+- Never leave a bullet without a commit link.
+- One sentence per bullet. If it needs two sentences, split it into two bullets, or the
+  extra detail belongs in the commit message instead.
+- Skip internal-only commits (matches the `release` skill's CHANGELOG sync policy).
+
+## When NOT to use
+
+- Finalizing a release's version header/date, or the compare link at the bottom of the
+  file — that's the `release` skill's job (its CHANGELOG sync step supersedes this one).
+- Writing release notes for work that isn't committed yet.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1d11dcd4..c8aafb3f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,14 +14,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `MaskedEncodingEncoder` also tries `vortex.sparse` for a mixed-validity bitmap (fill = the majority value, patches = the minority positions) and keeps it over the raw bitmap when smaller — the patch-index array is compressed through a dedicated `vortex.sequence`/`fastlanes.delta`/FoR/bitpack cascade, so a clustered or regular null pattern (e.g. one null every 10 rows) collapses to a handful of bytes instead of a full bitmap. On the 200k-row, 10-chunk nullable low-cardinality Utf8 benchmark this drops the file a further 20% (107 KB → 86 KB) and the gap to the Rust reference from ~1.45× to ~1.17×. Also: `SequenceEncodingEncoder.encodeCascade` no longer lets `encode`'s "not an arithmetic sequence" exception escape when a stratified sample looked arithmetic but the full data wasn't — it now reports the cascade step as not applicable, like every other encoder's contract requires. (internal)
 - `ConstantEncodingEncoder` now accepts `DType.Bool` (previously `DType.Primitive` only), so any all-true or all-false dense Bool column — not just a masked/nullable one — can win `vortex.constant` through the normal per-column cascade, not only through `MaskedEncodingEncoder`'s dedicated check. `MaskedEncodingEncoder`'s validity encoding now delegates to it instead of duplicating the scalar-building logic. `RunEndEncodingEncoder` also gains a `vortex.runend` path for Bool: `MaskedEncodingEncoder` tries it alongside `vortex.sparse` and keeps whichever is smallest — `vortex.runend` wins on long clustered runs of valid/invalid rows (regardless of which value dominates), a shape `vortex.sparse`'s per-flip patch cost handles poorly. (internal)
 - `RleEncodingEncoder`/`RleEncodingDecoder` (`fastlanes.rle`) round out Bool coverage: the same FastLanes 1024-row chunked layout the numeric path uses, with a `vortex.bool`-encoded values pool (at most 2 distinct values per chunk) instead of a ptype-width one. `MaskedEncodingEncoder` tries it alongside `vortex.sparse`/`vortex.runend` and keeps the smallest — `DType.Bool` now has every general-purpose encoding family (`vortex.constant`, `vortex.sparse`, `vortex.runend`, `fastlanes.rle`) that `DType.Primitive` has. (internal)
-- `CascadingCompressor` now runs `Utf8`/`Binary` columns through the same sample-and-measure encoding competition `Primitive` columns already get, instead of first-match dispatch. `DictEncodingEncoder` is registered ahead of `FsstEncodingEncoder`, so Dict always won regardless of actual output size — confirmed by inspecting a 50k-row, high-cardinality string file and finding `vortex.dict` where `vortex.fsst` is ~40% smaller. On `highCardinalityUtf8_javaVsJni` (50k distinct 6-byte strings) this drops the Java/JNI file-size ratio from 2.26× to 1.36× (801,908 → 483,336 bytes). ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
-- `FsstEncodingDecoder` no longer throws on a valid file whose `vortex.fsst` metadata segment is absent. Proto3 omits fields at their default (zero/`U8`) value, so an all-`U8` metadata message legitimately encodes to zero bytes and the writer omits the segment entirely; the decoder previously treated any absent metadata as corruption. Also adds explicit buffer-count (3) and child-count (≥2) bounds checks, which the removed null-metadata check had been incidentally — and incorrectly — covering for a separate malformed-input case. ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
-- `FsstEncodingEncoder` now writes its symbol table in the order the Rust reference requires (`FSSTData::validate_symbol_lengths`: multi-byte symbols non-decreasing by length, then all length-1 symbols) — a Java-written file with an out-of-order table previously failed to open in the JNI/Rust reader. ([95e0dfb4](https://github.com/dfa1/vortex-java/commit/95e0dfb4))
+- `Utf8`/`Binary` columns now use cost-based encoding selection instead of first-match dispatch (Dict no longer always beats FSST regardless of size). ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
+- `FsstEncodingDecoder` no longer throws on valid files with an all-default (empty) `vortex.fsst` metadata segment. ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
+- `FsstEncodingEncoder` writes symbol tables in the order the Rust reference requires; previously an out-of-order table could fail to open in the JNI/Rust reader. ([95e0dfb4](https://github.com/dfa1/vortex-java/commit/95e0dfb4))
 
 ### Performance
 
-- `FsstEncodingEncoder` replaces its fixed 2-byte bigram symbol table with iterative FSST-paper-style training (longest-match-first parsing, gain-ranked candidate refinement over several passes), producing variable-length 1–8 byte symbols instead of a fixed 2-byte cap. Also narrows the `uncompressed_lengths`/`codes_offsets` metadata buffers to the smallest ptype that fits the actual data (previously always 4-byte `I32`) using wire-format fields that already existed for this. ([95e0dfb4](https://github.com/dfa1/vortex-java/commit/95e0dfb4), [1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
-- `FsstEncodingEncoder`'s symbol-match lookup replaces a `HashMap<SymbolCandidate, Integer>` (boxing a record key on every probe) with a flat open-addressing table over primitive arrays; training sampling switches from "first N rows" to the same stratified stride sampling `CascadingCompressor` uses elsewhere, avoiding bias on sorted/clustered input. ~18–23% faster encode on incompressible data, ~10% on repetitive text. ([4a3fa0a9](https://github.com/dfa1/vortex-java/commit/4a3fa0a9))
+- `FsstEncodingEncoder`: iterative symbol-table training (variable-length 1–8 byte symbols, was a fixed 2-byte bigram scheme) and narrower metadata buffer ptypes. ([95e0dfb4](https://github.com/dfa1/vortex-java/commit/95e0dfb4), [1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
+- `FsstEncodingEncoder`: non-boxing symbol-match lookup and stratified training-sample selection. ([4a3fa0a9](https://github.com/dfa1/vortex-java/commit/4a3fa0a9))
 
 ## [0.12.2] — 2026-07-12
 

From 70053ea762fb66e29342ad28b98074e068eb9a2d Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Mon, 13 Jul 2026 09:26:58 +0200
Subject: [PATCH 09/11] docs: apply terse CHANGELOG convention retroactively

Rewrite the 5 remaining paragraph-style Unreleased entries (nullable
global dict, Bool constant/sparse/runend/rle) as one-sentence titles
with commit links, matching the new changelog skill. Splits the
combined Sparse+Sequence and Constant+RunEnd bullets into one entry
per logical change (same source commit, distinct fixes).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 CHANGELOG.md | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c8aafb3f..d1659d11 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,11 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
-- Nullable low-cardinality columns now use the shared global dictionary across chunks instead of re-emitting a per-chunk dictionary. The writer previously excluded any nullable column from global-dict candidacy, so a nullable low-cardinality string/numeric column (extremely common in real data — most Raincloud categorical columns have nulls) re-wrote its dictionary in every chunk. Nullable columns now run the same cardinality/ratio gate against their non-null values and emit one shared `vortex.dict` layout with per-chunk masked codes; the reader combines codes-side and pool-side validity per row. On a 200k-row, 10-chunk nullable low-cardinality Utf8 column this shrinks the file 54% (232 KB → 107 KB) and closes the gap to the Rust reference from ~4× to ~1.5×. (internal)
-- `MaskedEncodingEncoder` now encodes an all-valid or all-invalid validity bitmap as `vortex.constant` instead of a raw per-row bitmap. A nullable column chunk with zero nulls previously still paid a full `n/8`-byte bitmap for no information; an all-valid or all-invalid chunk now costs a few bytes regardless of row count. (internal)
-- `MaskedEncodingEncoder` also tries `vortex.sparse` for a mixed-validity bitmap (fill = the majority value, patches = the minority positions) and keeps it over the raw bitmap when smaller — the patch-index array is compressed through a dedicated `vortex.sequence`/`fastlanes.delta`/FoR/bitpack cascade, so a clustered or regular null pattern (e.g. one null every 10 rows) collapses to a handful of bytes instead of a full bitmap. On the 200k-row, 10-chunk nullable low-cardinality Utf8 benchmark this drops the file a further 20% (107 KB → 86 KB) and the gap to the Rust reference from ~1.45× to ~1.17×. Also: `SequenceEncodingEncoder.encodeCascade` no longer lets `encode`'s "not an arithmetic sequence" exception escape when a stratified sample looked arithmetic but the full data wasn't — it now reports the cascade step as not applicable, like every other encoder's contract requires. (internal)
-- `ConstantEncodingEncoder` now accepts `DType.Bool` (previously `DType.Primitive` only), so any all-true or all-false dense Bool column — not just a masked/nullable one — can win `vortex.constant` through the normal per-column cascade, not only through `MaskedEncodingEncoder`'s dedicated check. `MaskedEncodingEncoder`'s validity encoding now delegates to it instead of duplicating the scalar-building logic. `RunEndEncodingEncoder` also gains a `vortex.runend` path for Bool: `MaskedEncodingEncoder` tries it alongside `vortex.sparse` and keeps whichever is smallest — `vortex.runend` wins on long clustered runs of valid/invalid rows (regardless of which value dominates), a shape `vortex.sparse`'s per-flip patch cost handles poorly. (internal)
-- `RleEncodingEncoder`/`RleEncodingDecoder` (`fastlanes.rle`) round out Bool coverage: the same FastLanes 1024-row chunked layout the numeric path uses, with a `vortex.bool`-encoded values pool (at most 2 distinct values per chunk) instead of a ptype-width one. `MaskedEncodingEncoder` tries it alongside `vortex.sparse`/`vortex.runend` and keeps the smallest — `DType.Bool` now has every general-purpose encoding family (`vortex.constant`, `vortex.sparse`, `vortex.runend`, `fastlanes.rle`) that `DType.Primitive` has. (internal)
+- Nullable low-cardinality columns now share one global dictionary across chunks instead of re-emitting a per-chunk dictionary. ([5fe8b544](https://github.com/dfa1/vortex-java/commit/5fe8b544))
+- `MaskedEncodingEncoder` encodes an all-valid or all-invalid validity bitmap as `vortex.constant` instead of a raw per-row bitmap. ([ecd47ead](https://github.com/dfa1/vortex-java/commit/ecd47ead))
+- `MaskedEncodingEncoder` tries `vortex.sparse` for a mixed-validity bitmap and keeps it over the raw bitmap when smaller. ([506d036f](https://github.com/dfa1/vortex-java/commit/506d036f))
+- `SequenceEncodingEncoder.encodeCascade` no longer lets a full-data encode exception escape when a sampled prefix looked arithmetic but the full data wasn't. ([506d036f](https://github.com/dfa1/vortex-java/commit/506d036f))
+- `ConstantEncodingEncoder` now accepts `DType.Bool` (previously `Primitive` only). ([5379af66](https://github.com/dfa1/vortex-java/commit/5379af66))
+- `RunEndEncodingEncoder` gains a `vortex.runend` path for Bool. ([5379af66](https://github.com/dfa1/vortex-java/commit/5379af66))
+- `fastlanes.rle` now supports `DType.Bool`, completing Bool coverage across all general-purpose encodings (`vortex.constant`, `vortex.sparse`, `vortex.runend`, `fastlanes.rle`). ([fe6f132b](https://github.com/dfa1/vortex-java/commit/fe6f132b))
 - `Utf8`/`Binary` columns now use cost-based encoding selection instead of first-match dispatch (Dict no longer always beats FSST regardless of size). ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
 - `FsstEncodingDecoder` no longer throws on valid files with an all-default (empty) `vortex.fsst` metadata segment. ([1bbc3549](https://github.com/dfa1/vortex-java/commit/1bbc3549))
 - `FsstEncodingEncoder` writes symbol tables in the order the Rust reference requires; previously an out-of-order table could fail to open in the JNI/Rust reader. ([95e0dfb4](https://github.com/dfa1/vortex-java/commit/95e0dfb4))

From d320d426d77bb1f6b3cf406196aefd5cf6d6f6f2 Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Tue, 14 Jul 2026 07:37:41 +0200
Subject: [PATCH 10/11] fix: chunked List columns and NullArray chunks in
 ChunkedLayoutDecoder

ChunkedLayoutDecoder.decodeChunkedLayout blindly cast dtype to
DType.Primitive for anything that wasn't Bool/Utf8/Binary, throwing a
raw ClassCastException instead of VortexException for chunked List
columns (#268). VarBinArray.ChunkedMode.of separately threw on a chunk
that decoded to NullArray (an all-null vortex.null/vortex.constant
chunk) instead of a VarBinArray (#269).

Extracts the per-chunk dispatch into ChunkedArrayCombiner: primitives
and Bool keep their existing ChunkedXxxArray shapes, Utf8/Binary route
through VarBinArray.ChunkedMode (now NullArray-aware, materializing an
all-null chunk as a zero-length OffsetMode run), List columns stitch
into one ListArray via a recursive combine of their element arrays plus
a rebuilt cumulative offsets table, and any other dtype now fails with
a VortexException instead of a raw cast. combineChunkValidity also
folds NullArray chunks into the row-validity bitmap it reconstructs
across chunks.

Found via the Raincloud conformance corpus (squad-v2, oasst1, mnist).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 CHANGELOG.md                                  |   2 +
 .../reader/array/ChunkedArrayCombiner.java    | 195 ++++++++++++++++++
 .../dfa1/vortex/reader/array/VarBinArray.java |  45 +++-
 .../reader/layout/ChunkedLayoutDecoder.java   |  87 +-------
 .../array/ChunkedArrayCombinerTest.java       | 124 +++++++++++
 .../reader/array/VarBinChunkedModeTest.java   |  41 ++++
 6 files changed, 414 insertions(+), 80 deletions(-)
 create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java
 create mode 100644 reader/src/test/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombinerTest.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d1659d11..f159b8e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- A chunked `List` column spanning several flat chunks now decodes into a single stitched array instead of throwing a raw `ClassCastException`; any other unhandled dtype now fails with `VortexException`. ([#268](https://github.com/dfa1/vortex-java/issues/268))
+- A chunked `Utf8`/`Binary` column with an entirely-null chunk (`NullArray`) no longer throws `chunk is not a VarBinArray`; the chunk materializes as an all-null run. ([#269](https://github.com/dfa1/vortex-java/issues/269))
 - Nullable low-cardinality columns now share one global dictionary across chunks instead of re-emitting a per-chunk dictionary. ([5fe8b544](https://github.com/dfa1/vortex-java/commit/5fe8b544))
 - `MaskedEncodingEncoder` encodes an all-valid or all-invalid validity bitmap as `vortex.constant` instead of a raw per-row bitmap. ([ecd47ead](https://github.com/dfa1/vortex-java/commit/ecd47ead))
 - `MaskedEncodingEncoder` tries `vortex.sparse` for a mixed-validity bitmap and keeps it over the raw bitmap when smaller. ([506d036f](https://github.com/dfa1/vortex-java/commit/506d036f))
diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java
new file mode 100644
index 00000000..64212539
--- /dev/null
+++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombiner.java
@@ -0,0 +1,195 @@
+package io.github.dfa1.vortex.reader.array;
+
+import io.github.dfa1.vortex.core.error.VortexException;
+import io.github.dfa1.vortex.core.io.VortexFormat;
+import io.github.dfa1.vortex.core.model.DType;
+import io.github.dfa1.vortex.core.model.PType;
+
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.SegmentAllocator;
+import java.lang.foreign.ValueLayout;
+import java.util.ArrayList;
+import java.util.List;
+
+/// Stitches the per-chunk arrays of one logical column into a single view, dispatching on the
+/// column's [DType]. Each family gets its zero-copy composite shape (ADR 0012): primitive and
+/// boolean chunks fold into the `ChunkedXxxArray` records, variable-length chunks into
+/// [VarBinArray.ChunkedMode], and list chunks into a stitched [ListArray] whose bulk element data
+/// stays zero-copy (the child element arrays are themselves combined recursively) while only the
+/// small outer offsets table is rebuilt so per-chunk offsets — which each reset to zero — become one
+/// cumulative table.
+///
+/// Extracted from `ChunkedLayoutDecoder` so the same dispatch drives both the top-level chunked
+/// column and the recursive combine of a chunked list's elements. Every unsupported dtype fails with
+/// a [VortexException] rather than a raw `ClassCastException` — the reader parses untrusted input and
+/// must never leak a JDK runtime exception (issue #268).
+public final class ChunkedArrayCombiner {
+
+    private ChunkedArrayCombiner() {
+    }
+
+    /// Combines the per-chunk arrays of one column into a single logical array, choosing the shape
+    /// from `dtype`. Masked chunks keep their per-chunk validity: it is concatenated into one
+    /// row-level bitmap and the result re-wrapped in a [MaskedArray], so a nullable column that
+    /// spans more than one chunk retains its nulls.
+    ///
+    /// @param dtype     the column's logical type
+    /// @param totalRows the total logical row count across all chunks
+    /// @param chunks    the decoded per-chunk arrays, in row order (non-empty)
+    /// @param arena     allocator for the combined validity bitmap and, for lists, the rebuilt offsets
+    /// @return the combined [Array], masked when any chunk contributes nulls
+    /// @throws VortexException on an empty chunk list or a dtype with no chunked shape
+    public static Array combine(DType dtype, long totalRows, List<Array> chunks,
+            SegmentAllocator arena) {
+        if (chunks.isEmpty()) {
+            throw new VortexException("chunked combine: empty chunk list");
+        }
+        Array data = switch (dtype) {
+            case DType.Bool ignored -> ChunkedBoolArray.of(dtype, totalRows, chunks);
+            case DType.Utf8 ignored -> VarBinArray.ChunkedMode.of(dtype, totalRows, chunks, arena);
+            case DType.Binary ignored -> VarBinArray.ChunkedMode.of(dtype, totalRows, chunks, arena);
+            case DType.List list -> combineLists(list, totalRows, chunks, arena);
+            case DType.Primitive prim -> combinePrimitive(prim.ptype(), dtype, totalRows, chunks);
+            default -> throw new VortexException("unsupported dtype for chunked layout: " + dtype);
+        };
+        BoolArray validity = combineChunkValidity(chunks, totalRows, arena);
+        return validity != null ? new MaskedArray(data, validity) : data;
+    }
+
+    private static Array combinePrimitive(PType ptype, DType dtype, long totalRows,
+            List<Array> chunks) {
+        return switch (ptype) {
+            case I64, U64 -> ChunkedLongArray.of(dtype, totalRows, chunks);
+            case I32, U32 -> ChunkedIntArray.of(dtype, totalRows, chunks);
+            case F64 -> ChunkedDoubleArray.of(dtype, totalRows, chunks);
+            case F32 -> ChunkedFloatArray.of(dtype, totalRows, chunks);
+            case I16, U16 -> ChunkedShortArray.of(dtype, totalRows, chunks);
+            case I8, U8 -> ChunkedByteArray.of(dtype, totalRows, chunks);
+            default -> throw new VortexException("unsupported ptype for chunked layout: " + ptype);
+        };
+    }
+
+    /// Stitches per-chunk [ListArray] chunks into one [ListArray]. The element arrays of every chunk
+    /// are combined recursively (staying zero-copy), and one cumulative I64 offsets table of
+    /// `totalRows + 1` entries is built in `arena`: each chunk's local offsets are shifted by the
+    /// number of elements contributed by earlier chunks, since per-chunk offsets each restart at
+    /// zero.
+    ///
+    /// @param dtype     the list's logical type
+    /// @param totalRows the total outer-list row count across all chunks
+    /// @param chunks    the per-chunk list arrays (each a [ListArray], possibly wrapped [MaskedArray])
+    /// @param arena     allocator for the rebuilt offsets segment
+    /// @return a single [ListArray] over the combined chunks
+    private static ListArray combineLists(DType.List dtype, long totalRows, List<Array> chunks,
+            SegmentAllocator arena) {
+        var listChunks = new ArrayList<ListArray>(chunks.size());
+        for (Array chunk : chunks) {
+            Array unwrapped = chunk instanceof MaskedArray m ? m.inner() : chunk;
+            if (unwrapped instanceof ListArray la) {
+                listChunks.add(la);
+            } else {
+                throw new VortexException("chunked list: chunk is not a ListArray: "
+                        + unwrapped.getClass().getSimpleName());
+            }
+        }
+        long outerRows = 0;
+        for (ListArray la : listChunks) {
+            outerRows += la.length();
+        }
+        if (outerRows != totalRows) {
+            throw new VortexException("chunked list: chunk rows sum to " + outerRows
+                    + ", expected " + totalRows);
+        }
+
+        var elementChunks = new ArrayList<Array>(listChunks.size());
+        for (ListArray la : listChunks) {
+            elementChunks.add(la.elements());
+        }
+        Array combinedElements = combine(dtype.elementType(), sumLengths(elementChunks),
+                elementChunks, arena);
+
+        MemorySegment offsets = arena.allocate((totalRows + 1) * Long.BYTES, Long.BYTES);
+        offsets.setAtIndex(VortexFormat.LE_LONG, 0, 0L);
+        long outRow = 0;
+        long elementBase = 0;
+        for (ListArray la : listChunks) {
+            long localRows = la.length();
+            Array localOffsets = la.offsets();
+            for (long i = 0; i < localRows; i++) {
+                long localEnd = readOffset(localOffsets, i + 1);
+                offsets.setAtIndex(VortexFormat.LE_LONG, outRow + i + 1, elementBase + localEnd);
+            }
+            outRow += localRows;
+            elementBase += readOffset(localOffsets, localRows);
+        }
+        Array offsetsArray = new MaterializedLongArray(DType.I64, totalRows + 1, offsets.asReadOnly());
+        return new ListArray(dtype, totalRows, combinedElements, offsetsArray);
+    }
+
+    private static long sumLengths(List<Array> arrays) {
+        long total = 0;
+        for (Array a : arrays) {
+            total += a.length();
+        }
+        return total;
+    }
+
+    /// Reads offset `idx` from a list offsets array as a non-negative long, widening whatever integer
+    /// ptype the encoder chose (it picks the narrowest that fits the max offset).
+    ///
+    /// @param offsets the offsets array
+    /// @param idx     the offset index to read
+    /// @return the offset value as a long
+    private static long readOffset(Array offsets, long idx) {
+        return switch (offsets) {
+            case LongArray la -> la.getLong(idx);
+            case IntArray ia -> Integer.toUnsignedLong(ia.getInt(idx));
+            case ShortArray sa -> sa.getInt(idx);
+            case ByteArray ba -> ba.getInt(idx);
+            default -> throw new VortexException("unexpected list offsets type: "
+                    + offsets.getClass().getSimpleName());
+        };
+    }
+
+    /// Concatenates the per-chunk validity of masked chunks into one row-level bitmap, or returns
+    /// `null` when no chunk is nullable. Unmasked chunks (and masked chunks with a `null` validity,
+    /// which mean all-valid) contribute all-valid rows; an entirely-null [NullArray] chunk (#269)
+    /// contributes all-invalid rows.
+    ///
+    /// @param chunkArrays the decoded per-chunk arrays, in row order
+    /// @param totalRows    the total logical row count across all chunks
+    /// @param arena        allocator for the combined bitmap
+    /// @return a bit-packed row-validity [BoolArray] of `totalRows` bits, or `null` if all valid
+    private static BoolArray combineChunkValidity(List<Array> chunkArrays, long totalRows,
+            SegmentAllocator arena) {
+        boolean anyNullable = false;
+        for (Array chunk : chunkArrays) {
+            if ((chunk instanceof MaskedArray m && m.validity() != null) || chunk instanceof NullArray) {
+                anyNullable = true;
+                break;
+            }
+        }
+        if (!anyNullable) {
+            return null;
+        }
+        MemorySegment bits = arena.allocate((totalRows + 7) >>> 3);
+        long row = 0;
+        for (Array chunk : chunkArrays) {
+            long chunkLen = chunk.length();
+            if (!(chunk instanceof NullArray)) {
+                BoolArray validity = chunk instanceof MaskedArray m ? m.validity() : null;
+                for (long i = 0; i < chunkLen; i++) {
+                    if (validity == null || validity.getBoolean(i)) {
+                        long globalRow = row + i;
+                        long byteIdx = globalRow >>> 3;
+                        byte cur = bits.get(ValueLayout.JAVA_BYTE, byteIdx);
+                        bits.set(ValueLayout.JAVA_BYTE, byteIdx,
+                                (byte) ((cur & 0xff) | (1 << (globalRow & 7))));
+                    }
+                }
+            }
+            row += chunkLen;
+        }
+        return new MaterializedBoolArray(DType.BOOL, totalRows, bits.asReadOnly());
+    }
+}
diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java
index 36a9ccd3..bcae7dce 100644
--- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java
+++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java
@@ -442,7 +442,9 @@ private long dictReadOff(long i) {
     record ChunkedMode(DType dtype, long length, VarBinArray[] children, long[] offsets)
             implements VarBinArray {
 
-        /// Builds a `ChunkedMode` from a list of chunk arrays.
+        /// Builds a `ChunkedMode` from a list of chunk arrays that are already all typed
+        /// [VarBinArray]s (no all-null [NullArray] chunks). Used by [#limited(long)],
+        /// whose children are always concrete `VarBinArray`s.
         ///
         /// @param dtype     logical element type
         /// @param totalRows expected total row count
@@ -451,6 +453,28 @@ record ChunkedMode(DType dtype, long length, VarBinArray[] children, long[] offs
         /// @throws VortexException on empty input, non-[VarBinArray] chunks, or row-count mismatch
         public static ChunkedMode of(DType dtype, long totalRows,
                 java.util.List<? extends Array> chunks) {
+            return of(dtype, totalRows, chunks, null);
+        }
+
+        /// Builds a `ChunkedMode` from a list of chunk arrays.
+        ///
+        /// An entirely-null chunk decodes to a [NullArray] rather than a [VarBinArray]
+        /// (e.g. a `vortex.null` flat, or `vortex.constant` with a null scalar, #269).
+        /// Such a chunk is materialized into an all-null [OffsetMode] of the same row
+        /// count — every row zero-length, all offsets zero — so the chunked column keeps
+        /// a uniform `VarBinArray` shape. Row-level nullability is preserved separately by
+        /// the caller's validity bitmap.
+        ///
+        /// @param dtype     logical element type
+        /// @param totalRows expected total row count
+        /// @param chunks    non-empty list of chunk arrays; each a [VarBinArray] or [NullArray]
+        /// @param arena     allocator for the offsets segment of a materialized null chunk;
+        ///                  may be `null` only when no chunk is a [NullArray]
+        /// @return a new `ChunkedMode`
+        /// @throws VortexException on empty input, non-`VarBinArray`/`NullArray` chunks,
+        ///                         or row-count mismatch
+        public static ChunkedMode of(DType dtype, long totalRows,
+                java.util.List<? extends Array> chunks, SegmentAllocator arena) {
             if (chunks.isEmpty()) {
                 throw new VortexException("VarBinArray.ChunkedMode: empty chunk list");
             }
@@ -461,6 +485,12 @@ public static ChunkedMode of(DType dtype, long totalRows,
                     java.util.Collections.addAll(typed, nested.children);
                 } else if (data instanceof VarBinArray vb) {
                     typed.add(vb);
+                } else if (data instanceof NullArray na) {
+                    if (arena == null) {
+                        throw new VortexException(
+                                "VarBinArray.ChunkedMode: null chunk requires an allocator");
+                    }
+                    typed.add(allNull(dtype, na.length(), arena));
                 } else {
                     throw new VortexException("VarBinArray.ChunkedMode: chunk is not a VarBinArray: "
                             + data.getClass().getSimpleName());
@@ -477,6 +507,19 @@ public static ChunkedMode of(DType dtype, long totalRows,
             return new ChunkedMode(dtype, totalRows, typed.toArray(VarBinArray[]::new), off);
         }
 
+        /// Builds an all-null [OffsetMode] of `n` rows: an empty bytes segment and an
+        /// offsets segment of `n + 1` zeros, so every row is zero-length. Row nullability
+        /// is carried by the caller's validity bitmap, not the byte data.
+        ///
+        /// @param dtype logical element type (Utf8 or Binary)
+        /// @param n     number of all-null rows
+        /// @param arena allocator for the offsets segment
+        /// @return an [OffsetMode] with `n` zero-length rows
+        private static OffsetMode allNull(DType dtype, long n, SegmentAllocator arena) {
+            MemorySegment offsets = arena.allocate((n + 1) * Long.BYTES, Long.BYTES);
+            return new OffsetMode(dtype, n, MemorySegment.NULL, offsets, PType.I64);
+        }
+
         private int findChunk(long i) {
             int hit = java.util.Arrays.binarySearch(offsets, i);
             int idx = hit >= 0 ? hit : -hit - 2;
diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ChunkedLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ChunkedLayoutDecoder.java
index a39ff081..ce7ff504 100644
--- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ChunkedLayoutDecoder.java
+++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ChunkedLayoutDecoder.java
@@ -4,22 +4,9 @@
 import io.github.dfa1.vortex.core.model.DType;
 import io.github.dfa1.vortex.core.model.EncodingId;
 import io.github.dfa1.vortex.core.model.LayoutId;
-import io.github.dfa1.vortex.core.model.PType;
 import io.github.dfa1.vortex.reader.array.Array;
-import io.github.dfa1.vortex.reader.array.BoolArray;
-import io.github.dfa1.vortex.reader.array.ChunkedBoolArray;
-import io.github.dfa1.vortex.reader.array.ChunkedByteArray;
-import io.github.dfa1.vortex.reader.array.ChunkedDoubleArray;
-import io.github.dfa1.vortex.reader.array.ChunkedFloatArray;
-import io.github.dfa1.vortex.reader.array.ChunkedIntArray;
-import io.github.dfa1.vortex.reader.array.ChunkedLongArray;
-import io.github.dfa1.vortex.reader.array.ChunkedShortArray;
-import io.github.dfa1.vortex.reader.array.MaskedArray;
-import io.github.dfa1.vortex.reader.array.MaterializedBoolArray;
-import io.github.dfa1.vortex.reader.array.VarBinArray;
+import io.github.dfa1.vortex.reader.array.ChunkedArrayCombiner;
 
-import java.lang.foreign.MemorySegment;
-import java.lang.foreign.SegmentAllocator;
 import java.lang.foreign.ValueLayout;
 import java.util.ArrayList;
 import java.util.List;
@@ -41,46 +28,6 @@ public Array decode(LayoutDecodeContext ctx, Layout layout, DType dtype) {
         return decodeChunkedLayout(ctx, flats, dtype, layout.rowCount());
     }
 
-    /// Concatenates the per-chunk validity of masked chunks into one row-level bitmap, or returns
-    /// `null` when no chunk is nullable. Unmasked chunks (and masked chunks with a `null` validity,
-    /// which mean all-valid) contribute all-valid rows. This preserves nullability that the
-    /// `ChunkedXxxArray.of` constructors deliberately drop when flattening masked chunks — needed so
-    /// a nullable chunked column (e.g. the codes of a nullable global-dict column) keeps its nulls.
-    ///
-    /// @param chunkArrays the decoded per-chunk arrays, in row order
-    /// @param totalRows    the total logical row count across all chunks
-    /// @param arena        allocator for the combined bitmap
-    /// @return a bit-packed row-validity [BoolArray] of `totalRows` bits, or `null` if all valid
-    private static BoolArray combineChunkValidity(List<Array> chunkArrays, long totalRows,
-            SegmentAllocator arena) {
-        boolean anyMasked = false;
-        for (Array chunk : chunkArrays) {
-            if (chunk instanceof MaskedArray m && m.validity() != null) {
-                anyMasked = true;
-                break;
-            }
-        }
-        if (!anyMasked) {
-            return null;
-        }
-        MemorySegment bits = arena.allocate((totalRows + 7) >>> 3);
-        long row = 0;
-        for (Array chunk : chunkArrays) {
-            long chunkLen = chunk.length();
-            BoolArray validity = chunk instanceof MaskedArray m ? m.validity() : null;
-            for (long i = 0; i < chunkLen; i++) {
-                if (validity == null || validity.getBoolean(i)) {
-                    long globalRow = row + i;
-                    long byteIdx = globalRow >>> 3;
-                    byte cur = bits.get(ValueLayout.JAVA_BYTE, byteIdx);
-                    bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) ((cur & 0xff) | (1 << (globalRow & 7))));
-                }
-            }
-            row += chunkLen;
-        }
-        return new MaterializedBoolArray(DType.BOOL, totalRows, bits.asReadOnly());
-    }
-
     /// Flattens a layout subtree into its ordered flat (and dict) leaves. A private copy of
     /// `ScanIterator.collectFlats`: the scan keeps its own for chunk-shape planning, while the
     /// chunked decoder needs the same flattening to build the leaf array list.
@@ -117,36 +64,18 @@ private static Array decodeChunkedLayout(LayoutDecodeContext ctx, List<Layout> f
         if (flats.size() == 1) {
             return ctx.decodeChild(flats.getFirst(), dtype);
         }
-        // ADR 0012: every primitive ptype gets the zero-copy ChunkedXxxArray shape.
-        // The concat path is gone.
+        // ADR 0012: every ptype gets the zero-copy ChunkedXxxArray shape. The concat path is gone.
         var chunkArrays = new ArrayList<Array>(flats.size());
         for (Layout flat : flats) {
             // Registry dispatch, not a direct decodeFlat call — a custom decoder registered for
             // a leaf's layout id must be honored under a chunked parent too.
             chunkArrays.add(ctx.decodeChild(flat, dtype));
         }
-        Array data;
-        if (dtype instanceof DType.Bool) {
-            data = ChunkedBoolArray.of(dtype, totalRows, chunkArrays);
-        } else if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) {
-            data = VarBinArray.ChunkedMode.of(dtype, totalRows, chunkArrays);
-        } else {
-            PType ptype = ((DType.Primitive) dtype).ptype();
-            data = switch (ptype) {
-                case I64, U64 -> ChunkedLongArray.of(dtype, totalRows, chunkArrays);
-                case I32, U32 -> ChunkedIntArray.of(dtype, totalRows, chunkArrays);
-                case F64 -> ChunkedDoubleArray.of(dtype, totalRows, chunkArrays);
-                case F32 -> ChunkedFloatArray.of(dtype, totalRows, chunkArrays);
-                case I16, U16 -> ChunkedShortArray.of(dtype, totalRows, chunkArrays);
-                case I8, U8 -> ChunkedByteArray.of(dtype, totalRows, chunkArrays);
-                default -> throw new VortexException("unsupported ptype for chunked layout: " + ptype);
-            };
-        }
-        // The ChunkedXxxArray.of constructors flatten masked chunks to their inner data, dropping
-        // validity. Recover per-chunk nullability by concatenating chunk validity into one bitmap and
-        // re-wrapping — otherwise a nullable chunked column (e.g. nullable global-dict codes) loses
-        // its nulls when it spans more than one chunk.
-        BoolArray validity = combineChunkValidity(chunkArrays, totalRows, ctx.arena());
-        return validity != null ? new MaskedArray(data, validity) : data;
+        // ChunkedArrayCombiner dispatches on dtype and fails with a VortexException — never a raw
+        // ClassCastException — for any dtype without a chunked shape (issue #268). It also stitches
+        // list columns into a single ListArray and recovers the per-chunk validity that the
+        // ChunkedXxxArray.of constructors drop when flattening masked chunks, so a nullable chunked
+        // column keeps its nulls.
+        return ChunkedArrayCombiner.combine(dtype, totalRows, chunkArrays, ctx.arena());
     }
 }
diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombinerTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombinerTest.java
new file mode 100644
index 00000000..1934d6e8
--- /dev/null
+++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/ChunkedArrayCombinerTest.java
@@ -0,0 +1,124 @@
+package io.github.dfa1.vortex.reader.array;
+
+import io.github.dfa1.vortex.core.error.VortexException;
+import io.github.dfa1.vortex.core.model.DType;
+import org.junit.jupiter.api.Test;
+
+import java.lang.foreign.Arena;
+import java.util.List;
+
+import static io.github.dfa1.vortex.reader.array.TestArrays.ints;
+import static io.github.dfa1.vortex.reader.array.TestArrays.longs;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/// Covers [ChunkedArrayCombiner], the dtype-driven stitcher for multi-chunk columns. The focus is
+/// the list branch and the security guard for issue #268: a chunked `list<int>` column previously hit
+/// a blind `(DType.Primitive) dtype` cast in `ChunkedLayoutDecoder`, throwing a raw
+/// `ClassCastException` instead of a handled [VortexException].
+class ChunkedArrayCombinerTest {
+
+    private static final DType.List LIST_OF_INT = new DType.List(DType.I32, false);
+
+    @Test
+    void stitchesMultiChunkListIntoOneListArray() {
+        // Given — two chunks of list<int>, each with offsets that restart at 0.
+        // chunk0: [ [10,11], [12] ]   chunk1: [ [20,21,22], [] ]
+        ListArray chunk0 = new ListArray(LIST_OF_INT, 2, ints(10, 11, 12), longs(0L, 2L, 3L));
+        ListArray chunk1 = new ListArray(LIST_OF_INT, 2, ints(20, 21, 22), longs(0L, 3L, 3L));
+
+        // When
+        try (Arena arena = Arena.ofConfined()) {
+            Array result = ChunkedArrayCombiner.combine(LIST_OF_INT, 4, List.of(chunk0, chunk1), arena);
+
+            // Then — a single ListArray with cumulative offsets and concatenated elements. The
+            // second chunk's offsets are shifted by chunk0's 3 elements, so list2 spans [3,6).
+            assertThat(result).isInstanceOf(ListArray.class);
+            ListArray list = (ListArray) result;
+            assertThat(list.length()).isEqualTo(4);
+            LongArray offsets = (LongArray) list.offsets();
+            assertThat(offsets.length()).isEqualTo(5);
+            assertThat(readAll(offsets, 5)).containsExactly(0L, 2L, 3L, 6L, 6L);
+
+            IntArray elements = (IntArray) list.elements();
+            assertThat(elements.length()).isEqualTo(6);
+            assertThat(elements.getInt(0)).isEqualTo(10);
+            assertThat(elements.getInt(3)).isEqualTo(20);
+            assertThat(elements.getInt(5)).isEqualTo(22);
+        }
+    }
+
+    @Test
+    void nullableListChunksKeepTheirNulls() {
+        // Given — a nullable list<int> column across two chunks: chunk0 row 1 is null, chunk1 row 0
+        // is null. The ChunkedXxxArray.of flatteners drop per-chunk validity, so the combiner must
+        // rebuild it into one row-level bitmap wrapped in a MaskedArray (else the nulls vanish when
+        // the column spans more than one chunk).
+        ListArray data0 = new ListArray(LIST_OF_INT, 2, ints(10, 11), longs(0L, 2L, 2L));
+        ListArray data1 = new ListArray(LIST_OF_INT, 2, ints(20), longs(0L, 0L, 1L));
+        MaskedArray chunk0 = new MaskedArray(data0, TestArrays.bools(true, false));
+        MaskedArray chunk1 = new MaskedArray(data1, TestArrays.bools(false, true));
+
+        // When
+        try (Arena arena = Arena.ofConfined()) {
+            Array result = ChunkedArrayCombiner.combine(LIST_OF_INT, 4, List.of(chunk0, chunk1), arena);
+
+            // Then — the combined validity marks rows 1 and 2 null.
+            assertThat(result).isInstanceOf(MaskedArray.class);
+            MaskedArray masked = (MaskedArray) result;
+            assertThat(masked.inner()).isInstanceOf(ListArray.class);
+            assertThat(masked.isValid(0)).isTrue();
+            assertThat(masked.isValid(1)).isFalse();
+            assertThat(masked.isValid(2)).isFalse();
+            assertThat(masked.isValid(3)).isTrue();
+        }
+    }
+
+    @Test
+    void emptyChunkListThrowsVortexException() {
+        // Given / When / Then
+        try (Arena arena = Arena.ofConfined()) {
+            assertThatThrownBy(() -> ChunkedArrayCombiner.combine(LIST_OF_INT, 0, List.of(), arena))
+                    .isInstanceOf(VortexException.class)
+                    .hasMessageContaining("empty chunk list");
+        }
+    }
+
+    @Test
+    void rowCountMismatchThrowsVortexException() {
+        // Given — the declared totalRows disagrees with the summed chunk lengths.
+        ListArray chunk = new ListArray(LIST_OF_INT, 2, ints(10, 11, 12), longs(0L, 2L, 3L));
+
+        // When / Then
+        try (Arena arena = Arena.ofConfined()) {
+            assertThatThrownBy(() -> ChunkedArrayCombiner.combine(LIST_OF_INT, 99, List.of(chunk, chunk), arena))
+                    .isInstanceOf(VortexException.class)
+                    .hasMessageContaining("expected 99");
+        }
+    }
+
+    @Test
+    void unsupportedDtypeThrowsVortexExceptionNotClassCastException() {
+        // Given — a dtype with no chunked shape (a nested struct). The pre-#268 code path blindly
+        // cast to DType.Primitive and threw a raw ClassCastException; the guard must be a
+        // VortexException instead.
+        DType.Struct struct = new DType.Struct(List.of(), List.of(), false);
+        Array chunk0 = ints(1);
+        Array chunk1 = ints(2);
+
+        // When / Then
+        try (Arena arena = Arena.ofConfined()) {
+            assertThatThrownBy(() -> ChunkedArrayCombiner.combine(struct, 2, List.of(chunk0, chunk1), arena))
+                    .isInstanceOf(VortexException.class)
+                    .hasMessageContaining("unsupported dtype for chunked layout");
+        }
+    }
+
+    private static long[] readAll(LongArray arr, int n) {
+        long[] out = new long[n];
+        for (int i = 0; i < n; i++) {
+            out[i] = arr.getLong(i);
+        }
+        return out;
+    }
+}
diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedModeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedModeTest.java
index 71e2ba83..31c00d6a 100644
--- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedModeTest.java
+++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinChunkedModeTest.java
@@ -55,6 +55,47 @@ void nonVarBinChunkRejected() {
             }
         }
 
+        @Test
+        void nullChunkMaterializesAsAllNullRun() {
+            try (Arena arena = Arena.ofConfined()) {
+                // Given — a chunked Utf8 column where the middle chunk is entirely null and
+                // decoded to NullArray (via vortex.null or a null-scalar vortex.constant, #269)
+                // rather than a VarBinArray. Before the fix ChunkedMode.of threw on it.
+                VarBinArray c0 = stringChunk(arena, "a", "b");
+                NullArray nullChunk = new NullArray(UTF8, 3);
+                VarBinArray c2 = stringChunk(arena, "z");
+
+                // When
+                VarBinArray.ChunkedMode result =
+                        VarBinArray.ChunkedMode.of(UTF8, 6, List.of(c0, nullChunk, c2), arena);
+
+                // Then — the null chunk contributes 3 zero-length rows, keeping row alignment
+                assertThat(result.length()).isEqualTo(6);
+                assertThat(result.children()).hasSize(3);
+                assertThat(result.getString(0)).isEqualTo("a");
+                assertThat(result.getString(1)).isEqualTo("b");
+                assertThat(result.getByteLength(2)).isZero();
+                assertThat(result.getByteLength(3)).isZero();
+                assertThat(result.getByteLength(4)).isZero();
+                // Accessing a null row must not crash even though the bytes segment is NULL:
+                // a zero-length copy from MemorySegment.NULL is well-defined.
+                assertThat(result.getBytes(3)).isEmpty();
+                assertThat(result.getString(3)).isEmpty();
+                assertThat(result.getString(5)).isEqualTo("z");
+            }
+        }
+
+        @Test
+        void nullChunkWithoutAllocatorRejected() {
+            // Given — a NullArray chunk but no allocator to materialize it
+            NullArray nullChunk = new NullArray(UTF8, 2);
+
+            // When / Then
+            assertThatThrownBy(
+                    () -> VarBinArray.ChunkedMode.of(UTF8, 2, List.of(nullChunk), null))
+                    .isInstanceOf(VortexException.class);
+        }
+
         @Test
         void nestedChunkedFlattens() {
             try (Arena arena = Arena.ofConfined()) {

From 937eb4967988844c5e8abcfff7140b7d69215ddb Mon Sep 17 00:00:00 2001
From: Davide Angelocola <davide.angelocola@gmail.com>
Date: Tue, 14 Jul 2026 07:37:55 +0200
Subject: [PATCH 11/11] raincloud: triage 34 slugs, fix VARIANT oracle crash,
 matrix at 158 ok
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Oracle guard: hardwood's NestedRowReader.getStruct() mis-casts a
  VARIANT-shredded group (metadata/value/typed_value) to its plain
  Struct FieldDesc and throws ClassCastException — an oracle bug, not
  a vortex-java gap. RaincloudConformanceIntegrationTest now aborts on
  group.isVariant() before hitting it, same as the existing isMap()
  guard (unblocks countries-of-the-world, which regressed to a hard
  failure once the corpus was rebuilt with a VARIANT column).
- Triage: 32 untriaged slugs verified ok against the parquet oracle
  (13 large BI slugs + 19 smaller HF/benchmark slugs). squad-v2 and
  mnist flip from gap:268 to ok now that the previous commit's fix
  lands. oasst1 hits a different, new bug — a null Struct row renders
  as {"field":null,...} instead of an empty cell (#270, filed) — so it
  moves from gap:268 to gap:270 rather than to ok.

Matrix: 124 ok / 0 gaps / 123 untriaged -> 158 ok / 1 gap / 85 untriaged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../RaincloudConformanceIntegrationTest.java  |  6 ++
 .../resources/raincloud/expected-status.csv   | 70 +++++++++----------
 2 files changed, 41 insertions(+), 35 deletions(-)

diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java
index bb6d0168..222b5dd9 100644
--- a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java
+++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java
@@ -366,6 +366,12 @@ private static String oracleCell(SchemaNode node, RowReader rows, int fieldIndex
             if (group.isMap()) {
                 throw new TestAbortedException("oracle cannot format MAP column: " + group.name());
             }
+            // hardwood's own NestedRowReader.getStruct() mis-casts a VARIANT-shredded group
+            // (metadata/value/typed_value) to its plain-Struct FieldDesc and throws
+            // ClassCastException — an oracle bug, not a vortex-java gap. Abort before hitting it.
+            if (group.isVariant()) {
+                throw new TestAbortedException("oracle cannot format VARIANT column: " + group.name());
+            }
             return oracleJsonObject(group, rows.getStruct(fieldIndex));
         }
         SchemaNode.PrimitiveNode prim = (SchemaNode.PrimitiveNode) node;
diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv
index a99c092a..9999a406 100644
--- a/integration/src/test/resources/raincloud/expected-status.csv
+++ b/integration/src/test/resources/raincloud/expected-status.csv
@@ -29,11 +29,11 @@ aya-dataset,ok
 bank-account-fraud-dataset-neurips-2022,missing_auth
 basketball,untriaged
 behavioral-risk-factor-surveillance-system,ok
-beir-msmarco,untriaged
+beir-msmarco,ok
 bi-arade,ok
 bi-bimbo,ok
 bi-citymaxcapita,ok
-bi-cmsprovider,untriaged
+bi-cmsprovider,ok
 bi-commongovernment,untriaged
 bi-corporations,ok
 bi-eixo,ok
@@ -43,28 +43,28 @@ bi-generico,untriaged
 bi-hashtags,ok
 bi-hatred,ok
 bi-iglocations1,ok
-bi-iglocations2,untriaged
+bi-iglocations2,ok
 bi-iublibrary,ok
-bi-medicare1,untriaged
-bi-medicare2,untriaged
+bi-medicare1,ok
+bi-medicare2,ok
 bi-medicare3,ok
 bi-medpayment1,ok
 bi-medpayment2,ok
-bi-mlb,untriaged
-bi-motos,untriaged
+bi-mlb,ok
+bi-motos,ok
 bi-mulheresmil,ok
-bi-nyc,untriaged
+bi-nyc,ok
 bi-pancreactomy1,ok
-bi-pancreactomy2,untriaged
+bi-pancreactomy2,ok
 bi-physicians,ok
 bi-provider,untriaged
 bi-realestate1,untriaged
 bi-realestate2,untriaged
-bi-redfin1,untriaged
-bi-redfin2,untriaged
-bi-redfin3,untriaged
+bi-redfin1,ok
+bi-redfin2,ok
+bi-redfin3,ok
 bi-redfin4,ok
-bi-rentabilidad,untriaged
+bi-rentabilidad,ok
 bi-romance,ok
 bi-salariesfrance,untriaged
 bi-tablerosistemapenal,ok
@@ -76,7 +76,7 @@ bi-uberlandia,ok
 bi-uscensus,untriaged
 bi-wins,untriaged
 bi-yalelanguages,ok
-bigscience-p3-eval-subset,untriaged
+bigscience-p3-eval-subset,ok
 building-permit-applications-data,untriaged
 c4-en-validation,untriaged
 california-housing-prices,ok
@@ -86,14 +86,14 @@ clickbench-hits,untriaged
 cnn-dailymail,untriaged
 codeparrot-clean-valid,untriaged
 cohere-wikipedia-simple-embed,untriaged
-coig,untriaged
+coig,ok
 coronahack-chest-xraydataset,untriaged
 cosmopedia-stanford,untriaged
 countries-of-the-world,ok
 covid-world-vaccination-progress,ok
 covid19-data-from-john-hopkins-university,untriaged
 crimes-in-boston,untriaged
-databricks-dolly-15k,untriaged
+databricks-dolly-15k,ok
 dbpedia-embeddings,untriaged
 diabetes-health-indicators-dataset,ok
 disease-symptom-description-dataset,untriaged
@@ -103,13 +103,13 @@ emotions-dataset-for-nlp,ok
 fhv_tripdata_2025,ok
 fhvhv_tripdata_2025,untriaged
 finemath-4plus,untriaged
-finepdfs-en-test,untriaged
+finepdfs-en-test,ok
 finetranslations-swedish,untriaged
 fineweb-2-swedish,untriaged
 fineweb-sample-10bt,untriaged
 fitbit,untriaged
 formula-1-world-championship-1950-2020,untriaged
-frames-benchmark,untriaged
+frames-benchmark,ok
 ghcn-daily,untriaged
 glass,ok
 global-fossil-co2-emissions-by-country-2002-2022,untriaged
@@ -119,16 +119,16 @@ glove-6b-50d,ok
 goodbooks-10k,ok
 google-cluster-trace-2011-machine-events,ok
 green_tripdata_2025,ok
-gsm8k,untriaged
+gsm8k,ok
 gtsrb-german-traffic-sign,ok
 hacker-news,untriaged
 heart-disease-health-indicators-dataset,ok
-hellaswag,untriaged
-helpsteer2,untriaged
+hellaswag,ok
+helpsteer2,ok
 hotel-booking-demand,ok
 hotpotqa-fullwiki,untriaged
 housing-prices-dataset,untriaged
-humaneval,untriaged
+humaneval,ok
 insurance,ok
 international-football-results-from-1872-to-2017,ok
 iowa-liquor-sales,untriaged
@@ -139,24 +139,24 @@ librispeech-test-clean,untriaged
 loan-default-dataset,untriaged
 lung-cancer,ok
 marketing-data,ok
-mbpp,untriaged
-medmcqa,untriaged
+mbpp,ok
+medmcqa,ok
 mlcourse,untriaged
-mmlu,untriaged
-mmlu-pro,untriaged
-mmmlu,untriaged
+mmlu,ok
+mmlu-pro,ok
+mmmlu,ok
 mmmu,untriaged
-mnist,untriaged
+mnist,ok
 movies,ok
 new-york-city-airbnb-open-data,untriaged
-news-headlines-dataset-for-sarcasm-detection,untriaged
+news-headlines-dataset-for-sarcasm-detection,ok
 nips-papers,ok
-no-robots,untriaged
+no-robots,ok
 nyc-311,untriaged
 nyc-parking-tickets,untriaged
 nyc-property-sales,ok
 nypd-complaints,untriaged
-oasst1,untriaged
+oasst1,gap:270
 open-food-facts,untriaged
 openlibrary-authors,untriaged
 openlibrary-editions,untriaged
@@ -178,21 +178,21 @@ palmer-archipelago-antarctica-penguin-data,ok
 peoples-speech-clean-validation,untriaged
 personal-key-indicators-of-heart-disease,ok
 pleias-synth,untriaged
-pubmedqa-labeled,untriaged
+pubmedqa-labeled,ok
 sf-salaries,ok
 slimpajama-6b,untriaged
-squad-v2,untriaged
+squad-v2,ok
 stack-overflow-2018-developer-survey,ok
 stackoverflow-badges,untriaged
 stackoverflow-postlinks,ok
 stackoverflow-posts,untriaged
 stackoverflow-tags,ok
 stackoverflow-users,untriaged
-synthetic-text-to-sql,untriaged
+synthetic-text-to-sql,ok
 temperature-change,ok
 tinystories,untriaged
 titanic-dataset,untriaged
-truthfulqa-mc,untriaged
+truthfulqa-mc,ok
 uber-pickups-in-new-york-city,ok
 uci-abalone,ok
 uci-adult,ok