Fix Misaligned buffer on read by reordering OnPair buffers

Root cause: Vortex's flat-layout segment writer aligns each segment to
the alignment of its *first* buffer only. With the old buffer order

  [dict_bytes, dict_offsets, codes, codes_offsets]

`dict_bytes` is variable-length and has no alignment requirement, so
the segment was written u8-aligned. The next buffer (`dict_offsets`)
was a u32 array but ended up at an offset that was only u8-aligned in
the file, and on read `PrimitiveArray<u32>::deserialize` rejected it
with `Misaligned buffer cannot be used to build PrimitiveArray of u32`.

Single-column tests happened to pass because typical OnPair
dictionaries are coincidentally a multiple of 4 bytes; ClickBench's
wide string tables (and TPC-H's `supplier` post-encoding) hit the bad
case.

New buffer order:

    Buffer 0  dict_offsets   u32[]  ← segment alignment = 4
    Buffer 1  codes_offsets  u32[]  ← length already 4-multiple
    Buffer 2  codes          u16[]  ← starts at 4-aligned offset, OK for u16
    Buffer 3  dict_bytes     u8[]   ← variable length, no alignment needed

Each buffer's natural length is a multiple of its alignment, so every
buffer inside the segment stays correctly aligned. The 16-byte
over-copy padding on `dict_bytes` still applies for the decoder.

Verified
* `cargo test -p vortex-onpair -p vortex-btrblocks -p vortex-file`
  all green (5 new file-roundtrip tests pass, including a new
  `odd_dict_length_alignment` test specifically exercising the
  previously-broken case).
* `datafusion-bench tpch --opt scale-factor=0.01 --formats vortex
   --queries 1,2,3,6 --iterations 1` runs all four queries
  successfully end-to-end (Parquet → Vortex with OnPair → DataFusion).

Signed-off-by: Claude <noreply@anthropic.com>

Author: claude

encodings/onpair/src/array.rs

@@ -59,16 +59,21 @@ pub const DEFAULT_BITS: u32 = 12;
 ///
 /// On disk the layout is:
 ///
-/// * Buffer 0 — `dict_bytes`: dictionary blob built by the C++ trainer.
-/// * Buffer 1 — `dict_offsets`: `dict_size + 1` u32 offsets into `dict_bytes`,
-///   stored as raw little-endian bytes.
-/// * Buffer 2 — `codes`: per-token `u16` ids, stored as raw little-endian
-///   bytes. Each value only uses its low `bits` bits, but we keep the u16
-///   width on disk so the decode loop is a straight indexed lookup without
-///   bit-unpacking. Downstream compaction can still re-encode this buffer
-///   externally.
-/// * Buffer 3 — `codes_offsets`: `num_rows + 1` u32 offsets into `codes`,
-///   stored as raw little-endian bytes.
+/// * Buffer 0 — `dict_offsets`: `dict_size + 1` u32 offsets into `dict_bytes`,
+///   stored as raw little-endian bytes. **First so the segment-level
+///   alignment is u32 (4 bytes).**
+/// * Buffer 1 — `codes_offsets`: `num_rows + 1` u32 offsets into `codes`.
+///   Lengths of buffers 0 and 1 are both multiples of 4, so buffer 2 starts
+///   at a 4-aligned (and thus 2-aligned) offset within the segment.
+/// * Buffer 2 — `codes`: per-token `u16` ids. Each value only uses its low
+///   `bits` bits, but we keep the u16 width on disk so the decode loop is
+///   a straight indexed lookup without bit-unpacking. Downstream compaction
+///   can still re-encode this buffer externally.
+/// * Buffer 3 — `dict_bytes`: dictionary blob built by the C++ trainer,
+///   padded with [`MAX_TOKEN_SIZE`][crate::MAX_TOKEN_SIZE] trailing zero
+///   bytes so the over-copy decoder can safely read 16 bytes past the last
+///   token. **Last because its length is variable and it has no alignment
+///   requirement; any padding pressure on later buffers is moot.**
 /// * Slot 0   — `uncompressed_lengths`: `PrimitiveArray<integer>`.
 /// * Slot 1   — optional validity child.
 ///
@@ -103,10 +108,19 @@ pub(crate) const NUM_SLOTS: usize = 2;
 pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = ["uncompressed_lengths", "validity"];
 
 /// Buffer indices.
-pub(crate) const DICT_BYTES_BUF: usize = 0;
-pub(crate) const DICT_OFFSETS_BUF: usize = 1;
+///
+/// Order matters for on-disk alignment: the Vortex flat-segment writer
+/// aligns each segment to the first buffer's alignment only, so we put the
+/// strictest-alignment buffers first. Both `u32` offsets buffers have
+/// length-multiple-of-4 by construction, and `codes` has
+/// length-multiple-of-2; that means every later buffer's relative offset
+/// inside the segment stays aligned to its own type's requirement. The
+/// variable-length `dict_bytes` blob (no alignment) is last so nothing
+/// downstream can be tripped up by its length.
+pub(crate) const DICT_OFFSETS_BUF: usize = 0;
+pub(crate) const CODES_OFFSETS_BUF: usize = 1;
 pub(crate) const CODES_BUF: usize = 2;
-pub(crate) const CODES_OFFSETS_BUF: usize = 3;
+pub(crate) const DICT_BYTES_BUF: usize = 3;
 
 /// Inner data for an OnPair-encoded array.
 ///
@@ -388,20 +402,20 @@ impl VTable for OnPair {
 
     fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle {
         match idx {
-            DICT_BYTES_BUF => array.dict_bytes_handle().clone(),
             DICT_OFFSETS_BUF => array.dict_offsets_handle().clone(),
-            CODES_BUF => array.codes_handle().clone(),
             CODES_OFFSETS_BUF => array.codes_offsets_handle().clone(),
+            CODES_BUF => array.codes_handle().clone(),
+            DICT_BYTES_BUF => array.dict_bytes_handle().clone(),
             _ => vortex_panic!("OnPairArray buffer index {idx} out of bounds"),
         }
     }
 
     fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option<String> {
         match idx {
-            DICT_BYTES_BUF => Some("dict_bytes".to_string()),
             DICT_OFFSETS_BUF => Some("dict_offsets".to_string()),
-            CODES_BUF => Some("codes".to_string()),
             CODES_OFFSETS_BUF => Some("codes_offsets".to_string()),
+            CODES_BUF => Some("codes".to_string()),
+            DICT_BYTES_BUF => Some("dict_bytes".to_string()),
             _ => vortex_panic!("OnPairArray buffer_name index {idx} out of bounds"),
         }
     }

vortex-file/tests/test_onpair_string_roundtrip.rs

@@ -301,6 +301,55 @@ async fn tpch_supplier_shape() {
     assert_eq!(row, n);
 }
 
+/// 30 short fixed strings where the dictionary blob length is unlikely to
+/// be a multiple of 4. Earlier buffer orderings (dict_bytes first) tripped
+/// the segment writer's first-buffer-only alignment, surfacing
+/// `Misaligned buffer cannot be used to build PrimitiveArray of u32` on
+/// read.
+#[tokio::test]
+async fn odd_dict_length_alignment() {
+    let words: &[&str] = &[
+        "a", "bb", "ccc", "dddd", "eeeee", "fffff", "ggggggg", "h", "ii", "jjj",
+    ];
+    let n = 20_000usize;
+    let strings: Vec<&str> = (0..n).map(|i| words[i % words.len()]).collect();
+    let str_array = VarBinViewArray::from_iter(
+        strings.iter().map(|s| Some(*s)),
+        DType::Utf8(Nullability::NonNullable),
+    )
+    .into_array();
+    let data = StructArray::new(
+        FieldNames::from(["w"]),
+        vec![str_array],
+        n,
+        Validity::NonNullable,
+    )
+    .into_array();
+
+    let chunks = write_and_read_back(data).await;
+    let mut row = 0;
+    for chunk in chunks {
+        let strct = chunk
+            .try_downcast::<vortex_array::arrays::Struct>()
+            .expect("Struct");
+        let mut ctx = SESSION.create_execution_ctx();
+        let s = strct
+            .unmasked_field(0)
+            .clone()
+            .execute::<VarBinViewArray>(&mut ctx)
+            .unwrap();
+        s.with_iterator(|iter| {
+            for b in iter {
+                assert_eq!(b, Some(strings[row].as_bytes()), "row {row}");
+                row += 1;
+            }
+            Ok::<_, vortex_error::VortexError>(())
+        })
+        .unwrap();
+    }
+    assert_eq!(row, n);
+}
+
 /// Mixed-shape strings: empty, short, very long, with a fair chunk of nulls
 /// — exercising the validity child + edge offsets.
 #[tokio::test]