Make OnPair the default string-fragmentation scheme + register globally

Two changes that together stop FSST from being the default and make
OnPair work end-to-end through the file writer + reader.

vortex-btrblocks
* Remove `FSSTScheme` from `ALL_SCHEMES`. The struct and `Scheme` impl
  stay in place so callers can opt back in via
  `BtrBlocksCompressorBuilder::with_new_scheme(&FSSTScheme)`; it just
  isn't in the default cascade anymore. OnPair fills the
  string-fragmentation slot.
* Tighten `only_cuda_compatible` to exclude OnPair (heavier toolchain
  dep) instead of FSST.
* Tests: drop the FSST-vs-OnPair tie-break test; add
  `test_onpair_compressed` (FSST-style corpus → OnPair) and
  `test_fsst_opt_in_still_works` (empty builder + with_new_scheme +
  FSSTScheme).

vortex-file
* New `onpair` Cargo feature (default-on, mirrors `vortex-btrblocks`'s)
  that pulls in `vortex-onpair` and registers `OnPair` in both
  `register_default_encodings` and `ALLOWED_ENCODINGS`. Without this
  the normalizer rejects vortex.onpair with "normalize forbids
  encoding (vortex.onpair)" when round-tripping a file. Consumers
  without a C++ toolchain can `default-features = false`.

CI / reproducibility
* Pin `onpair_cpp` to a full commit SHA in `cmake/onpair_pin.cmake`
  (was tracking `main`). CI's `FetchContent` step is now reproducible
  and won't break when upstream's main branch moves.

Tests: 109 across vortex-onpair, vortex-btrblocks, vortex-file; all
green. Clippy clean.

Signed-off-by: Claude <noreply@anthropic.com>

Author: claude

Cargo.lock

@@ -1,4 +1,5 @@
 # Pin of gargiulofrancesco/onpair_cpp consumed by FetchContent.
-# Bump both fields when updating.
+# Bump `ONPAIR_CPP_TAG` to a full commit SHA when updating — never use a
+# branch name in CI, otherwise builds become non-reproducible.
 set(ONPAIR_CPP_REPO "https://github.com/gargiulofrancesco/onpair_cpp.git")
-set(ONPAIR_CPP_TAG  "main")
+set(ONPAIR_CPP_TAG  "ae590713515c7bb7893e14a757b484545e5339c3")

encodings/onpair-sys/cmake/onpair_pin.cmake

@@ -53,7 +53,6 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
     // String schemes.
     ////////////////////////////////////////////////////////////////////////////////////////////////
     &string::StringDictScheme,
-    &string::FSSTScheme,
     #[cfg(feature = "onpair")]
     &string::OnPairScheme,
     &string::StringConstantScheme,
@@ -170,14 +169,20 @@ impl BtrBlocksCompressorBuilder {
     /// preserves the array buffer layout for zero-conversion GPU decompression. Without it,
     /// interleaved Zstd compression is used.
     pub fn only_cuda_compatible(self) -> Self {
-        let builder = self.exclude_schemes([
+        // String fragmentation schemes (OnPair, FSST) require host-side
+        // dictionary expansion at decode time, which is incompatible with
+        // pure-GPU decompression paths. Strip whichever string-fragment
+        // scheme is enabled by feature.
+        let mut excluded: Vec<SchemeId> = vec![
             integer::SparseScheme.id(),
             integer::IntRLEScheme.id(),
             float::FloatRLEScheme.id(),
             float::NullDominatedSparseScheme.id(),
             string::StringDictScheme.id(),
-            string::FSSTScheme.id(),
-        ]);
+        ];
+        #[cfg(feature = "onpair")]
+        excluded.push(string::OnPairScheme.id());
+        let builder = self.exclude_schemes(excluded);
 
         #[cfg(all(feature = "zstd", feature = "unstable_encodings"))]
         let builder = builder.with_new_scheme(&string::ZstdBuffersScheme);

vortex-btrblocks/src/builder.rs

@@ -41,17 +41,22 @@ use crate::Scheme;
 use crate::SchemeExt;
 
 /// FSST (Fast Static Symbol Table) compression.
+///
+/// Retained for callers that want to opt back in via
+/// [`BtrBlocksCompressorBuilder::with_new_scheme`]; it is **not** part of the
+/// default [`ALL_SCHEMES`] anymore — the default string-fragmentation slot is
+/// filled by [`OnPairScheme`].
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub struct FSSTScheme;
 
-/// OnPair short-string compression (dict-12, FSST-shape children).
+/// OnPair short-string compression (dict-12).
 ///
-/// Targets the same workload as FSST — large columns of short-to-medium
-/// strings with high lexical overlap — but uses a learned dictionary of
-/// frequent adjacent substrings and 12-bit codes. The codes / offsets /
-/// uncompressed-lengths children all flow through the cascading compressor
-/// the same way FSST's do, so any downstream bit-packing / FoR / etc. still
-/// applies.
+/// The default string-fragmentation scheme — targets large columns of
+/// short-to-medium strings with high lexical overlap, like URLs or log lines.
+/// Uses a learned dictionary of frequent adjacent substrings (built by the
+/// OnPair C++ trainer at compress time) and 12-bit token codes stored as a
+/// u16 child, with offsets / uncompressed-lengths flowing through the
+/// cascading compressor like any other primitive children.
 #[cfg(feature = "onpair")]
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub struct OnPairScheme;
@@ -546,11 +551,12 @@ mod scheme_selection_tests {
         );
     }
 
+    #[cfg(feature = "onpair")]
     #[test]
-    fn test_dictionary_string_scheme_compressed() -> VortexResult<()> {
+    fn test_onpair_compressed() -> VortexResult<()> {
         // Dictionary-style string corpus: high lexical overlap, short rows.
-        // FSST and OnPair both target this shape; the cascading compressor
-        // picks whichever samples better, so accept either.
+        // OnPair is the only string-fragmentation scheme in the default
+        // builder, so it should win the sample-based comparison.
         let mut strings = Vec::with_capacity(1000);
         for i in 0..1000 {
             strings.push(Some(format!(
@@ -561,14 +567,47 @@ mod scheme_selection_tests {
         let array_ref = array.into_array();
         let compressed = BtrBlocksCompressor::default()
             .compress(&array_ref, &mut SESSION.create_execution_ctx())?;
-        let is_fsst = compressed.is::<FSST>();
-        #[cfg(feature = "onpair")]
-        let is_onpair = compressed.is::<vortex_onpair::OnPair>();
-        #[cfg(not(feature = "onpair"))]
-        let is_onpair = false;
         assert!(
-            is_fsst || is_onpair,
-            "expected FSST or OnPair, got {}",
+            compressed.is::<vortex_onpair::OnPair>(),
+            "expected OnPair, got {}",
+            compressed.encoding_id()
+        );
+        Ok(())
+    }
+
+    /// FSST is no longer in the default scheme list, but `with_new_scheme`
+    /// still lets callers opt it back in.
+    #[test]
+    fn test_fsst_opt_in_still_works() -> VortexResult<()> {
+        use crate::BtrBlocksCompressorBuilder;
+        use crate::SchemeExt;
+        use crate::schemes::string::FSSTScheme;
+
+        // FSST must not be registered by default.
+        assert!(
+            !crate::ALL_SCHEMES.iter().any(|s| s.id() == FSSTScheme.id()),
+            "FSSTScheme should not be in ALL_SCHEMES anymore",
+        );
+
+        // ...but explicitly adding it back should still produce a compressor
+        // that returns an FSST array for FSST-favourable input. Start from an
+        // empty builder so the sample-based comparison can't pick OnPair.
+        let mut strings = Vec::with_capacity(1000);
+        for i in 0..1000 {
+            strings.push(Some(format!(
+                "this_is_a_common_prefix_with_some_variation_{i}_and_a_common_suffix_pattern"
+            )));
+        }
+        let array = VarBinViewArray::from_iter(strings, DType::Utf8(Nullability::NonNullable));
+        let array_ref = array.into_array();
+
+        let compressor = BtrBlocksCompressorBuilder::empty()
+            .with_new_scheme(&FSSTScheme)
+            .build();
+        let compressed = compressor.compress(&array_ref, &mut SESSION.create_execution_ctx())?;
+        assert!(
+            compressed.is::<FSST>(),
+            "expected FSST when only FSSTScheme is registered, got {}",
             compressed.encoding_id()
         );
         Ok(())

vortex-btrblocks/src/schemes/string.rs

@@ -46,6 +46,7 @@ vortex-io = { workspace = true }
 vortex-layout = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-metrics = { workspace = true }
+vortex-onpair = { workspace = true, optional = true }
 vortex-pco = { workspace = true }
 vortex-runend = { workspace = true }
 vortex-scan = { workspace = true }
@@ -68,7 +69,9 @@ vortex-scan = { workspace = true }
 workspace = true
 
 [features]
+default = ["onpair"]
 object_store = ["dep:object_store", "vortex-io/object_store", "tokio"]
+onpair = ["dep:vortex-onpair", "vortex-btrblocks/onpair"]
 tokio = [
     "dep:tokio",
     "vortex-error/tokio",

vortex-file/Cargo.toml

@@ -115,6 +115,8 @@ use vortex_array::arrays::patched::use_experimental_patches;
 use vortex_array::session::ArraySessionExt;
 use vortex_bytebool::ByteBool;
 use vortex_fsst::FSST;
+#[cfg(feature = "onpair")]
+use vortex_onpair::OnPair;
 use vortex_pco::Pco;
 use vortex_session::VortexSession;
 use vortex_sparse::Sparse;
@@ -163,6 +165,8 @@ pub fn register_default_encodings(session: &VortexSession) {
         arrays.register(ByteBool);
         arrays.register(Dict);
         arrays.register(FSST);
+        #[cfg(feature = "onpair")]
+        arrays.register(OnPair);
         arrays.register(Pco);
         arrays.register(Sparse);
         arrays.register(ZigZag);

vortex-file/src/lib.rs

@@ -52,6 +52,8 @@ use vortex_layout::layouts::repartition::RepartitionWriterOptions;
 use vortex_layout::layouts::table::TableStrategy;
 use vortex_layout::layouts::zoned::writer::ZonedLayoutOptions;
 use vortex_layout::layouts::zoned::writer::ZonedStrategy;
+#[cfg(feature = "onpair")]
+use vortex_onpair::OnPair;
 use vortex_pco::Pco;
 use vortex_runend::RunEnd;
 use vortex_sequence::Sequence;
@@ -100,6 +102,8 @@ pub static ALLOWED_ENCODINGS: LazyLock<HashSet<ArrayId>> = LazyLock::new(|| {
     allowed.insert(Delta.id());
     allowed.insert(FoR.id());
     allowed.insert(FSST.id());
+    #[cfg(feature = "onpair")]
+    allowed.insert(OnPair.id());
     allowed.insert(Pco.id());
     allowed.insert(RLE.id());
     allowed.insert(RunEnd.id());