From bcdbd766db34e087dcf942f34371b3fc5beb3b54 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 16 Jul 2026 17:24:47 +0100 Subject: [PATCH 1/2] feat[onpair]: enable the onpair reader Signed-off-by: Joe Isaacs --- vortex-btrblocks/src/schemes/string/mod.rs | 1 - vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs | 4 +--- vortex-file/Cargo.toml | 3 +-- vortex-file/src/lib.rs | 1 - 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/vortex-btrblocks/src/schemes/string/mod.rs b/vortex-btrblocks/src/schemes/string/mod.rs index ac8e5b4b8df..ff244bf6599 100644 --- a/vortex-btrblocks/src/schemes/string/mod.rs +++ b/vortex-btrblocks/src/schemes/string/mod.rs @@ -11,7 +11,6 @@ mod zstd; #[cfg(all(feature = "zstd", feature = "unstable_encodings"))] mod zstd_buffers; -#[cfg(feature = "unstable_encodings")] mod onpair; pub use fsst::FSSTScheme; diff --git a/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs index c1473146607..bf80089bb16 100644 --- a/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs @@ -17,6 +17,7 @@ use vortex_fsst::FSST; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; +use crate::schemes::string::onpair::OnPairScheme; static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); @@ -46,11 +47,9 @@ fn test_dict_compressed() -> VortexResult<()> { Ok(()) } -#[cfg(feature = "unstable_encodings")] #[test] fn test_onpair_in_default_scheme_list() { use crate::SchemeExt; - use crate::schemes::string::OnPairScheme; let ids: Vec<_> = crate::ALL_SCHEMES.iter().map(|s| s.id()).collect(); assert!( @@ -59,7 +58,6 @@ fn test_onpair_in_default_scheme_list() { ); } -#[cfg(feature = "unstable_encodings")] #[test] fn test_onpair_compressed() -> VortexResult<()> { // Dictionary-style string corpus: high lexical overlap, short rows. diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index a7af1b293dd..347bd0fc69e 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -47,7 +47,7 @@ vortex-io = { workspace = true } vortex-layout = { workspace = true } vortex-mask = { workspace = true } vortex-metrics = { workspace = true } -vortex-onpair = { workspace = true, optional = true } +vortex-onpair = { workspace = true } vortex-pco = { workspace = true } vortex-runend = { workspace = true } vortex-scan = { workspace = true } @@ -81,7 +81,6 @@ tokio = [ zstd = ["dep:vortex-zstd", "vortex-btrblocks/zstd", "vortex-btrblocks/pco"] # This feature enables unstable encodings for which we don't guarantee stability. unstable_encodings = [ - "dep:vortex-onpair", "dep:vortex-tensor", "vortex-zstd?/unstable_encodings", "vortex-btrblocks/unstable_encodings", diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 261ba7191d6..7651c66ff83 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -165,7 +165,6 @@ mod forever_constant { pub fn register_default_encodings(session: &VortexSession) { vortex_bytebool::initialize(session); vortex_fsst::initialize(session); - #[cfg(feature = "unstable_encodings")] vortex_onpair::initialize(session); vortex_zigzag::initialize(session); From fc8ced8d5d6c6c625741463967db10245be6ce8a Mon Sep 17 00:00:00 2001 From: Francesco Gargiulo Date: Mon, 20 Jul 2026 11:57:54 +0100 Subject: [PATCH 2/2] feat(onpair): promote onpair from experimental to stable (reader-only) Move encodings/experimental/onpair -> encodings/onpair and register the OnPair reader unconditionally. The writer stays gated behind the `unstable_encodings` feature (file ALLOWED_ENCODINGS and btrblocks ALL_SCHEMES/scheme module), so files with OnPair can be read but not yet written by default. Signed-off-by: Francesco Gargiulo Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 5 ++--- encodings/{experimental => }/onpair/Cargo.toml | 0 encodings/{experimental => }/onpair/README.md | 0 .../{experimental => }/onpair/benches/decode.rs | 0 .../onpair/goldenfiles/onpair.metadata | 0 encodings/{experimental => }/onpair/src/array.rs | 0 .../{experimental => }/onpair/src/canonical.rs | 0 .../{experimental => }/onpair/src/compress.rs | 0 .../onpair/src/compute/byte_length.rs | 0 .../{experimental => }/onpair/src/compute/cast.rs | 0 .../onpair/src/compute/compare.rs | 0 .../onpair/src/compute/filter.rs | 0 .../{experimental => }/onpair/src/compute/mod.rs | 0 .../onpair/src/compute/slice.rs | 0 encodings/{experimental => }/onpair/src/decode.rs | 0 encodings/{experimental => }/onpair/src/kernel.rs | 0 encodings/{experimental => }/onpair/src/lib.rs | 0 encodings/{experimental => }/onpair/src/ops.rs | 0 encodings/{experimental => }/onpair/src/rules.rs | 0 encodings/{experimental => }/onpair/src/tests.rs | 0 .../{experimental => }/onpair/tests/big_data.rs | 0 vortex-btrblocks/src/schemes/string/mod.rs | 1 + .../src/schemes/string/scheme_selection_tests.rs | 15 +++++++++------ vortex-file/src/lib.rs | 13 +++++++++++++ 24 files changed, 25 insertions(+), 9 deletions(-) rename encodings/{experimental => }/onpair/Cargo.toml (100%) rename encodings/{experimental => }/onpair/README.md (100%) rename encodings/{experimental => }/onpair/benches/decode.rs (100%) rename encodings/{experimental => }/onpair/goldenfiles/onpair.metadata (100%) rename encodings/{experimental => }/onpair/src/array.rs (100%) rename encodings/{experimental => }/onpair/src/canonical.rs (100%) rename encodings/{experimental => }/onpair/src/compress.rs (100%) rename encodings/{experimental => }/onpair/src/compute/byte_length.rs (100%) rename encodings/{experimental => }/onpair/src/compute/cast.rs (100%) rename encodings/{experimental => }/onpair/src/compute/compare.rs (100%) rename encodings/{experimental => }/onpair/src/compute/filter.rs (100%) rename encodings/{experimental => }/onpair/src/compute/mod.rs (100%) rename encodings/{experimental => }/onpair/src/compute/slice.rs (100%) rename encodings/{experimental => }/onpair/src/decode.rs (100%) rename encodings/{experimental => }/onpair/src/kernel.rs (100%) rename encodings/{experimental => }/onpair/src/lib.rs (100%) rename encodings/{experimental => }/onpair/src/ops.rs (100%) rename encodings/{experimental => }/onpair/src/rules.rs (100%) rename encodings/{experimental => }/onpair/src/tests.rs (100%) rename encodings/{experimental => }/onpair/tests/big_data.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 68be0c50653..65b158ed524 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,8 +59,7 @@ members = [ "encodings/zstd", "encodings/bytebool", "encodings/parquet-variant", - # Experimental encodings - "encodings/experimental/onpair", + "encodings/onpair", # Benchmarks "benchmarks/lance-bench", "benchmarks/compress-bench", @@ -307,7 +306,7 @@ vortex-json = { version = "0.1.0", path = "./vortex-json", default-features = fa vortex-layout = { version = "0.1.0", path = "./vortex-layout", default-features = false } vortex-mask = { version = "0.1.0", path = "./vortex-mask", default-features = false } vortex-metrics = { version = "0.1.0", path = "./vortex-metrics", default-features = false } -vortex-onpair = { version = "0.1.0", path = "./encodings/experimental/onpair", default-features = false } +vortex-onpair = { version = "0.1.0", path = "./encodings/onpair", default-features = false } vortex-parquet-variant = { version = "0.1.0", path = "./encodings/parquet-variant" } vortex-pco = { version = "0.1.0", path = "./encodings/pco", default-features = false } vortex-proto = { version = "0.1.0", path = "./vortex-proto", default-features = false } diff --git a/encodings/experimental/onpair/Cargo.toml b/encodings/onpair/Cargo.toml similarity index 100% rename from encodings/experimental/onpair/Cargo.toml rename to encodings/onpair/Cargo.toml diff --git a/encodings/experimental/onpair/README.md b/encodings/onpair/README.md similarity index 100% rename from encodings/experimental/onpair/README.md rename to encodings/onpair/README.md diff --git a/encodings/experimental/onpair/benches/decode.rs b/encodings/onpair/benches/decode.rs similarity index 100% rename from encodings/experimental/onpair/benches/decode.rs rename to encodings/onpair/benches/decode.rs diff --git a/encodings/experimental/onpair/goldenfiles/onpair.metadata b/encodings/onpair/goldenfiles/onpair.metadata similarity index 100% rename from encodings/experimental/onpair/goldenfiles/onpair.metadata rename to encodings/onpair/goldenfiles/onpair.metadata diff --git a/encodings/experimental/onpair/src/array.rs b/encodings/onpair/src/array.rs similarity index 100% rename from encodings/experimental/onpair/src/array.rs rename to encodings/onpair/src/array.rs diff --git a/encodings/experimental/onpair/src/canonical.rs b/encodings/onpair/src/canonical.rs similarity index 100% rename from encodings/experimental/onpair/src/canonical.rs rename to encodings/onpair/src/canonical.rs diff --git a/encodings/experimental/onpair/src/compress.rs b/encodings/onpair/src/compress.rs similarity index 100% rename from encodings/experimental/onpair/src/compress.rs rename to encodings/onpair/src/compress.rs diff --git a/encodings/experimental/onpair/src/compute/byte_length.rs b/encodings/onpair/src/compute/byte_length.rs similarity index 100% rename from encodings/experimental/onpair/src/compute/byte_length.rs rename to encodings/onpair/src/compute/byte_length.rs diff --git a/encodings/experimental/onpair/src/compute/cast.rs b/encodings/onpair/src/compute/cast.rs similarity index 100% rename from encodings/experimental/onpair/src/compute/cast.rs rename to encodings/onpair/src/compute/cast.rs diff --git a/encodings/experimental/onpair/src/compute/compare.rs b/encodings/onpair/src/compute/compare.rs similarity index 100% rename from encodings/experimental/onpair/src/compute/compare.rs rename to encodings/onpair/src/compute/compare.rs diff --git a/encodings/experimental/onpair/src/compute/filter.rs b/encodings/onpair/src/compute/filter.rs similarity index 100% rename from encodings/experimental/onpair/src/compute/filter.rs rename to encodings/onpair/src/compute/filter.rs diff --git a/encodings/experimental/onpair/src/compute/mod.rs b/encodings/onpair/src/compute/mod.rs similarity index 100% rename from encodings/experimental/onpair/src/compute/mod.rs rename to encodings/onpair/src/compute/mod.rs diff --git a/encodings/experimental/onpair/src/compute/slice.rs b/encodings/onpair/src/compute/slice.rs similarity index 100% rename from encodings/experimental/onpair/src/compute/slice.rs rename to encodings/onpair/src/compute/slice.rs diff --git a/encodings/experimental/onpair/src/decode.rs b/encodings/onpair/src/decode.rs similarity index 100% rename from encodings/experimental/onpair/src/decode.rs rename to encodings/onpair/src/decode.rs diff --git a/encodings/experimental/onpair/src/kernel.rs b/encodings/onpair/src/kernel.rs similarity index 100% rename from encodings/experimental/onpair/src/kernel.rs rename to encodings/onpair/src/kernel.rs diff --git a/encodings/experimental/onpair/src/lib.rs b/encodings/onpair/src/lib.rs similarity index 100% rename from encodings/experimental/onpair/src/lib.rs rename to encodings/onpair/src/lib.rs diff --git a/encodings/experimental/onpair/src/ops.rs b/encodings/onpair/src/ops.rs similarity index 100% rename from encodings/experimental/onpair/src/ops.rs rename to encodings/onpair/src/ops.rs diff --git a/encodings/experimental/onpair/src/rules.rs b/encodings/onpair/src/rules.rs similarity index 100% rename from encodings/experimental/onpair/src/rules.rs rename to encodings/onpair/src/rules.rs diff --git a/encodings/experimental/onpair/src/tests.rs b/encodings/onpair/src/tests.rs similarity index 100% rename from encodings/experimental/onpair/src/tests.rs rename to encodings/onpair/src/tests.rs diff --git a/encodings/experimental/onpair/tests/big_data.rs b/encodings/onpair/tests/big_data.rs similarity index 100% rename from encodings/experimental/onpair/tests/big_data.rs rename to encodings/onpair/tests/big_data.rs diff --git a/vortex-btrblocks/src/schemes/string/mod.rs b/vortex-btrblocks/src/schemes/string/mod.rs index ff244bf6599..ac8e5b4b8df 100644 --- a/vortex-btrblocks/src/schemes/string/mod.rs +++ b/vortex-btrblocks/src/schemes/string/mod.rs @@ -11,6 +11,7 @@ mod zstd; #[cfg(all(feature = "zstd", feature = "unstable_encodings"))] mod zstd_buffers; +#[cfg(feature = "unstable_encodings")] mod onpair; pub use fsst::FSSTScheme; diff --git a/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs index bf80089bb16..ec51c5104bb 100644 --- a/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs @@ -17,7 +17,6 @@ use vortex_fsst::FSST; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; -use crate::schemes::string::onpair::OnPairScheme; static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); @@ -47,9 +46,11 @@ fn test_dict_compressed() -> VortexResult<()> { Ok(()) } +#[cfg(feature = "unstable_encodings")] #[test] -fn test_onpair_in_default_scheme_list() { +fn test_unstable_all_schemes_includes_onpair() { use crate::SchemeExt; + use crate::schemes::string::onpair::OnPairScheme; let ids: Vec<_> = crate::ALL_SCHEMES.iter().map(|s| s.id()).collect(); assert!( @@ -58,11 +59,13 @@ fn test_onpair_in_default_scheme_list() { ); } +#[cfg(feature = "unstable_encodings")] #[test] -fn test_onpair_compressed() -> VortexResult<()> { +fn test_unstable_default_btrblocks_compressor_selects_onpair() -> VortexResult<()> { // Dictionary-style string corpus: high lexical overlap, short rows. // OnPair beats FSST on this corpus, so it wins the sample-based - // comparison even though both are registered by default. + // comparison even though both are registered when `unstable_encodings` + // is enabled. let mut strings = Vec::with_capacity(1000); for i in 0..1000 { strings.push(Some(format!( @@ -81,8 +84,8 @@ fn test_onpair_compressed() -> VortexResult<()> { Ok(()) } -/// FSST is registered in the default scheme list (alongside OnPair), and an -/// FSST-only builder still produces an FSST array. +/// FSST is registered in the default scheme list, and an FSST-only builder +/// still produces an FSST array. #[test] fn test_fsst_in_default_scheme_list() -> VortexResult<()> { use crate::BtrBlocksCompressorBuilder; diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 7651c66ff83..4ece7d9cdab 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -200,6 +200,7 @@ mod default_encoding_tests { use vortex_array::optimizer::kernels::ArrayKernelsExt as _; use vortex_array::session::ArraySessionExt as _; use vortex_fsst::FSST; + use vortex_onpair::OnPair; use crate::register_default_encodings; @@ -214,5 +215,17 @@ mod default_encoding_tests { assert!(session.arrays().registry().find(&FSST.id()).is_some()); assert!(session.kernels().has_execute_parent(Filter.id(), FSST.id())); + assert!(session.arrays().registry().find(&OnPair.id()).is_some()); + assert!( + session + .kernels() + .has_execute_parent(Filter.id(), OnPair.id()) + ); + } + + #[cfg(not(feature = "unstable_encodings"))] + #[test] + fn default_writer_does_not_allow_onpair() { + assert!(!crate::ALLOWED_ENCODINGS.contains(&OnPair.id())); } }