add CompressorConfig and clean up builder interfaces

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>

Author: connortsui20

fuzz/fuzz_targets/file_io.rs

@@ -17,6 +17,7 @@ use vortex_array::dtype::StructFields;
 use vortex_array::expr::lit;
 use vortex_array::expr::root;
 use vortex_array::scalar_fn::fns::operators::Operator;
+use vortex_btrblocks::BtrBlocksCompressorBuilder;
 use vortex_buffer::ByteBufferMut;
 use vortex_error::VortexExpect;
 use vortex_error::vortex_panic;
@@ -59,12 +60,11 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus {
 
     let write_options = match compressor_strategy {
         CompressorStrategy::Default => SESSION.write_options(),
-        CompressorStrategy::Compact => {
-            let strategy = WriteStrategyBuilder::default()
-                .with_compact_encodings()
-                .build();
-            SESSION.write_options().with_strategy(strategy)
-        }
+        CompressorStrategy::Compact => SESSION.write_options().with_strategy(
+            WriteStrategyBuilder::default()
+                .with_btrblocks_builder(BtrBlocksCompressorBuilder::default().with_compact())
+                .build(),
+        ),
     };
 
     let mut full_buff = ByteBufferMut::empty();

vortex-bench/src/lib.rs

@@ -26,6 +26,7 @@ use tpcds::TpcDsBenchmark;
 use tpch::benchmark::TpcHBenchmark;
 pub use utils::file::*;
 pub use utils::logging::*;
+use vortex::compressor::BtrBlocksCompressorBuilder;
 use vortex::error::VortexExpect;
 use vortex::error::vortex_err;
 use vortex::file::VortexWriteOptions;
@@ -231,7 +232,7 @@ impl CompactionStrategy {
         match self {
             CompactionStrategy::Compact => options.with_strategy(
                 WriteStrategyBuilder::default()
-                    .with_compact_encodings()
+                    .with_btrblocks_builder(BtrBlocksCompressorBuilder::default().with_compact())
                     .build(),
             ),
             CompactionStrategy::Default => options,

vortex-btrblocks/public-api.lock

@@ -614,11 +614,13 @@ impl vortex_btrblocks::BtrBlocksCompressorBuilder
 
 pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::build(self) -> vortex_btrblocks::BtrBlocksCompressor
 
-pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::exclude(self, ids: impl core::iter::traits::collect::IntoIterator<Item = vortex_compressor::scheme::SchemeId>) -> Self
+pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::exclude_schemes(self, ids: impl core::iter::traits::collect::IntoIterator<Item = vortex_compressor::scheme::SchemeId>) -> Self
+
+pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::only_cuda_compatible(self) -> Self
 
 pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::with_compact(self) -> Self
 
-pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::with_new_scheme(self, scheme: &'static dyn vortex_compressor::scheme::Scheme) -> Self
+pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::with_scheme(self, scheme: &'static dyn vortex_compressor::scheme::Scheme) -> Self
 
 impl core::clone::Clone for vortex_btrblocks::BtrBlocksCompressorBuilder
 

vortex-btrblocks/src/builder.rs

@@ -67,7 +67,7 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
 ///
 /// By default, all schemes in [`ALL_SCHEMES`] are enabled. Feature-gated schemes (Pco, Zstd)
 /// are not in `ALL_SCHEMES` and must be added explicitly via
-/// [`with_new_scheme`](BtrBlocksCompressorBuilder::with_new_scheme) or
+/// [`with_scheme`](BtrBlocksCompressorBuilder::with_scheme) or
 /// [`with_compact`](BtrBlocksCompressorBuilder::with_compact).
 ///
 /// # Examples
@@ -79,9 +79,9 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
 /// // Default compressor with all schemes in ALL_SCHEMES.
 /// let compressor = BtrBlocksCompressorBuilder::default().build();
 ///
-/// // Exclude specific schemes.
+/// // Remove specific schemes.
 /// let compressor = BtrBlocksCompressorBuilder::default()
-///     .exclude([IntDictScheme.id()])
+///     .exclude_schemes([IntDictScheme.id()])
 ///     .build();
 /// ```
 #[derive(Debug, Clone)]
@@ -100,13 +100,13 @@ impl Default for BtrBlocksCompressorBuilder {
 impl BtrBlocksCompressorBuilder {
     /// Adds an external compression scheme not in [`ALL_SCHEMES`].
     ///
-    /// This allows encoding crates outside of `vortex-btrblocks` to register their own schemes with
-    /// the compressor.
+    /// This allows encoding crates outside of `vortex-btrblocks` to register their own schemes
+    /// with the compressor.
     ///
     /// # Panics
     ///
     /// Panics if a scheme with the same [`SchemeId`] is already present.
-    pub fn with_new_scheme(mut self, scheme: &'static dyn Scheme) -> Self {
+    pub fn with_scheme(mut self, scheme: &'static dyn Scheme) -> Self {
         assert!(
             !self.schemes.iter().any(|s| s.id() == scheme.id()),
             "scheme {:?} is already present in the builder",
@@ -128,19 +128,42 @@ impl BtrBlocksCompressorBuilder {
     /// Panics if any of the compact schemes are already present.
     #[cfg(feature = "zstd")]
     pub fn with_compact(self) -> Self {
-        // This should be fast since we don't have that many schemes.
-        let builder = self.with_new_scheme(&string::ZstdScheme);
+        let builder = self.with_scheme(&string::ZstdScheme);
 
         #[cfg(feature = "pco")]
         let builder = builder
-            .with_new_scheme(&integer::PcoScheme)
-            .with_new_scheme(&float::PcoScheme);
+            .with_scheme(&integer::PcoScheme)
+            .with_scheme(&float::PcoScheme);
 
         builder
     }
 
-    /// Excludes the specified compression schemes by their [`SchemeId`].
-    pub fn exclude(mut self, ids: impl IntoIterator<Item = SchemeId>) -> Self {
+    /// Excludes schemes without CUDA kernel support and adds Zstd for string compression.
+    ///
+    /// With the `unstable_encodings` feature, buffer-level Zstd compression is used which
+    /// preserves the array buffer layout for zero-conversion GPU decompression. Without it,
+    /// interleaved Zstd compression is used.
+    #[cfg(feature = "zstd")]
+    pub fn only_cuda_compatible(self) -> Self {
+        let builder = self.exclude_schemes([
+            integer::SparseScheme.id(),
+            rle::RLE_INTEGER_SCHEME.id(),
+            rle::RLE_FLOAT_SCHEME.id(),
+            float::NullDominatedSparseScheme.id(),
+            string::StringDictScheme.id(),
+            string::FSSTScheme.id(),
+        ]);
+
+        #[cfg(feature = "unstable_encodings")]
+        let builder = builder.with_scheme(&string::ZstdBuffersScheme);
+        #[cfg(not(feature = "unstable_encodings"))]
+        let builder = builder.with_scheme(&string::ZstdScheme);
+
+        builder
+    }
+
+    /// Removes the specified compression schemes by their [`SchemeId`].
+    pub fn exclude_schemes(mut self, ids: impl IntoIterator<Item = SchemeId>) -> Self {
         let ids: HashSet<_> = ids.into_iter().collect();
         self.schemes.retain(|s| !ids.contains(&s.id()));
         self

vortex-btrblocks/src/canonical_compressor.rs

@@ -25,9 +25,9 @@ use crate::CascadingCompressor;
 /// // Default compressor - all schemes allowed.
 /// let compressor = BtrBlocksCompressor::default();
 ///
-/// // Exclude specific schemes using the builder.
+/// // Remove specific schemes using the builder.
 /// let compressor = BtrBlocksCompressorBuilder::default()
-///     .exclude([IntDictScheme.id()])
+///     .exclude_schemes([IntDictScheme.id()])
 ///     .build();
 /// ```
 #[derive(Clone)]

vortex-btrblocks/src/lib.rs

@@ -46,9 +46,9 @@
 //! // Default compressor with all schemes enabled.
 //! let compressor = BtrBlocksCompressor::default();
 //!
-//! // Configure with builder to exclude specific schemes.
+//! // Remove specific schemes using the builder.
 //! let compressor = BtrBlocksCompressorBuilder::default()
-//!     .exclude([IntDictScheme.id()])
+//!     .exclude_schemes([IntDictScheme.id()])
 //!     .build();
 //! ```
 //!

vortex-cuda/gpu-scan-cli/src/main.rs

@@ -21,6 +21,7 @@ use vortex::VortexSessionDefault;
 use vortex::array::ToCanonical;
 use vortex::array::arrays::Dict;
 use vortex::buffer::ByteBufferMut;
+use vortex::compressor::BtrBlocksCompressorBuilder;
 use vortex::error::VortexResult;
 use vortex::file::OpenOptionsSessionExt;
 use vortex::file::WriteOptionsSessionExt;
@@ -92,7 +93,7 @@ async fn main() -> VortexResult<()> {
 #[cuda_available]
 fn cuda_write_strategy() -> Arc<dyn vortex::layout::LayoutStrategy> {
     WriteStrategyBuilder::default()
-        .with_cuda_compatible_encodings()
+        .with_btrblocks(BtrBlocksCompressorBuilder::default().only_cuda_compatible())
         .with_flat_strategy(Arc::new(CudaFlatLayoutStrategy::default()))
         .build()
 }

vortex-file/public-api.lock

@@ -346,12 +346,10 @@ pub fn vortex_file::WriteStrategyBuilder::build(self) -> alloc::sync::Arc<dyn vo
 
 pub fn vortex_file::WriteStrategyBuilder::with_allow_encodings(self, allow_encodings: vortex_array::session::ArrayRegistry) -> Self
 
-pub fn vortex_file::WriteStrategyBuilder::with_compact_encodings(self) -> Self
+pub fn vortex_file::WriteStrategyBuilder::with_btrblocks_builder(self, builder: vortex_btrblocks::builder::BtrBlocksCompressorBuilder) -> Self
 
 pub fn vortex_file::WriteStrategyBuilder::with_compressor<C: vortex_layout::layouts::compressed::CompressorPlugin>(self, compressor: C) -> Self
 
-pub fn vortex_file::WriteStrategyBuilder::with_cuda_compatible_encodings(self) -> Self
-
 pub fn vortex_file::WriteStrategyBuilder::with_field_writer(self, field: impl core::convert::Into<vortex_array::dtype::field::FieldPath>, writer: alloc::sync::Arc<dyn vortex_layout::strategy::LayoutStrategy>) -> Self
 
 pub fn vortex_file::WriteStrategyBuilder::with_flat_strategy(self, flat: alloc::sync::Arc<dyn vortex_layout::strategy::LayoutStrategy>) -> Self