fix[turboquant]: address PR review findings

- Reject nullable FixedSizeListArray input in both turboquant_encode_mse
  and turboquant_encode_qjl with a clear error message. TurboQuant is
  lossy and cannot preserve null positions.
- Fix with_vector_quantization composability: store TurboQuantConfig in
  the builder and apply at build() time, so it doesn't discard a
  previously-configured compressor. Document precedence rules.
- Export VECTOR_EXT_ID and FIXED_SHAPE_TENSOR_EXT_ID as public constants
  from vortex-turboquant; import in vortex-btrblocks instead of
  hardcoding duplicate string literals.
- Add QJL roundtrip and inner product bias tests for dim=768 (non-
  power-of-2 requiring padding to 1024).
- Move function-scoped imports to top of test module and benchmark file
  per CLAUDE.md conventions.
- Regenerate public-api.lock.

Total: 88 unit tests + 1 doctest.

Signed-off-by: Will Manning <will@spiraldb.com>
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Will Manning <will@willmanning.io>

Author: lwwmanning

encodings/turboquant/public-api.lock

@@ -332,6 +332,10 @@ pub fn vortex_turboquant::TurboQuantQJLMetadata::clear(&mut self)
 
 pub fn vortex_turboquant::TurboQuantQJLMetadata::encoded_len(&self) -> usize
 
+pub const vortex_turboquant::FIXED_SHAPE_TENSOR_EXT_ID: &str
+
+pub const vortex_turboquant::VECTOR_EXT_ID: &str
+
 pub fn vortex_turboquant::initialize(session: &mut vortex_session::VortexSession)
 
 pub fn vortex_turboquant::turboquant_encode_mse(fsl: &vortex_array::arrays::fixed_size_list::array::FixedSizeListArray, config: &vortex_turboquant::TurboQuantConfig) -> vortex_error::VortexResult<vortex_turboquant::TurboQuantMSEArray>

encodings/turboquant/src/compress.rs

@@ -7,6 +7,7 @@ use vortex_array::IntoArray;
 use vortex_array::arrays::BoolArray;
 use vortex_array::arrays::FixedSizeListArray;
 use vortex_array::arrays::PrimitiveArray;
+use vortex_array::dtype::Nullability;
 use vortex_array::dtype::PType;
 use vortex_array::validity::Validity;
 use vortex_buffer::BitBufferMut;
@@ -59,10 +60,17 @@ fn l2_norm(x: &[f32]) -> f32 {
 }
 
 /// Encode a FixedSizeListArray into a `TurboQuantMSEArray`.
+///
+/// The input must be non-nullable. TurboQuant is a lossy encoding that does not
+/// preserve null positions; callers must handle validity externally.
 pub fn turboquant_encode_mse(
     fsl: &FixedSizeListArray,
     config: &TurboQuantConfig,
 ) -> VortexResult<TurboQuantMSEArray> {
+    vortex_ensure!(
+        fsl.dtype().nullability() == Nullability::NonNullable,
+        "TurboQuant requires non-nullable input, got nullable FixedSizeListArray"
+    );
     vortex_ensure!(
         config.bit_width >= 1 && config.bit_width <= 8,
         "MSE bit_width must be 1-8, got {}",
@@ -148,10 +156,16 @@ pub fn turboquant_encode_mse(
 /// Encode a FixedSizeListArray into a `TurboQuantQJLArray`.
 ///
 /// Produces a cascaded structure: QJLArray wrapping an MSEArray at `bit_width - 1`.
+/// The input must be non-nullable. TurboQuant is a lossy encoding that does not
+/// preserve null positions; callers must handle validity externally.
 pub fn turboquant_encode_qjl(
     fsl: &FixedSizeListArray,
     config: &TurboQuantConfig,
 ) -> VortexResult<TurboQuantQJLArray> {
+    vortex_ensure!(
+        fsl.dtype().nullability() == Nullability::NonNullable,
+        "TurboQuant requires non-nullable input, got nullable FixedSizeListArray"
+    );
     vortex_ensure!(
         config.bit_width >= 2 && config.bit_width <= 9,
         "QJL bit_width must be 2-9, got {}",

encodings/turboquant/src/lib.rs

@@ -94,6 +94,12 @@ mod mse;
 mod qjl;
 pub mod rotation;
 
+/// Extension ID for the `Vector` type from `vortex-tensor`.
+pub const VECTOR_EXT_ID: &str = "vortex.tensor.vector";
+
+/// Extension ID for the `FixedShapeTensor` type from `vortex-tensor`.
+pub const FIXED_SHAPE_TENSOR_EXT_ID: &str = "vortex.tensor.fixed_shape_tensor";
+
 use vortex_array::session::ArraySessionExt;
 use vortex_session::VortexSession;
 
@@ -108,6 +114,11 @@ pub fn initialize(session: &mut VortexSession) {
 mod tests {
     use std::sync::LazyLock;
 
+    use rand::RngExt;
+    use rand::SeedableRng;
+    use rand::rngs::StdRng;
+    use rand_distr::Distribution;
+    use rand_distr::Normal;
     use rstest::rstest;
     use vortex_array::IntoArray;
     use vortex_array::VortexSessionExecute;
@@ -128,11 +139,6 @@ mod tests {
 
     /// Create a FixedSizeListArray of random f32 vectors (i.i.d. standard normal).
     fn make_fsl(num_rows: usize, dim: usize, seed: u64) -> FixedSizeListArray {
-        use rand::SeedableRng;
-        use rand::rngs::StdRng;
-        use rand_distr::Distribution;
-        use rand_distr::Normal;
-
         let mut rng = StdRng::seed_from_u64(seed);
         let normal = Normal::new(0.0f32, 1.0).unwrap();
 
@@ -339,6 +345,7 @@ mod tests {
     #[case(128, 6)]
     #[case(128, 8)]
     #[case(128, 9)]
+    #[case(768, 3)]
     fn roundtrip_qjl(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> {
         let fsl = make_fsl(10, dim, 42);
         let config = TurboQuantConfig {
@@ -357,6 +364,8 @@ mod tests {
     #[case(128, 6)]
     #[case(128, 8)]
     #[case(128, 9)]
+    #[case(768, 3)]
+    #[case(768, 4)]
     fn qjl_inner_product_bias(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> {
         let num_rows = 100;
         let fsl = make_fsl(num_rows, dim, 42);
@@ -367,14 +376,10 @@ mod tests {
         let (original, decoded) = encode_decode_qjl(&fsl, &config)?;
 
         let num_pairs = 500;
-        let mut rng = {
-            use rand::SeedableRng;
-            rand::rngs::StdRng::seed_from_u64(0)
-        };
+        let mut rng = StdRng::seed_from_u64(0);
         let mut signed_errors = Vec::with_capacity(num_pairs);
 
         for _ in 0..num_pairs {
-            use rand::RngExt;
             let qi = rng.random_range(0..num_rows);
             let xi = rng.random_range(0..num_rows);
             if qi == xi {

vortex-btrblocks/src/compressor/turboquant.rs

@@ -7,13 +7,11 @@ use vortex_array::ArrayRef;
 use vortex_array::IntoArray;
 use vortex_array::arrays::ExtensionArray;
 use vortex_error::VortexResult;
+use vortex_turboquant::FIXED_SHAPE_TENSOR_EXT_ID;
 use vortex_turboquant::TurboQuantConfig;
+use vortex_turboquant::VECTOR_EXT_ID;
 use vortex_turboquant::turboquant_encode_qjl;
 
-/// Extension IDs for tensor types (from vortex-tensor).
-const VECTOR_EXT_ID: &str = "vortex.tensor.vector";
-const FIXED_SHAPE_TENSOR_EXT_ID: &str = "vortex.tensor.fixed_shape_tensor";
-
 /// Check if an extension array has a tensor extension type.
 pub(crate) fn is_tensor_extension(ext_array: &ExtensionArray) -> bool {
     let ext_id = ext_array.ext_dtype().id();

vortex-file/src/strategy.rs

@@ -125,6 +125,7 @@ pub static ALLOWED_ENCODINGS: LazyLock<ArrayRegistry> = LazyLock::new(|| {
 /// bulk decoding performance, and IOPS required to perform an indexed read.
 pub struct WriteStrategyBuilder {
     compressor: Option<Arc<dyn CompressorPlugin>>,
+    turboquant_config: Option<vortex_turboquant::TurboQuantConfig>,
     row_block_size: usize,
     field_writers: HashMap<FieldPath, Arc<dyn LayoutStrategy>>,
     allow_encodings: Option<ArrayRegistry>,
@@ -137,6 +138,7 @@ impl Default for WriteStrategyBuilder {
     fn default() -> Self {
         Self {
             compressor: None,
+            turboquant_config: None,
             row_block_size: 8192,
             field_writers: HashMap::new(),
             allow_encodings: Some(ALLOWED_ENCODINGS.clone()),
@@ -237,18 +239,19 @@ impl WriteStrategyBuilder {
     /// The TurboQuant array's children (norms, codes) are recursively compressed by the
     /// BtrBlocks compressor.
     ///
+    /// This can be combined with other builder methods. If a custom compressor is also set
+    /// via [`with_compressor`](Self::with_compressor), the custom compressor takes precedence
+    /// and the TurboQuant config is ignored.
+    ///
     /// # Examples
     ///
     /// ```ignore
     /// WriteStrategyBuilder::default()
-    ///     .with_vector_quantization(TurboQuantConfig { bit_width: 3, .. })
+    ///     .with_vector_quantization(TurboQuantConfig { bit_width: 3, seed: None })
     ///     .build()
     /// ```
     pub fn with_vector_quantization(mut self, config: vortex_turboquant::TurboQuantConfig) -> Self {
-        let btrblocks = BtrBlocksCompressorBuilder::default()
-            .with_turboquant(config)
-            .build();
-        self.compressor = Some(Arc::new(btrblocks));
+        self.turboquant_config = Some(config);
         self
     }
 
@@ -270,6 +273,14 @@ impl WriteStrategyBuilder {
         // 5. compress each chunk
         let compressing = if let Some(ref compressor) = self.compressor {
             CompressingStrategy::new_opaque(buffered, compressor.clone())
+        } else if let Some(tq_config) = self.turboquant_config {
+            let btrblocks = BtrBlocksCompressorBuilder::default()
+                .with_turboquant(tq_config)
+                .build();
+            CompressingStrategy::new_opaque(
+                buffered,
+                Arc::new(btrblocks) as Arc<dyn CompressorPlugin>,
+            )
         } else {
             CompressingStrategy::new_btrblocks(buffered, true)
         };

vortex/benches/single_encoding_throughput.rs

@@ -17,10 +17,12 @@ use rand::prelude::IndexedRandom;
 use rand::rngs::StdRng;
 use vortex::array::IntoArray;
 use vortex::array::ToCanonical;
+use vortex::array::arrays::FixedSizeListArray;
 use vortex::array::arrays::PrimitiveArray;
 use vortex::array::arrays::VarBinViewArray;
 use vortex::array::builders::dict::dict_encode;
 use vortex::array::builtins::ArrayBuiltins;
+use vortex::array::validity::Validity;
 use vortex::dtype::PType;
 use vortex::encodings::alp::RDEncoder;
 use vortex::encodings::alp::alp_encode;
@@ -39,6 +41,7 @@ use vortex::encodings::zstd::ZstdArray;
 use vortex_array::VortexSessionExecute;
 use vortex_array::dtype::Nullability;
 use vortex_array::session::ArraySession;
+use vortex_buffer::BufferMut;
 use vortex_sequence::SequenceArray;
 use vortex_session::VortexSession;
 
@@ -410,10 +413,6 @@ fn bench_zstd_decompress_string(bencher: Bencher) {
 
 // TurboQuant vector quantization benchmarks
 
-use vortex::array::arrays::FixedSizeListArray;
-use vortex::array::validity::Validity;
-use vortex_buffer::BufferMut;
-
 const NUM_VECTORS: usize = 1_000;
 
 /// Generate `num_vectors` random f32 vectors of the given dimension using i.i.d.