Revert "permutation"

This reverts commit 00ee4fe.

Author: lwwmanning

vortex-tensor/src/encodings/turboquant/array.rs

@@ -35,9 +35,6 @@ pub struct TurboQuantMetadata {
     /// Whether QJL correction children are present.
     #[prost(bool, tag = "3")]
     pub has_qjl: bool,
-    /// Whether a pre-SRHT permutation is stored (for non-power-of-2 dims).
-    #[prost(bool, tag = "4")]
-    pub has_permutation: bool,
 }
 
 /// Optional QJL (Quantized Johnson-Lindenstrauss) correction for unbiased
@@ -80,11 +77,10 @@ pub(crate) enum Slot {
     QjlSigns = 4,
     QjlResidualNorms = 5,
     QjlRotationSigns = 6,
-    Permutation = 7,
 }
 
 impl Slot {
-    pub(crate) const COUNT: usize = 8;
+    pub(crate) const COUNT: usize = 7;
 
     pub(crate) fn name(self) -> &'static str {
         match self {
@@ -95,7 +91,6 @@ impl Slot {
             Self::QjlSigns => "qjl_signs",
             Self::QjlResidualNorms => "qjl_residual_norms",
             Self::QjlRotationSigns => "qjl_rotation_signs",
-            Self::Permutation => "permutation",
         }
     }
 
@@ -108,7 +103,6 @@ impl Slot {
             4 => Self::QjlSigns,
             5 => Self::QjlResidualNorms,
             6 => Self::QjlRotationSigns,
-            7 => Self::Permutation,
             _ => vortex_error::vortex_panic!("invalid slot index {idx}"),
         }
     }
@@ -126,9 +120,6 @@ impl Slot {
 /// - 4: `qjl_signs` — `FixedSizeListArray<u8>` (num_rows * padded_dim, 1-bit)
 /// - 5: `qjl_residual_norms` — `PrimitiveArray<f32>` (one per row)
 /// - 6: `qjl_rotation_signs` — `BitPackedArray` (3 * padded_dim, 1-bit, QJL rotation)
-///
-/// Optional permutation slot (None for power-of-2 dims):
-/// - 7: `permutation` — `BitPackedArray<u16>` (padded_dim, ceil(log2(padded_dim))-bit)
 #[derive(Clone, Debug)]
 pub struct TurboQuantArray {
     pub(crate) dtype: DType,
@@ -256,11 +247,6 @@ impl TurboQuantArray {
         })
     }
 
-    /// The optional pre-SRHT permutation (for non-power-of-2 dims).
-    pub fn permutation(&self) -> Option<&ArrayRef> {
-        self.slots[Slot::Permutation as usize].as_ref()
-    }
-
     /// Set the QJL correction fields on this array.
     pub(crate) fn set_qjl(&mut self, qjl: QjlCorrection) {
         self.slots[Slot::QjlSigns as usize] = Some(qjl.signs);

vortex-tensor/src/encodings/turboquant/compress.rs

@@ -80,8 +80,6 @@ struct MseQuantizationResult {
     all_indices: BufferMut<u8>,
     norms: BufferMut<f32>,
     padded_dim: usize,
-    /// Random permutation for non-power-of-2 dims (shared by MSE and QJL).
-    perm: Option<Vec<u16>>,
 }
 
 /// Core quantization: extract f32 elements, build rotation, normalize/rotate/quantize all rows.
@@ -93,17 +91,9 @@ fn turboquant_quantize_core(
     let dimension = fsl.list_size() as usize;
     let num_rows = fsl.len();
 
-    let mut rotation = RotationMatrix::try_new(seed, dimension)?;
+    let rotation = RotationMatrix::try_new(seed, dimension)?;
     let padded_dim = rotation.padded_dim();
 
-    // For non-power-of-2 dims, generate a random permutation to scatter
-    // zero-padded entries uniformly before the SRHT.
-    let perm = (dimension < padded_dim)
-        .then(|| RotationMatrix::gen_permutation(seed.wrapping_add(42), padded_dim));
-    if let Some(ref p) = perm {
-        rotation = rotation.with_permutation(p.clone());
-    }
-
     let f32_elements = extract_f32_elements(fsl)?;
 
     let centroids = get_centroids(padded_dim as u32, bit_width)?;
@@ -142,7 +132,6 @@ fn turboquant_quantize_core(
         all_indices,
         norms,
         padded_dim,
-        perm,
     })
 }
 
@@ -177,23 +166,15 @@ fn build_turboquant_mse(
 
     let rotation_signs = bitpack_rotation_signs(&core.rotation)?;
 
-    let mut array = TurboQuantArray::try_new_mse(
+    TurboQuantArray::try_new_mse(
         fsl.dtype().clone(),
         codes,
         norms_array,
         centroids_array,
         rotation_signs,
         dimension,
         bit_width,
-    )?;
-
-    // Store permutation for non-power-of-2 dims.
-    if let Some(ref perm) = core.perm {
-        array.slots[crate::encodings::turboquant::array::Slot::Permutation as usize] =
-            Some(bitpack_permutation(perm)?);
-    }
-
-    Ok(array)
+    )
 }
 
 /// Encode a FixedSizeListArray into a MSE-only `TurboQuantArray`.
@@ -266,16 +247,7 @@ pub fn turboquant_encode_qjl(
 
     // QJL uses a different rotation than the MSE stage to ensure statistical
     // independence between the quantization noise and the sign projection.
-    // The same permutation is shared: it's a property of the padded embedding
-    // space, not of the rotation itself.
-    let qjl_rotation = {
-        let rot = RotationMatrix::try_new(seed.wrapping_add(25), dim)?;
-        if let Some(ref p) = core.perm {
-            rot.with_permutation(p.clone())
-        } else {
-            rot
-        }
-    };
+    let qjl_rotation = RotationMatrix::try_new(seed.wrapping_add(25), dim)?;
 
     let num_rows = fsl.len();
     let mut residual_norms_buf = BufferMut::<f32>::with_capacity(num_rows);
@@ -378,12 +350,3 @@ fn bitpack_rotation_signs(rotation: &RotationMatrix) -> VortexResult<ArrayRef> {
     let prim = PrimitiveArray::new::<u8>(buf.freeze(), Validity::NonNullable);
     Ok(bitpack_encode(&prim, 1, None)?.into_array())
 }
-
-/// Bitpack a permutation of u16 indices for efficient storage.
-fn bitpack_permutation(perm: &[u16]) -> VortexResult<ArrayRef> {
-    let mut buf = BufferMut::<u16>::with_capacity(perm.len());
-    buf.extend_from_slice(perm);
-    let prim = PrimitiveArray::new::<u16>(buf.freeze(), Validity::NonNullable);
-    let bit_width = (perm.len() as f64).log2().ceil() as u8;
-    Ok(bitpack_encode(&prim, bit_width, None)?.into_array())
-}

vortex-tensor/src/encodings/turboquant/compute/slice.rs

@@ -9,7 +9,6 @@ use vortex_array::arrays::slice::SliceReduce;
 use vortex_error::VortexResult;
 
 use crate::encodings::turboquant::array::QjlCorrection;
-use crate::encodings::turboquant::array::Slot;
 use crate::encodings::turboquant::array::TurboQuant;
 use crate::encodings::turboquant::array::TurboQuantArray;
 
@@ -41,8 +40,6 @@ impl SliceReduce for TurboQuant {
         if let Some(qjl) = sliced_qjl {
             result.set_qjl(qjl);
         }
-        // Permutation is shared (not per-row), clone unchanged.
-        result.slots[Slot::Permutation as usize] = array.permutation().cloned();
 
         Ok(Some(result.into_array()))
     }

vortex-tensor/src/encodings/turboquant/compute/take.rs

@@ -9,7 +9,6 @@ use vortex_array::arrays::dict::TakeExecute;
 use vortex_error::VortexResult;
 
 use crate::encodings::turboquant::array::QjlCorrection;
-use crate::encodings::turboquant::array::Slot;
 use crate::encodings::turboquant::array::TurboQuant;
 use crate::encodings::turboquant::array::TurboQuantArray;
 
@@ -46,8 +45,6 @@ impl TakeExecute for TurboQuant {
         if let Some(qjl) = taken_qjl {
             result.set_qjl(qjl);
         }
-        // Permutation is shared (not per-row), clone unchanged.
-        result.slots[Slot::Permutation as usize] = array.permutation().cloned();
 
         Ok(Some(result.into_array()))
     }

vortex-tensor/src/encodings/turboquant/decompress.rs

@@ -52,30 +52,14 @@ pub fn execute_decompress(
     let centroids_prim = array.centroids().clone().execute::<PrimitiveArray>(ctx)?;
     let centroids = centroids_prim.as_slice::<f32>();
 
-    // Unpack optional permutation (for non-power-of-2 dims).
-    let perm: Option<Vec<u16>> = array
-        .permutation()
-        .map(|arr| {
-            let prim = arr.clone().execute::<PrimitiveArray>(ctx)?;
-            Ok::<_, vortex_error::VortexError>(prim.as_slice::<u16>().to_vec())
-        })
-        .transpose()?;
-
     // FastLanes SIMD-unpacks the 1-bit bitpacked rotation signs into u8 0/1 values,
     // then we expand to u32 XOR masks once (amortized over all rows). This enables
     // branchless XOR-based sign application in the per-row SRHT hot loop.
     let signs_prim = array
         .rotation_signs()
         .clone()
         .execute::<PrimitiveArray>(ctx)?;
-    let rotation = {
-        let rot = RotationMatrix::from_u8_slice(signs_prim.as_slice::<u8>(), dim)?;
-        if let Some(ref p) = perm {
-            rot.with_permutation(p.clone())
-        } else {
-            rot
-        }
-    };
+    let rotation = RotationMatrix::from_u8_slice(signs_prim.as_slice::<u8>(), dim)?;
 
     // Unpack codes from FixedSizeListArray → flat u8 elements.
     let codes_fsl = array.codes().clone().execute::<FixedSizeListArray>(ctx)?;
@@ -129,14 +113,7 @@ pub fn execute_decompress(
     let residual_norms = residual_norms_prim.as_slice::<f32>();
 
     let qjl_rot_signs_prim = qjl.rotation_signs.clone().execute::<PrimitiveArray>(ctx)?;
-    let qjl_rot = {
-        let rot = RotationMatrix::from_u8_slice(qjl_rot_signs_prim.as_slice::<u8>(), dim)?;
-        if let Some(ref p) = perm {
-            rot.with_permutation(p.clone())
-        } else {
-            rot
-        }
-    };
+    let qjl_rot = RotationMatrix::from_u8_slice(qjl_rot_signs_prim.as_slice::<u8>(), dim)?;
 
     let qjl_scale = qjl_correction_scale(padded_dim);
     let mse_elements = mse_output.as_ref();

vortex-tensor/src/encodings/turboquant/mod.rs

@@ -415,10 +415,9 @@ mod tests {
 
         // For power-of-2 dims, QJL bias should be small (< 0.15).
         // For non-power-of-2 dims (e.g., 768 padded to 1024), the bias is
-        // larger due to distributional mismatch: the zero-padded vector has
-        // fewer effective nonzero terms per SRHT coordinate, changing the
-        // kurtosis. The pre-SRHT permutation helps with butterfly alignment
-        // but does not fully resolve this; dimension-aware centroids would.
+        // inherently larger because the SRHT centroids are optimized for the
+        // padded dimension's coordinate distribution, which differs from the
+        // actual distribution of a zero-padded lower-dimensional vector.
         let threshold = if dim.is_power_of_two() { 0.15 } else { 0.25 };
         assert!(
             mean_rel_error.abs() < threshold,