Invert thevortex-tensor dependency (#7271)

## Summary

We want to start using the tensor types, but as very unstable encodings
and types. This PR makes `vortex-tensor` a dependency of both `vortex`
and `vortex-file`, gated under the `unstable_encodings` flag
(technically there are also unstable types and expressions here too bet
that isn't that important).

This will let us write unstable types and encodings (like Vectors with
TurboQuant) to files easily.

## API Changes

The dependency direction has changed.

## Testing

N/A

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>

Author: connortsui20

Cargo.lock

@@ -54,6 +54,7 @@ vortex-scan = { workspace = true }
 vortex-sequence = { workspace = true }
 vortex-session = { workspace = true }
 vortex-sparse = { workspace = true }
+vortex-tensor = { workspace = true, optional = true }
 vortex-utils = { workspace = true, features = ["dashmap"] }
 vortex-zigzag = { workspace = true }
 vortex-zstd = { workspace = true, optional = true }
@@ -79,6 +80,7 @@ tokio = [
 zstd = ["dep:vortex-zstd", "vortex-btrblocks/zstd", "vortex-btrblocks/pco"]
 # This feature enables unstable encodings for which we don't guarantee stability.
 unstable_encodings = [
+    "dep:vortex-tensor",
     "vortex-zstd?/unstable_encodings",
     "vortex-btrblocks/unstable_encodings",
 ]

vortex-file/Cargo.toml

@@ -178,4 +178,7 @@ pub fn register_default_encodings(session: &VortexSession) {
     vortex_fastlanes::initialize(session);
     vortex_runend::initialize(session);
     vortex_sequence::initialize(session);
+
+    #[cfg(feature = "unstable_encodings")]
+    vortex_tensor::initialize(session);
 }

vortex-file/src/lib.rs

@@ -17,11 +17,14 @@ version = { workspace = true }
 workspace = true
 
 [dependencies]
-vortex = { workspace = true }
+vortex-array = { workspace = true }
+vortex-error = { workspace = true }
+vortex-session = { workspace = true }
 
 itertools = { workspace = true }
 num-traits = { workspace = true }
 prost = { workspace = true }
 
 [dev-dependencies]
 rstest = { workspace = true }
+vortex-buffer = { workspace = true }

vortex-tensor/Cargo.toml

@@ -259,3 +259,5 @@ pub fn vortex_tensor::vector::Vector::serialize_metadata(&self, _metadata: &Self
 pub fn vortex_tensor::vector::Vector::unpack_native<'a>(_ext_dtype: &'a vortex_array::dtype::extension::typed::ExtDType<Self>, storage_value: &'a vortex_array::scalar::scalar_value::ScalarValue) -> vortex_error::VortexResult<Self::NativeValue>
 
 pub fn vortex_tensor::vector::Vector::validate_dtype(ext_dtype: &vortex_array::dtype::extension::typed::ExtDType<Self>) -> vortex_error::VortexResult<()>
+
+pub fn vortex_tensor::initialize(session: &vortex_session::VortexSession)

vortex-tensor/public-api.lock

@@ -4,10 +4,10 @@
 use std::fmt;
 
 use itertools::Either;
-use vortex::error::VortexExpect;
-use vortex::error::VortexResult;
-use vortex::error::vortex_ensure;
-use vortex::error::vortex_ensure_eq;
+use vortex_error::VortexExpect;
+use vortex_error::VortexResult;
+use vortex_error::vortex_ensure;
+use vortex_error::vortex_ensure_eq;
 
 /// Metadata for a `FixedShapeTensor` extension type.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]

vortex-tensor/src/fixed_shape/metadata.rs

@@ -4,9 +4,9 @@
 //! Protobuf serialization for [`FixedShapeTensorMetadata`].
 
 use prost::Message;
-use vortex::error::VortexExpect;
-use vortex::error::VortexResult;
-use vortex::error::vortex_err;
+use vortex_error::VortexExpect;
+use vortex_error::VortexResult;
+use vortex_error::vortex_err;
 
 use crate::fixed_shape::FixedShapeTensorMetadata;
 

vortex-tensor/src/fixed_shape/proto.rs

@@ -1,15 +1,15 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use vortex::dtype::DType;
-use vortex::dtype::extension::ExtDType;
-use vortex::dtype::extension::ExtId;
-use vortex::dtype::extension::ExtVTable;
-use vortex::error::VortexResult;
-use vortex::error::vortex_bail;
-use vortex::error::vortex_ensure;
-use vortex::error::vortex_ensure_eq;
-use vortex::scalar::ScalarValue;
+use vortex_array::dtype::DType;
+use vortex_array::dtype::extension::ExtDType;
+use vortex_array::dtype::extension::ExtId;
+use vortex_array::dtype::extension::ExtVTable;
+use vortex_array::scalar::ScalarValue;
+use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
+use vortex_error::vortex_ensure;
+use vortex_error::vortex_ensure_eq;
 
 use crate::fixed_shape::FixedShapeTensor;
 use crate::fixed_shape::FixedShapeTensorMetadata;
@@ -77,8 +77,8 @@ impl ExtVTable for FixedShapeTensor {
 #[cfg(test)]
 mod tests {
     use rstest::rstest;
-    use vortex::dtype::extension::ExtVTable;
-    use vortex::error::VortexResult;
+    use vortex_array::dtype::extension::ExtVTable;
+    use vortex_error::VortexResult;
 
     use crate::fixed_shape::FixedShapeTensor;
     use crate::fixed_shape::FixedShapeTensorMetadata;

vortex-tensor/src/fixed_shape/vtable.rs

@@ -5,6 +5,15 @@
 //! including unit vectors, spherical coordinates, and similarity measures such as cosine
 //! similarity.
 
+use vortex_array::dtype::session::DTypeSessionExt;
+use vortex_array::scalar_fn::session::ScalarFnSessionExt;
+use vortex_session::VortexSession;
+
+use crate::fixed_shape::FixedShapeTensor;
+use crate::scalar_fns::cosine_similarity::CosineSimilarity;
+use crate::scalar_fns::l2_norm::L2Norm;
+use crate::vector::Vector;
+
 pub mod matcher;
 pub mod scalar_fns;
 
@@ -14,3 +23,11 @@ pub mod vector;
 pub mod encodings;
 
 mod utils;
+
+/// Registers the tensor extension dtypes and scalar functions with the given session.
+pub fn initialize(session: &VortexSession) {
+    session.dtypes().register(Vector);
+    session.dtypes().register(FixedShapeTensor);
+    session.scalar_fns().register(CosineSimilarity);
+    session.scalar_fns().register(L2Norm);
+}

vortex-tensor/src/lib.rs

@@ -3,8 +3,8 @@
 
 //! Matcher for tensor-like extension types.
 
-use vortex::dtype::extension::ExtDTypeRef;
-use vortex::dtype::extension::Matcher;
+use vortex_array::dtype::extension::ExtDTypeRef;
+use vortex_array::dtype::extension::Matcher;
 
 use crate::fixed_shape::FixedShapeTensor;
 use crate::fixed_shape::FixedShapeTensorMetadata;