chore: remove custom cast_with_options in favor of upstream (#6344)

`arrow-cast` 57.0.0 added native support for `FixedSizeList →
FixedSizeList` casting, which was the only reason
`lance_arrow::cast::cast_with_options` existed. This removes the wrapper
and updates all call sites to use `arrow_cast::cast_with_options`
directly.

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: wjones127

Cargo.lock

@@ -15,6 +15,7 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow = { version = "57.0.0", features = ["pyarrow"] }
 arrow-array = "57.0.0"
+arrow-cast = "57.0.0"
 arrow-data = "57.0.0"
 arrow-schema = "57.0.0"
 object_store = "0.12.4"

python/Cargo.lock

@@ -12,6 +12,7 @@ use arrow::ffi_stream::ArrowArrayStreamReader;
 use arrow::pyarrow::*;
 use arrow_array::Array;
 use arrow_array::{RecordBatch, RecordBatchReader, make_array};
+use arrow_cast::cast_with_options;
 use arrow_data::ArrayData;
 use arrow_schema::{DataType, Schema as ArrowSchema};
 use async_trait::async_trait;
@@ -3610,14 +3611,13 @@ fn prepare_vector_index_params(
             // as the vectors that will be indexed.
             let mut centroids: Arc<dyn Array> = batch.column(0).clone();
             if centroids.data_type() != column_type {
-                centroids = lance_arrow::cast::cast_with_options(
-                    centroids.as_ref(),
-                    column_type,
-                    &Default::default(),
-                )
-                .map_err(|e| {
-                    PyValueError::new_err(format!("Failed to cast centroids to column type: {}", e))
-                })?;
+                centroids = cast_with_options(centroids.as_ref(), column_type, &Default::default())
+                    .map_err(|e| {
+                        PyValueError::new_err(format!(
+                            "Failed to cast centroids to column type: {}",
+                            e
+                        ))
+                    })?;
             }
             let centroids = as_fixed_size_list_array(centroids.as_ref());
 

python/Cargo.toml

@@ -30,7 +30,6 @@ pub mod floats;
 use crate::list::ListArrayExt;
 pub use floats::*;
 
-pub mod cast;
 pub mod ipc;
 pub mod json;
 pub mod list;

python/src/dataset.rs

@@ -13,6 +13,7 @@ version.workspace = true
 arrow = {workspace = true, features = ["ffi"]}
 arrow-array = {workspace = true, features = ["ffi"]}
 arrow-buffer.workspace = true
+arrow-cast.workspace = true
 arrow-ord.workspace = true
 arrow-schema.workspace = true
 arrow-select.workspace = true

rust/lance-arrow/src/cast.rs

@@ -14,6 +14,7 @@ use crate::sql::{parse_sql_expr, parse_sql_filter};
 use arrow::compute::CastOptions;
 use arrow_array::ListArray;
 use arrow_buffer::OffsetBuffer;
+use arrow_cast::cast_with_options;
 use arrow_schema::{DataType as ArrowDataType, Field, SchemaRef, TimeUnit};
 use arrow_select::concat::concat;
 use datafusion::common::DFSchema;
@@ -45,7 +46,6 @@ use datafusion::{
     scalar::ScalarValue,
 };
 use datafusion_functions::core::getfield::GetFieldFunc;
-use lance_arrow::cast::cast_with_options;
 use lance_core::datatypes::Schema;
 use lance_core::error::LanceOptionExt;
 

rust/lance-arrow/src/lib.rs

@@ -30,6 +30,7 @@ lance-geo = { workspace = true, optional = true }
 arrow-arith = { workspace = true }
 arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
+arrow-cast = { workspace = true }
 arrow-ipc = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-row = { workspace = true }

rust/lance-datafusion/Cargo.toml

@@ -12,6 +12,7 @@ use crate::{Error, Result, io::exec::Planner};
 use arrow::compute::CastOptions;
 use arrow::compute::can_cast_types;
 use arrow_array::{Array, RecordBatch, RecordBatchReader};
+use arrow_cast::cast_with_options;
 use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
 use datafusion::execution::SendableRecordBatchStream;
 use futures::stream::{StreamExt, TryStreamExt};
@@ -635,7 +636,7 @@ pub(super) async fn alter_columns(
             let mut columns = Vec::with_capacity(batch.num_columns());
             for (old, new) in &cast_fields {
                 let old_column = batch[&old.name].clone();
-                let new_column = lance_arrow::cast::cast_with_options(
+                let new_column = cast_with_options(
                     &old_column,
                     &new.data_type(),
                     // Safe: false means it will error if the cast is lossy.
@@ -1963,7 +1964,7 @@ mod test {
                 Arc::new(Float16Array::from_iter_values(
                     (0..nrows).map(|i| f16::from_f32(i as f32)),
                 )),
-                lance_arrow::cast::cast_with_options(
+                cast_with_options(
                     batch["vec"].as_ref(),
                     &DataType::FixedSizeList(
                         Arc::new(ArrowField::new("item", DataType::Float16, true)),