ffi: StructArray creation (#7260)

StructArrays are needed for Scan API to test filtering and
column selection from C side.

Signed-off-by: Mikhail Kot <to@myrrc.dev>

Author: myrrc

Cargo.lock

@@ -34,6 +34,7 @@ vortex = { workspace = true, features = ["object_store"] }
 
 [dev-dependencies]
 tempfile = { workspace = true }
+vortex-array = { workspace = true, features = ["_test-harness"] }
 
 [build-dependencies]
 cbindgen = { workspace = true }

vortex-ffi/Cargo.toml

@@ -413,6 +413,8 @@ typedef struct vx_session vx_session;
  */
 typedef struct vx_string vx_string;
 
+typedef struct vx_struct_column_builder vx_struct_column_builder;
+
 /**
  * Represents a Vortex struct data type, without top-level nullability.
  */
@@ -588,7 +590,7 @@ const vx_array *vx_array_new_null(size_t len);
  *
  * Example:
  *
- * const vx_error* error = nullptr;
+ * const vx_error* error = NULL;
  * vx_validity validity = {};
  * validity.type = VX_VALIDITY_NON_NULLABLE;
  * uint32_t buffer[] = {1, 2, 3};
@@ -1105,6 +1107,57 @@ size_t vx_string_len(const vx_string *ptr);
  */
 const char *vx_string_ptr(const vx_string *ptr);
 
+/**
+ * Free an owned [`vx_struct_column_builder`] object.
+ */
+void vx_struct_column_builder_free(vx_struct_column_builder *ptr);
+
+/**
+ * Create a new column-wise struct array builder with given validity and a
+ * capacity hint. validity can't be NULL.
+ * If you don't know capacity, pass 0.
+ * if validity is NULL, returns NULL.
+ */
+vx_struct_column_builder *vx_struct_column_builder_new(const vx_validity *validity, size_t capacity);
+
+/**
+ * Add a named field to a struct array builder.
+ * All arguments must be non-NULL.
+ * If field's length doesn't match lengths of previous fields, sets error.
+ * If an error is returned, the builder is still valid, and caller must
+ * deallocate it using vx_struct_column_builder_free.
+ */
+void vx_struct_column_builder_add_field(vx_struct_column_builder *builder,
+                                        const char *name,
+                                        const vx_array *field,
+                                        vx_error **error);
+
+/**
+ * Finalize a struct array builder, returning a struct array.
+ * Consumes the builder. Caller doesn't need to free the builder after calling
+ * this function.
+ *
+ * Example:
+ *
+ * vx_error* error = NULL;
+ *
+ * vx_validity validity = {};
+ * validity.type = VX_VALIDITY_NON_NULLABLE;
+ *
+ * const vx_array* field_array = vx_array_new_null(5);
+ * const vx_struct_column_builder* builder =
+ *     vx_struct_column_builder_new(&validity, 1);
+ *
+ * vx_struct_column_builder_add_field(builder, "age", field_array, &error);
+ *
+ * vx_array* struct_array = vx_struct_column_builder_finalize(builder, &error);
+ *
+ * vx_array_free(struct_array);
+ * vx_array_free(field_array);
+ *
+ */
+const vx_array *vx_struct_column_builder_finalize(vx_struct_column_builder *builder, vx_error **error);
+
 /**
  * Free an owned [`vx_struct_fields`] object.
  */

vortex-ffi/cinclude/vortex.h

@@ -118,11 +118,11 @@ pub enum vx_validity_type {
 
 #[repr(C)]
 pub struct vx_validity {
-    r#type: vx_validity_type,
+    pub r#type: vx_validity_type,
     /// If type is not VX_VALIDITY_ARRAY, this is NULL.
     /// If type is VX_VALIDITY_ARRAY, this is set to an owned boolean validity
     /// array which must be freed by the caller.
-    array: *const vx_array,
+    pub array: *const vx_array,
 }
 
 impl From<&vx_validity> for Validity {
@@ -281,7 +281,7 @@ unsafe fn primitive_from_raw<T: vortex::dtype::NativePType>(
 ///
 /// Example:
 ///
-/// const vx_error* error = nullptr;
+/// const vx_error* error = NULL;
 /// vx_validity validity = {};
 /// validity.type = VX_VALIDITY_NON_NULLABLE;
 /// uint32_t buffer[] = {1, 2, 3};
@@ -425,7 +425,6 @@ mod tests {
     use vortex::array::IntoArray;
     use vortex::array::arrays::BoolArray;
     use vortex::array::arrays::PrimitiveArray;
-    use vortex::array::arrays::StructArray;
     use vortex::array::arrays::VarBinViewArray;
     use vortex::array::validity::Validity;
     use vortex::buffer::buffer;
@@ -434,15 +433,28 @@ mod tests {
     use vortex::expr::eq;
     use vortex::expr::lit;
     use vortex::expr::root;
+    use vortex_array::arrays::StructArray;
 
     use crate::array::*;
     use crate::binary::vx_binary_free;
     use crate::dtype::vx_dtype_get_variant;
     use crate::dtype::vx_dtype_variant;
     use crate::error::vx_error_free;
+    use crate::error::vx_error_get_message;
     use crate::expression::vx_expression_free;
     use crate::string::vx_string_free;
 
+    fn assert_no_error(error: *mut vx_error) {
+        if !error.is_null() {
+            let message;
+            unsafe {
+                message = vx_string::as_str(vx_error_get_message(error)).to_owned();
+                vx_error_free(error);
+            }
+            panic!("{message}");
+        }
+    }
+
     #[test]
     // TODO(joe): enable once this is fixed https://github.com/Amanieu/parking_lot/issues/477
     #[cfg_attr(miri, ignore)]
@@ -491,7 +503,7 @@ mod tests {
 
             let mut error = ptr::null_mut();
             let sliced = vx_array_slice(ffi_array, 1, 4, &raw mut error);
-            assert!(error.is_null());
+            assert_no_error(error);
             assert_eq!(vx_array_len(sliced), 3);
             assert_eq!(vx_array_get_i32(sliced, 0), 2);
             assert_eq!(vx_array_get_i32(sliced, 1), 3);
@@ -515,14 +527,14 @@ mod tests {
 
             let mut error = ptr::null_mut();
             assert!(!vx_array_element_is_invalid(ffi_array, 0, &raw mut error));
-            assert!(error.is_null());
+            assert_no_error(error);
             assert!(vx_array_element_is_invalid(ffi_array, 1, &raw mut error));
-            assert!(error.is_null());
+            assert_no_error(error);
             assert!(!vx_array_element_is_invalid(ffi_array, 2, &raw mut error));
-            assert!(error.is_null());
+            assert_no_error(error);
 
             let null_count = vx_array_invalid_count(ffi_array, &raw mut error);
-            assert!(error.is_null());
+            assert_no_error(error);
             assert_eq!(null_count, 1);
 
             vx_array_free(ffi_array);
@@ -547,11 +559,11 @@ mod tests {
 
             let mut error = ptr::null_mut();
             let field0 = vx_array_get_field(ffi_array, 0, &raw mut error);
-            assert!(error.is_null());
+            assert_no_error(error);
             assert_eq!(vx_array_len(field0), 3);
 
             let field1 = vx_array_get_field(ffi_array, 1, &raw mut error);
-            assert!(error.is_null());
+            assert_no_error(error);
             assert_eq!(vx_array_len(field1), 3);
             assert_eq!(vx_array_get_u8(field1, 0), 30);
             assert_eq!(vx_array_get_u8(field1, 1), 25);
@@ -592,7 +604,7 @@ mod tests {
                 &raw const validity,
                 &raw mut error,
             );
-            assert!(error.is_null());
+            assert_no_error(error);
             assert!(!ffi_i32.is_null());
 
             assert!(vx_array_is_primitive(ffi_i32, vx_ptype::PTYPE_I32));
@@ -610,7 +622,7 @@ mod tests {
                 &raw const validity,
                 &raw mut error,
             );
-            assert!(error.is_null());
+            assert_no_error(error);
             assert!(!ffi_u64.is_null());
             assert!(vx_array_is_primitive(ffi_u64, vx_ptype::PTYPE_U64));
             assert_eq!(vx_array_get_u64(ffi_u64, 0), u64::MAX);
@@ -627,7 +639,7 @@ mod tests {
                 &raw const validity,
                 &raw mut error,
             );
-            assert!(error.is_null());
+            assert_no_error(error);
             assert!(!ffi_f64.is_null());
             assert!(vx_array_is_primitive(ffi_f64, vx_ptype::PTYPE_F64));
             assert_eq!(vx_array_get_f64(ffi_f64, 0), f64::NEG_INFINITY);
@@ -646,7 +658,7 @@ mod tests {
                     &raw const validity,
                     &raw mut error,
                 );
-                assert!(error.is_null());
+                assert_no_error(error);
                 assert!(!ffi_f16.is_null());
                 assert_eq!(vx_array_get_f16(ffi_f16, 0), f16::from_f32(1.0));
                 assert_eq!(vx_array_get_f16(ffi_f16, 1), f16::from_f32(-0.5));
@@ -746,8 +758,8 @@ mod tests {
             vx_error_free(error);
 
             let res = vx_array_apply(array, expression, &raw mut error);
+            assert_no_error(error);
             assert!(!res.is_null());
-            assert!(error.is_null());
             {
                 let res = vx_array::as_ref(res);
                 let buffer = res.to_bool().to_bit_buffer();

vortex-ffi/src/array.rs

@@ -9,6 +9,7 @@ use std::slice;
 use std::sync::Arc;
 
 use vortex::dtype::FieldName;
+use vortex::error::VortexExpect;
 use vortex::expr::Expression;
 use vortex::expr::and_collect;
 use vortex::expr::get_item;
@@ -22,6 +23,8 @@ use vortex::scalar_fn::ScalarFnVTableExt;
 use vortex::scalar_fn::fns::binary::Binary;
 use vortex::scalar_fn::fns::operators::Operator;
 
+use crate::to_field_names;
+
 // Expressions are Arc'ed inside
 crate::box_wrapper!(
     /// A node in a Vortex expression tree.
@@ -80,19 +83,8 @@ pub unsafe extern "C" fn vx_expression_select(
     if child.is_null() {
         return ptr::null_mut();
     }
-
-    #[expect(clippy::expect_used)]
-    let names: Vec<FieldName> = (0..len)
-        .map(|i| unsafe {
-            let name = *names.offset(i.try_into().expect("pointer offset overflow"));
-            let name = CStr::from_ptr(name)
-                .to_str()
-                .expect("converting pointer to str");
-            let name: Arc<str> = Arc::from(name);
-            name.into()
-        })
-        .collect();
-
+    let names =
+        unsafe { to_field_names(names, len) }.vortex_expect("converting names to field names");
     let expr = select(names, vx_expression::as_ref(child).clone());
     vx_expression::new(Box::new(expr))
 }

vortex-ffi/src/expression.rs

@@ -19,13 +19,19 @@ mod ptype;
 mod session;
 mod sink;
 mod string;
+mod struct_array;
 mod struct_fields;
 
 use std::ffi::CStr;
 use std::ffi::c_char;
+use std::sync::Arc;
 use std::sync::LazyLock;
 
 pub use log::vx_log_level;
+use vortex::dtype::FieldName;
+use vortex::error::VortexExpect;
+use vortex::error::VortexResult;
+use vortex::error::vortex_err;
 use vortex::io::runtime::current::CurrentThreadRuntime;
 
 #[cfg(all(feature = "mimalloc", not(miri)))]
@@ -49,3 +55,25 @@ pub(crate) unsafe fn to_string_vec(ptr: *const *const c_char, len: usize) -> Vec
         })
         .collect()
 }
+
+/// SAFETY: name must be a non-NULL pointer
+pub(crate) unsafe fn to_field_name(name: *const c_char) -> VortexResult<FieldName> {
+    let name = unsafe { CStr::from_ptr(name) }
+        .to_str()
+        .map_err(|e| vortex_err!("{e}"))?;
+    let name: Arc<str> = Arc::from(name);
+    Ok(name.into())
+}
+
+/// SAFETY: names must be a non-NULL pointer valid for reads up to len.
+pub(crate) unsafe fn to_field_names(
+    names: *const *const c_char,
+    len: usize,
+) -> VortexResult<Vec<FieldName>> {
+    (0..len)
+        .map(|i| unsafe {
+            let name = *names.offset(i.try_into().vortex_expect("pointer offset overflow"));
+            to_field_name(name)
+        })
+        .collect()
+}