feat: add append functions to ffi (delta-io#962)

This PR is a retake of
delta-io#550

## What changes are proposed in this pull request?
This PR implements the ffi functions required to do appends. The main
difference over delta-io#550 is
how it's tested: instead of using a C based example we now opt for a
rust-based test that is able to re-used various testing-related utility
functions that are used in the kernel tests. To do this cleanly, i've
moved several functions over to the test-utils crate.

## This PR affects the following public APIs
- Adds new ffi functions related to appends / transactions

## How was this change tested?
Using the `test_basic_append` in `ffi/src/transaction/mod.rs`. 


## Main points to comment on for reviewers of Draft PR:
I would like to draw the reviewer's attention to:
- the concept of testing new ffi functionality through rust tests
instead of a C example program
- usage of `tempfile::tempdir` to create a tmp directory for test data
during the test
- some outstanding `TODO`s in the `test_basic_append` test

Once the current approach is approved by reviewers, I can follow up with
more testing:
- the write schema
- appending to a partitioned table

Author: samansmink

ffi/Cargo.toml

@@ -38,6 +38,8 @@ serde_json = "1.0.140"
 test_utils = { path = "../test-utils" }
 tokio = { version = "1.44" }
 trybuild = "1.0"
+tempfile = "3.20.0"
+itertools = "0.14.0"
 
 [features]
 default = ["default-engine"]

ffi/src/engine_data.rs

@@ -1,11 +1,15 @@
 //! EngineData related ffi code
 
+#[cfg(feature = "default-engine-base")]
+use delta_kernel::arrow;
 #[cfg(feature = "default-engine-base")]
 use delta_kernel::arrow::array::{
     ffi::{FFI_ArrowArray, FFI_ArrowSchema},
-    ArrayData, StructArray,
+    ArrayData, RecordBatch, StructArray,
 };
 #[cfg(feature = "default-engine-base")]
+use delta_kernel::engine::arrow_data::ArrowEngineData;
+#[cfg(feature = "default-engine-base")]
 use delta_kernel::DeltaResult;
 use delta_kernel::EngineData;
 use std::ffi::c_void;
@@ -82,7 +86,7 @@ pub unsafe extern "C" fn get_raw_arrow_data(
 fn get_raw_arrow_data_impl(data: Box<dyn EngineData>) -> DeltaResult<*mut ArrowFFIData> {
     let record_batch: delta_kernel::arrow::array::RecordBatch = data
         .into_any()
-        .downcast::<delta_kernel::engine::arrow_data::ArrowEngineData>()
+        .downcast::<ArrowEngineData>()
         .map_err(|_| delta_kernel::Error::EngineDataType("ArrowEngineData".to_string()))?
         .into();
     let sa: StructArray = record_batch.into();
@@ -93,3 +97,35 @@ fn get_raw_arrow_data_impl(data: Box<dyn EngineData>) -> DeltaResult<*mut ArrowF
     let ret_data = Box::new(ArrowFFIData { array, schema });
     Ok(Box::leak(ret_data))
 }
+
+/// Creates engine data from Arrow C Data Interface array and schema.
+///
+/// Converts the provided Arrow C Data Interface array and schema into delta-kernel's internal
+/// engine data format. Note that ownership of the array is transferred to the kernel, whereas the
+/// ownership of the schema stays the engine's.
+///
+/// # Safety
+/// - `array` must be a valid FFI_ArrowArray
+/// - `schema` must be a valid pointer to a FFI_ArrowSchema
+/// - `engine` must be a valid Handle to a SharedExternEngine
+#[cfg(feature = "default-engine-base")]
+#[no_mangle]
+pub unsafe extern "C" fn get_engine_data(
+    array: FFI_ArrowArray,
+    schema: &FFI_ArrowSchema,
+    engine: Handle<SharedExternEngine>,
+) -> ExternResult<Handle<ExclusiveEngineData>> {
+    get_engine_data_impl(array, schema).into_extern_result(&engine.as_ref())
+}
+
+#[cfg(feature = "default-engine-base")]
+unsafe fn get_engine_data_impl(
+    array: FFI_ArrowArray,
+    schema: &FFI_ArrowSchema,
+) -> DeltaResult<Handle<ExclusiveEngineData>> {
+    let array_data = unsafe { arrow::array::ffi::from_ffi(array, schema) };
+    let record_batch: RecordBatch = StructArray::from(array_data?).into();
+    let arrow_engine_data: ArrowEngineData = record_batch.into();
+    let engine_data: Box<dyn EngineData> = Box::new(arrow_engine_data);
+    Ok(engine_data.into())
+}

ffi/src/engine_funcs.rs

@@ -223,7 +223,7 @@ mod tests {
 
     #[test]
     fn test_new_expression_evaluator() {
-        let engine = get_default_engine();
+        let engine = get_default_engine("memory:///doesntmatter/foo");
         let in_schema = Arc::new(StructType::new(vec![StructField::new(
             "a",
             DataType::LONG,

ffi/src/lib.rs

@@ -48,6 +48,7 @@ pub mod schema;
 mod ffi_test_utils;
 #[cfg(feature = "test-ffi")]
 pub mod test_ffi;
+pub mod transaction;
 
 pub(crate) type NullableCvoid = Option<NonNull<c_void>>;
 
@@ -810,16 +811,15 @@ mod tests {
         }
     }
 
-    pub(crate) fn get_default_engine() -> Handle<SharedExternEngine> {
-        let path = "memory:///doesntmatter/foo";
+    pub(crate) fn get_default_engine(path: &str) -> Handle<SharedExternEngine> {
         let path = kernel_string_slice!(path);
         let builder = unsafe { ok_or_panic(get_engine_builder(path, allocate_err)) };
         unsafe { ok_or_panic(builder_build(builder)) }
     }
 
     #[test]
     fn engine_builder() {
-        let engine = get_default_engine();
+        let engine = get_default_engine("memory:///doesntmatter/foo");
         unsafe {
             free_engine(engine);
         }