feat: add metadata bridge for Python CUDA export (#8604)

Add a private metadata-only bridge on PyVortex arrays so the optional
CUDA extension can reconstruct arrays through its own local Vortex
session instead of passing Rust ArrayRef values across Python extension
modules.

The CUDA extension now parses the metadata tree, calls local array
plugins for deserialization, and exports bufferless arrays through the
Arrow C Device capsule path. Physical buffer handoff remains a
follow-up.

---------

Signed-off-by: Alexander Droste <alexander.droste@protonmail.com>

Author: 0ax1

Cargo.lock

@@ -543,14 +543,14 @@ fn released_device_array(device_id: i64) -> ArrowDeviceArray {
 }
 
 /// Release an Arrow C schema if it is live.
-fn release_schema(schema: &mut FFI_ArrowSchema) {
+pub fn release_schema(schema: &mut FFI_ArrowSchema) {
     if let Some(release) = schema.release {
         unsafe { release(schema) };
     }
 }
 
 /// Release an Arrow device array if it is live.
-fn release_device_array(array: &mut ArrowDeviceArray) {
+pub fn release_device_array(array: &mut ArrowDeviceArray) {
     if let Some(release) = array.array.release {
         unsafe { release(&raw mut array.array) };
     }

vortex-cuda/src/arrow/mod.rs

@@ -27,5 +27,7 @@ default = ["extension-module"]
 extension-module = []
 
 [dependencies]
+arrow-schema = { workspace = true }
 pyo3 = { workspace = true, features = ["abi3", "abi3-py311"] }
+vortex = { workspace = true }
 vortex-cuda = { workspace = true }

vortex-python-cuda/Cargo.toml

@@ -4,8 +4,9 @@ name = "vortex-data-cuda"
 dynamic = ["version", "description", "authors"]
 requires-python = ">= 3.11"
 # The CUDA extension package must exactly match the base package version because the two extensions
-# exchange Vortex IPC buffers. Keep this in sync with the workspace package version; release
-# automation may rewrite it when publishing a non-workspace version.
+# exchange private Vortex vtable metadata. Future buffer-handoff support will extend this private
+# bridge. Keep this in sync with the workspace package version; release automation may rewrite it
+# when publishing a non-workspace version.
 dependencies = ["vortex-data==0.1.0"]
 classifiers = [
     "Development Status :: 4 - Beta",

vortex-python-cuda/pyproject.toml

@@ -1,6 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-from ._lib import cuda_available  # pyright: ignore[reportMissingModuleSource]
+from ._lib import (  # pyright: ignore[reportMissingModuleSource]
+    _debug_array_metadata_dtype as _debug_array_metadata_dtype,  # pyright: ignore[reportPrivateUsage]
+)
+from ._lib import (  # pyright: ignore[reportMissingModuleSource]
+    cuda_available,
+    export_device_array,
+)
 
-__all__ = ["cuda_available"]
+__all__ = ["cuda_available", "export_device_array"]

vortex-python-cuda/python/vortex_cuda/__init__.py

@@ -1,4 +1,8 @@
 #  SPDX-License-Identifier: Apache-2.0
 #  SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+def _debug_array_metadata_dtype(array: object) -> str: ...
 def cuda_available() -> bool: ...
+def export_device_array(
+    array: object, requested_schema: object | None = None, **kwargs: object
+) -> tuple[object, object]: ...