better

Author: myrrc

vortex-ffi/CMakeLists.txt

@@ -79,10 +79,10 @@ Static library path ${LIBRARY_PATH}
 Headers path ${LIBRARY_HEADERS}")
 
 if (NOT EXISTS "${LIBRARY_PATH_SHARED}")
-    message(FATAL_ERROR "Shared library not found")
+    message(FATAL_ERROR "Shared library not found, run `cargo build --release -p vortex-ffi`")
 endif()
 if (NOT EXISTS "${LIBRARY_PATH}")
-    message(FATAL_ERROR "Static library not found")
+    message(FATAL_ERROR "Static library not found, run `cargo build --release -p vortex-ffi`")
 endif()
 
 add_library(vortex_ffi STATIC IMPORTED)

vortex-ffi/README.md

@@ -1,58 +1,47 @@
-# Foreign Function Interface
-
-Vortex is a file format that can be used by any execution engine. Nearly every
-programming language supports the C ABI (Application Binary Interface), so by
-providing an FFI interface to work with Vortex objects we can make it easy to
-support a variety of languages.
-
-Check out the [`examples`](./examples/) directory to see an example of how to
-use scanning API from C to read vortex files.
-Full FFI API is documented in `docs/api/c` and in `cinclude/vortex.h` header.
-
+# Vortex C interface
 ## Usage from a CMake project
 
-1. Build .so/.dylib/.dll
-
-```sh
-cargo build --release -p vortex-ffi
 ```
+# in vortex folder
+cargo build --release -p vortex-ffi
 
-2. Add shared or static library
-
-```cmake
 # in your CMakeLists.txt
 include_directory(vortex/vortex-ffi)
 target_link_libraries(my_target, vortex_ffi_shared)
 # or target_link_libraries(my_target, vortex_ffi)
 ```
 
-## Updating Headers
-
-To rebuild the header file:
+## Running C examples:
 
 ```sh
-cargo +nightly build -p vortex-ffi
+cmake -Bbuild -DBUILD_EXAMPLES=1
+cmake --build build
+./build/examples/dtype
+./build/examples/scan
+./build/examples/scan_to_arrow
 ```
 
-Header generation uses cbindgen's macro expansion feature which requires nightly.
-Stable builds use header file at `cinclude/vortex.h`.
+## Updating Headers
+
+If you're developing FFI and want to rebuild `cinclude/vortex.h`, run
+`cargo +nightly build -p vortex-ffi`.
 
 ## Testing C part
 
-Build the test library
+Build the test library:
 
 ```sh
-cmake -Bbuild
-cmake --build build -j $(nproc)
+cmake -Bbuild -DBUILD_TESTS=1
+cmake --build build
 ```
 
-Run the tests
+Run the tests:
 
 ```sh
 ctest --test-dir build -j $(nproc)
 ```
 
-You would need C++ compiler toolchain to run the tests since they use Catch2.
+You will need C++ compiler toolchain to run the tests since they use Catch2.
 
 ## Testing Rust part with sanitizers
 
@@ -92,7 +81,7 @@ with sanitizers.
 - `allow-abi-mismatch` is safe because in our dependency graph only crates like
   `compiler_builtins` unset sanitization, and they do it on purpose.
 - Make sure to use `cargo test` and not `cargo nextest` as nextest reports less
-leaks.
+  leaks.
 - If you want stack trace symbolization, install `llvm-symbolizer`.
 
 ## Testing Rust and C with sanitizers
@@ -102,10 +91,10 @@ leaks.
 ```sh
 RUSTFLAGS="-Zsanitizer=address -Zexternal-clangrt" \
 cargo +nightly build -Zbuild-std --target=<target triple> \
---no-default-features -p vortex-ffi
+    --no-default-features -p vortex-ffi
 ```
 
-2. Build tests with target triple
+2. Build tests with target triple:
 
 ```sh
 cmake -Bbuild -DWITH_ASAN=1 -DTARGET_TRIPLE=<target triple>
@@ -116,11 +105,3 @@ cmake -Bbuild -DWITH_ASAN=1 -DTARGET_TRIPLE=<target triple>
 ```sh
 ./build/test/vortex_ffi_test 2>& 1 | rustfilt -i-
 ```
-
-## Running C example
-
-```sh
-cmake -Bbuild -DBUILD_EXAMPLES=1
-cmake --build build
-./build/examples/vortex_scan
-```

vortex-ffi/cinclude/vortex.h

@@ -175,10 +175,19 @@ typedef enum {
 } vx_validity_type;
 
 typedef enum {
-    VX_CARD_UNKNOWN = 0,
-    VX_CARD_ESTIMATE = 1,
-    VX_CARD_MAXIMUM = 2,
-} vx_cardinality;
+    /**
+     * No estimate is available.
+     */
+    VX_ESTIMATE_UNKNOWN = 0,
+    /**
+     * The value in vx_estimate.estimate is exact.
+     */
+    VX_ESTIMATE_EXACT = 1,
+    /**
+     * The value in vx_estimate.estimate is an upper bound.
+     */
+    VX_ESTIMATE_INEXACT = 2,
+} vx_estimate_type;
 
 /**
  * Equalities, inequalities, and boolean operations over possibly null values.
@@ -282,21 +291,6 @@ typedef enum {
     VX_SELECTION_EXCLUDE_RANGE = 2,
 } vx_scan_selection_include;
 
-typedef enum {
-    /**
-     * No estimate is available.
-     */
-    VX_ESTIMATE_UNKNOWN = 0,
-    /**
-     * The value in vx_estimate.estimate is exact.
-     */
-    VX_ESTIMATE_EXACT = 1,
-    /**
-     * The value in vx_estimate.estimate is an upper bound.
-     */
-    VX_ESTIMATE_INEXACT = 2,
-} vx_estimate_type;
-
 /**
  * Physical type enum, represents the in-memory physical layout but might represent a different logical type.
  */
@@ -490,6 +484,10 @@ typedef struct vx_file vx_file;
  */
 typedef struct vx_partition vx_partition;
 
+/**
+ * A scan is a single traversal of a data source with projections and
+ * filters. A scan can be consumed only once.
+ */
 typedef struct vx_scan vx_scan;
 
 /**
@@ -537,13 +535,17 @@ typedef struct {
     const char *paths;
 } vx_data_source_options;
 
+/**
+ * Used for estimating number of partitions in a data source or number of rows
+ * in a partition.
+ */
 typedef struct {
-    vx_cardinality cardinality;
+    vx_estimate_type type;
     /**
-     * Set only when "cardinality" is not VX_CARD_UNKNOWN
+     * Set only when "type" is not VX_ESTIMATE_UNKNOWN.
      */
-    uint64_t rows;
-} vx_data_source_row_count;
+    uint64_t estimate;
+} vx_estimate;
 
 /**
  * Options supplied for opening a file.
@@ -662,18 +664,6 @@ typedef struct {
     bool ordered;
 } vx_scan_options;
 
-/**
- * Used for estimating number of partitions in a data source or number of rows
- * in a partition.
- */
-typedef struct {
-    vx_estimate_type type;
-    /**
-     * Set only when "type" is not VX_ESTIMATE_UNKNOWN.
-     */
-    uint64_t estimate;
-} vx_estimate;
-
 #ifdef __cplusplus
 extern "C" {
 #endif // __cplusplus
@@ -921,7 +911,7 @@ const vx_dtype *vx_data_source_dtype(const vx_data_source *ds);
 /**
  * Write data source's row count estimate into "row_count".
  */
-void vx_data_source_get_row_count(const vx_data_source *ds, vx_data_source_row_count *row_count);
+void vx_data_source_get_row_count(const vx_data_source *ds, vx_estimate *row_count);
 
 /**
  * Clone a borrowed [`vx_dtype`], returning an owned [`vx_dtype`].
@@ -1319,6 +1309,17 @@ vx_partition *vx_scan_next_partition(vx_scan *scan, vx_error **err);
  */
 int vx_partition_row_count(const vx_partition *partition, vx_estimate *count, vx_error **err);
 
+/**
+ * Scan partition to ArrowArrayStream.
+ * Consumes partition fully: subsequent calls to vx_partition_scan_arrow or
+ * vx_partition_next are undefined behaviour.
+ * This call blocks current thread until underlying stream is fully consumed.
+ *
+ * Caller must not free partition after calling this function.
+ *
+ * On success, sets "stream" and returns 0.
+ * On error, sets "err" and returns 1, freeing the partition.
+ */
 int vx_partition_scan_arrow(const vx_session *session,
                             vx_partition *partition,
                             FFI_ArrowArrayStream *stream,

vortex-ffi/examples/CMakeLists.txt

@@ -1,5 +1,14 @@
 # SPDX-License-Identifier: CC-BY-4.0
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+# allow linking with vortex_ffi_shared although it's not in current folder
 cmake_policy(SET CMP0079 NEW)
-add_executable(vortex_scan scan.c)
-target_link_libraries(vortex_scan PRIVATE vortex_ffi_shared)
+
+add_executable(scan scan.c)
+target_link_libraries(scan PRIVATE vortex_ffi_shared)
+
+add_executable(scan_to_arrow scan_to_arrow.c)
+target_link_libraries(scan_to_arrow PRIVATE nanoarrow_shared vortex_ffi_shared)
+
+add_executable(dtype dtype.c)
+target_link_libraries(dtype PRIVATE vortex_ffi_shared)