Merge branch 'develop' into nemo/geo-bounds-zonemap

Author: HarukiMoriarty

.clang-format

@@ -25,7 +25,7 @@ DerivePointerAlignment: false
 IncludeBlocks: Regroup
 InsertBraces: true
 Language: Cpp
-NamespaceIndentation: Inner
+NamespaceIndentation: None
 PointerAlignment: Right
 SpaceBeforeCpp11BracedList: true
 SpaceBeforeCtorInitializerColon: true

.github/workflows/ci.yml

@@ -355,7 +355,7 @@ jobs:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7
       - name: C/C++ Lint - clang-format
         run: |
-          git ls-files vortex-cuda vortex-duckdb vortex-ffi \
+          git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \
             | grep -E '\.(cpp|hpp|cu|cuh|h)$' \
             | grep -v 'arrow/reference/arrow_c_device\.h$' \
             | grep -v 'kernels/src/bit_unpack_.*\.cu$' \
@@ -522,6 +522,40 @@ jobs:
         with:
           command: check ${{ matrix.checks }}
 
+  cxx-api:
+    name: "C++ API"
+    timeout-minutes: 5
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id)
+          || 'ubuntu-latest' }}
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7
+      - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: "true"
+      - name: Install Rust nightly toolchain
+        run: |
+          rustup toolchain install $NIGHTLY_TOOLCHAIN
+          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
+      - name: Build FFI library (asan)
+        run: |
+          RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \
+                -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \
+                -Zsanitizer=address,leak" \
+          cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \
+            --target x86_64-unknown-linux-gnu -Zbuild-std \
+            -p vortex-ffi
+      - name: Build C++ library (asan)
+        run: |
+          mkdir -p build
+          cmake -S lang/cpp -B build -DSANITIZER=asan -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
+          cmake --build build --parallel $(nproc)
+
   sqllogic-test:
     name: "SQL logic tests"
     needs: duckdb-ready

docs/Doxyfile.cpp

@@ -0,0 +1,28 @@
+# Doxygen configuration for Vortex C++ API documentation.
+# XML output is consumed by Sphinx via the Breathe extension.
+
+PROJECT_NAME           = "Vortex C++"
+OUTPUT_DIRECTORY       = _build/doxygen-cpp
+
+# Input sources
+INPUT                  = ../lang/cpp/include/vortex
+FILE_PATTERNS          = *.hpp
+RECURSIVE              = NO
+
+# We only care about XML output for Breathe
+GENERATE_XML           = YES
+GENERATE_HTML          = NO
+GENERATE_LATEX         = NO
+XML_PROGRAMLISTING     = YES
+
+# Extract everything, even if not fully documented yet
+EXTRACT_ALL            = YES
+EXTRACT_PRIVATE        = NO
+EXTRACT_STATIC         = YES
+
+# Preprocessing — resolve includes but don't expand macros
+ENABLE_PREPROCESSING   = YES
+MACRO_EXPANSION        = NO
+
+# Suppress warnings about undocumented members (WIP API)
+WARN_IF_UNDOCUMENTED   = NO

docs/api/cpp/index.rst

@@ -187,7 +187,7 @@ Source code for this example is `writer.cpp
     Expression age_gt_10 = expr::gt(expr::col("age"), expr::lit<uint8_t>(10));
     Array validity_array = array.apply(age_gt_10);
 
-    const Validity validity {ValidityType::Array, validity_array};
+    const Validity validity = Validity::from_array(validity_array);
     Array array2 = make_struct({
         {"age", age},
         {"height", Array::primitive<uint16_t>(height_buffer, validity)},

docs/conf.py

@@ -125,33 +125,6 @@
 breathe_projects = {"vortex-cpp": _doxygen_xml_dir}
 breathe_default_project = "vortex-cpp"
 
-# C++ types from cxx bridge and standard library that Sphinx cannot resolve.
-nitpick_ignore += [
-    ("cpp:identifier", t)
-    for t in [
-        "vortex",
-        "rust",
-        "ffi",
-        "uint8_t",
-        "uint16_t",
-        "uint32_t",
-        "uint64_t",
-        "int8_t",
-        "int16_t",
-        "int32_t",
-        "int64_t",
-        "size_t",
-        "std::size_t",
-    ]
-]
-nitpick_ignore_regex = [
-    # cxx bridge internals that will never be resolvable in Sphinx.
-    (r"cpp:identifier", r"rust::.*"),
-    (r"cpp:identifier", r"ffi::.*"),
-    # Doxygen file-level labels (e.g. "dtype_8hpp") that we don't generate pages for.
-    (r"ref", r".*_8hpp"),
-]
-
 # -- Options for hawkmoth C API gen ----------------------------
 
 hawkmoth_root = str(git_root / "vortex-ffi/cinclude")

encodings/experimental/onpair/src/compress.rs

@@ -27,7 +27,10 @@ use crate::OnPair;
 use crate::OnPairArray;
 
 /// Default OnPair training configuration: 12-bit codes ("dict-12").
-pub const DEFAULT_DICT12_CONFIG: Config = onpair::DEFAULT_CONFIG;
+pub const DEFAULT_DICT12_CONFIG: Config = Config {
+    seed: Some(42),
+    ..onpair::DEFAULT_CONFIG
+};
 
 fn onpair_compress_varbinview<O>(
     array: VarBinViewArray,

lang/cpp/CMakeLists.txt

@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+cmake_minimum_required(VERSION 3.10)
+project(VortexCXX
+    VERSION 0.0.1
+    LANGUAGES CXX)
+
+include(FetchContent)
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+find_program(SCCACHE_PROGRAM sccache)
+if (SCCACHE_PROGRAM)
+    set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}")
+    set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}")
+    message(STATUS "Sccache found: ${SCCACHE_PROGRAM}")
+else ()
+    message(STATUS "Sccache not found")
+endif ()
+
+set(SANITIZER "" CACHE STRING "Build with sanitizers")
+set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library")
+set(RUST_BUILD_PROFILE "" CACHE STRING "Cargo profile name for Rust FFI library")
+
+if (NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Debug)
+endif()
+
+if (NOT SANITIZER STREQUAL "")
+    message(NOTICE "Sanitizer: ${SANITIZER}")
+    if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+        message(FATAL_ERROR "Only debug build is supported for sanitizer builds")
+    endif()
+
+    if (SANITIZER STREQUAL "asan")
+        set(SANITIZER_FLAGS "-fsanitize=address,undefined,leak")
+    elseif (SANITIZER STREQUAL "tsan")
+        set(SANITIZER_FLAGS "-fsanitize=thread")
+    else()
+        message(FATAL_ERROR "Unknown sanitizer ${SANITIZER}")
+    endif()
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZER_FLAGS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZER_FLAGS}")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZER_FLAGS}")
+endif()
+
+if (RUST_BUILD_PROFILE STREQUAL "")
+    if (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+        set(RUST_BUILD_PROFILE "release_debug")
+    else()
+        string(TOLOWER "${CMAKE_BUILD_TYPE}" RUST_BUILD_PROFILE)
+    endif()
+endif()
+
+set(VORTEX_FFI_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../vortex-ffi")
+set(VORTEX_FFI_LIB_DIR "${VORTEX_FFI_DIR}/../target/${TARGET_TRIPLE}/${RUST_BUILD_PROFILE}")
+set(VORTEX_FFI_HEADERS "${VORTEX_FFI_DIR}/cinclude")
+
+if(APPLE)
+    set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a")
+    set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dylib")
+elseif(WIN32)
+    set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.lib")
+    set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dll")
+else()
+    set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a")
+    set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.so")
+endif()
+
+if(NOT EXISTS "${VORTEX_FFI_STATIC}")
+    message(FATAL_ERROR
+        "vortex-ffi static library not found at ${VORTEX_FFI_STATIC}. "
+        "Run: cargo build --profile <profile> -p vortex-ffi")
+endif()
+
+add_library(vortex_ffi STATIC IMPORTED)
+set_target_properties(vortex_ffi PROPERTIES
+    IMPORTED_LOCATION "${VORTEX_FFI_STATIC}"
+    INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}")
+
+if(EXISTS "${VORTEX_FFI_SHARED}")
+    add_library(vortex_ffi_shared SHARED IMPORTED)
+    set_target_properties(vortex_ffi_shared PROPERTIES
+        IMPORTED_LOCATION "${VORTEX_FFI_SHARED}"
+        INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}"
+        INTERFACE_LINK_OPTIONS "LINKER:-rpath,${VORTEX_FFI_LIB_DIR}")
+endif()
+
+file(GLOB VORTEX_CXX_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp")
+add_library(vortex_cxx STATIC ${VORTEX_CXX_SOURCES})
+target_include_directories(vortex_cxx PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude
+)
+target_link_libraries(vortex_cxx PUBLIC vortex_ffi)
+add_library(vortex::cxx ALIAS vortex_cxx)
+target_compile_features(vortex_cxx PUBLIC cxx_std_20)
+
+set(APPLE_LINK_FLAGS "-framework CoreFoundation -framework Security")
+
+if(APPLE)
+    target_link_libraries(vortex_cxx PRIVATE ${APPLE_LINK_FLAGS})
+endif()
+
+if(TARGET vortex_ffi_shared)
+    add_library(vortex_cxx_shared SHARED ${VORTEX_CXX_SOURCES})
+    set_target_properties(vortex_cxx_shared PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    target_include_directories(vortex_cxx_shared PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}
+        ${CMAKE_CURRENT_SOURCE_DIR}/include
+        ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude
+    )
+    target_link_libraries(vortex_cxx_shared PUBLIC vortex_ffi_shared)
+    add_library(vortex::cxx_shared ALIAS vortex_cxx_shared)
+    target_compile_features(vortex_cxx_shared PUBLIC cxx_std_20)
+
+    if(APPLE)
+        target_link_libraries(vortex_cxx_shared PRIVATE ${APPLE_LINK_FLAGS})
+    endif()
+endif()

lang/cpp/README.md

@@ -0,0 +1,37 @@
+# Vortex C++ bindings
+
+For a usage guide, see docs/api/cpp/index.rst.
+
+## Requirements
+
+- CMake 3.10+
+- C++20 compiler (C++23 compiler for tests).
+- Rust toolchain for building `vortex-ffi`.
+
+## Build
+
+```sh
+cargo build --release -p vortex-ffi
+cmake -Bbuild -DCMAKE_BUILD_TYPE=Release
+cmake --build build -j
+```
+
+This will generate `libvortex_cxx` shared and static libraries.
+You can use `vortex_cxx` and `vortex_cxx_shared` CMake targets.
+
+## Test
+
+```sh
+cargo build --release -p vortex-ffi
+cmake -Bbuild -DBUILD_TESTS=ON
+cmake --build build -j
+ctest --test-dir build -j "$(nproc)"
+```
+
+## Run examples
+
+```sh
+cmake -Bbuild -DBUILD_EXAMPLES=ON
+cmake --build build -j
+./build/examples/hello-vortex
+```