diff --git a/.github/baseline.env b/.github/baseline.env new file mode 100644 index 0000000..deb034b --- /dev/null +++ b/.github/baseline.env @@ -0,0 +1 @@ +MACOSX_DEPLOYMENT_TARGET=11 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a089e5f..6b1a9c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ jobs: format: name: Rustfmt runs-on: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 timeout-minutes: 10 steps: - uses: actions/checkout@v4 @@ -31,6 +32,7 @@ jobs: clippy: name: Clippy runs-on: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -39,13 +41,18 @@ jobs: components: clippy - uses: Swatinem/rust-cache@v2 - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + run: | + ARCH=$(uname -m) + case "${ARCH}" in x86_64) PA="linux-x86_64" ;; aarch64) PA="linux-aarch_64" ;; esac + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-${PA}.zip" -o /tmp/protoc.zip + unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' && rm /tmp/protoc.zip - name: Clippy run: cargo clippy --all-targets -- -D warnings test-linux: name: Tests (Linux) runs-on: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 timeout-minutes: 30 env: CC: clang @@ -54,8 +61,14 @@ jobs: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 + - name: Install Clang 20 + run: manylinux-install-clang -v 20.1.8.0 - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + run: | + ARCH=$(uname -m) + case "${ARCH}" in x86_64) PA="linux-x86_64" ;; aarch64) PA="linux-aarch_64" ;; esac + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-${PA}.zip" -o /tmp/protoc.zip + unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' && rm /tmp/protoc.zip - name: Run tests run: cargo test - name: Run C/C++ compilation tests @@ -67,6 +80,13 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v4 + - name: Load baseline floors + shell: bash + run: | + set -a + source .github/baseline.env + set +a + echo "MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET" >>"$GITHUB_ENV" - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 - name: Install dependencies @@ -79,19 +99,25 @@ jobs: rustdoc: name: Rustdoc runs-on: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + run: | + ARCH=$(uname -m) + case "${ARCH}" in x86_64) PA="linux-x86_64" ;; aarch64) PA="linux-aarch_64" ;; esac + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-${PA}.zip" -o /tmp/protoc.zip + unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' && rm /tmp/protoc.zip - name: Check documentation run: RUSTDOCFLAGS="-D warnings" cargo doc --no-deps msrv: name: MSRV (1.91.0) runs-on: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -100,13 +126,18 @@ jobs: toolchain: "1.91.0" - uses: Swatinem/rust-cache@v2 - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + run: | + ARCH=$(uname -m) + case "${ARCH}" in x86_64) PA="linux-x86_64" ;; aarch64) PA="linux-aarch_64" ;; esac + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-${PA}.zip" -o /tmp/protoc.zip + unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' && rm /tmp/protoc.zip - name: Check MSRV run: cargo check --all-targets license-headers: name: License headers runs-on: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 timeout-minutes: 5 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/consumer-smoke-test.yml b/.github/workflows/consumer-smoke-test.yml index 7c649f7..544b4a7 100644 --- a/.github/workflows/consumer-smoke-test.yml +++ b/.github/workflows/consumer-smoke-test.yml @@ -28,28 +28,45 @@ env: jobs: consumer-smoke-test: name: find_package(LanceC) smoke test - runs-on: ${{ matrix.runner }} + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} timeout-minutes: 45 strategy: fail-fast: false matrix: include: - - runner: ubuntu-24.04 - os_label: linux - - runner: macos-14 - os_label: macos + - os: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 + - os: macos-14 env: CC: clang CXX: clang++ steps: - uses: actions/checkout@v4 + - name: Load baseline floors + shell: bash + run: | + set -a + source .github/baseline.env + set +a + { + echo "MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET" + } >>"$GITHUB_ENV" - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 - name: Install protobuf-compiler (Linux) - if: matrix.os_label == 'linux' - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + if: runner.os == 'Linux' + run: | + ARCH=$(uname -m) + case "${ARCH}" in x86_64) PA="linux-x86_64" ;; aarch64) PA="linux-aarch_64" ;; esac + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-${PA}.zip" -o /tmp/protoc.zip + unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' && rm /tmp/protoc.zip + - name: Install Clang 20 (Linux) + if: runner.os == 'Linux' + run: | + manylinux-install-clang -v 20.1.8.0 - name: Install protobuf (macOS) - if: matrix.os_label == 'macos' + if: runner.os == 'macOS' run: brew install protobuf - name: Configure + build + install lance-c run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 92d07a4..255907c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,40 +18,53 @@ env: jobs: build: name: Build (${{ matrix.target }}) - runs-on: ${{ matrix.runner }} + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} timeout-minutes: 90 strategy: fail-fast: false matrix: include: - - target: x86_64-unknown-linux-gnu - runner: ubuntu-24.04 - os_label: linux - - target: aarch64-unknown-linux-gnu - runner: ubuntu-24.04-arm - os_label: linux + - os: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 + target: x86_64-unknown-linux-gnu + - os: ubuntu-24.04-arm + container: quay.io/pypa/manylinux_2_28_aarch64 + target: aarch64-unknown-linux-gnu # x86_64-apple-darwin temporarily disabled: GitHub is deprecating # macos-13 runners and jobs queue 24h+ before timing out. Restore # this row once a macos-* Intel runner is reliably available again. - - target: aarch64-apple-darwin - runner: macos-14 - os_label: macos + - os: macos-14 + target: aarch64-apple-darwin env: CC: clang CXX: clang++ steps: - uses: actions/checkout@v4 + - name: Load baseline floors + shell: bash + run: | + set -a + source .github/baseline.env + set +a + { + echo "MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET" + } >>"$GITHUB_ENV" - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - target: ${{ matrix.target }} - uses: Swatinem/rust-cache@v2 - with: - key: ${{ matrix.target }} - name: Install protobuf (Linux) - if: matrix.os_label == 'linux' - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + if: runner.os == 'Linux' + run: | + ARCH=$(uname -m) + case "${ARCH}" in x86_64) PA="linux-x86_64" ;; aarch64) PA="linux-aarch_64" ;; esac + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-${PA}.zip" -o /tmp/protoc.zip + unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' && rm /tmp/protoc.zip + - name: Install Clang 20 (Linux) + if: runner.os == 'Linux' + run: | + manylinux-install-clang -v 20.1.8.0 - name: Install protobuf (macOS) - if: matrix.os_label == 'macos' + if: runner.os == 'macOS' run: brew install protobuf - name: Resolve version id: ver @@ -74,8 +87,13 @@ jobs: TARGET=${{ matrix.target }} ARCHIVE="lance-c-v${VERSION}-${TARGET}.tar.xz" tar -C stage -cJf "${ARCHIVE}" . - shasum -a 256 "${ARCHIVE}" > "${ARCHIVE}.sha256" - shasum -a 512 "${ARCHIVE}" > "${ARCHIVE}.sha512" + if command -v shasum > /dev/null 2>&1; then + shasum -a 256 "${ARCHIVE}" > "${ARCHIVE}.sha256" + shasum -a 512 "${ARCHIVE}" > "${ARCHIVE}.sha512" + else + sha256sum "${ARCHIVE}" > "${ARCHIVE}.sha256" + sha512sum "${ARCHIVE}" > "${ARCHIVE}.sha512" + fi echo "archive=${ARCHIVE}" >>"$GITHUB_OUTPUT" - name: Upload as workflow artifact uses: actions/upload-artifact@v4 @@ -92,6 +110,7 @@ jobs: name: Publish GitHub Release needs: build runs-on: ubuntu-24.04 + timeout-minutes: 15 if: startsWith(github.ref, 'refs/tags/v') steps: - uses: actions/checkout@v4 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6960752..aa5c7b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,8 +5,15 @@ cmake_minimum_required(VERSION 3.22) list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") include(cargo-version) +include(baseline) read_cargo_version("${CMAKE_CURRENT_LIST_DIR}/Cargo.toml" LANCE_C_VERSION) +lance_c_load_baseline_env("${CMAKE_CURRENT_LIST_DIR}/.github/baseline.env") + +if(APPLE AND NOT CMAKE_OSX_DEPLOYMENT_TARGET) + set(CMAKE_OSX_DEPLOYMENT_TARGET "${MACOSX_DEPLOYMENT_TARGET}" CACHE STRING "Minimum macOS deployment target" FORCE) +endif() + project(LanceC VERSION ${LANCE_C_VERSION} DESCRIPTION "C/C++ bindings for the Lance columnar data format" @@ -14,6 +21,22 @@ project(LanceC LANGUAGES C CXX ) +if(NOT DEFINED CMAKE_C_STANDARD OR CMAKE_C_STANDARD LESS 11) + set(CMAKE_C_STANDARD 11) +endif() +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) + +if(NOT DEFINED CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 20) + set(CMAKE_CXX_STANDARD 20) +endif() +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +set(CMAKE_C_VISIBILITY_PRESET hidden) +set(CMAKE_CXX_VISIBILITY_PRESET hidden) +set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) + include(GNUInstallDirs) # ─── Options ───────────────────────────────────────────────────────────────── @@ -42,6 +65,9 @@ if(NOT LANCE_C_USE_PREBUILT) CRATE_TYPES staticlib cdylib PROFILE release ) + if(APPLE) + corrosion_set_env_vars(lance_c "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() endif() # Imported targets and install rules are added in subsequent tasks. diff --git a/README.md b/README.md index 893589c..5883273 100644 --- a/README.md +++ b/README.md @@ -6,67 +6,15 @@ The C/C++ binding to [Lance](https://github.com/lancedb/lance), providing native - **C++ wrappers:** [`include/lance/lance.hpp`](include/lance/lance.hpp) (header-only, RAII, exceptions) - **Data exchange:** [Arrow C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html) for zero-copy interop -## Roadmap - -Based on the [liblance RFC](https://github.com/lance-format/lance/discussions/6035). - -### Phase 1: Core Read Path + C++ Wrappers (MVP) - -| Status | Component | Description | -|--------|-----------|-------------| -| [x] | Infrastructure | `lance-c` crate with Cargo.toml, Tokio runtime initialization | -| [x] | Error handling | Thread-local error codes/messages for cross-FFI safety | -| [x] | C header | `lance.h` with Arrow C Data Interface structs | -| [x] | Dataset operations | Open/close with URI + storage options + version support | -| [x] | Schema export | Arrow C Data Interface for zero-copy schema exchange | -| [x] | Scanner builder | Column projection, SQL filters, limit/offset, batch size, row ID, fragment filtering | -| [x] | ArrowArrayStream export | `lance_scanner_to_arrow_stream()` blocking API | -| [x] | Batch iteration | `lance_scanner_next()` blocking function | -| [x] | Poll + waker iteration | `lance_scanner_poll_next()` for async engines (Velox, Presto) | -| [x] | Random access | Index-based row retrieval via `lance_dataset_take()` | -| [x] | C++ wrappers | Header-only RAII library (`lance::Dataset`, `lance::Scanner`, `lance::Batch`) | -| [x] | Builder pattern | Fluent Scanner API (`.limit().offset().batch_size().with_row_id()`) | - -### Phase 2: Vector Search & Indexing - -| Status | Component | Description | -|--------|-----------|-------------| -| [x] | Vector search | Nearest-neighbor via scanner with metric/k/nprobes | -| [x] | Full-text search | FTS queries through scanner interface | -| [x] | Vector index creation | IVF_PQ, IVF_FLAT, IVF_SQ, HNSW variants | -| [x] | Scalar index creation | BTree, Bitmap, Inverted, Label-List indexes | -| [x] | Index management | List and drop index operations | -| [x] | C++ wrappers | `create_vector_index()` and `create_scalar_index()` methods | - -### Phase 3: Write Path & Mutations - -| Status | Component | Description | -|--------|-----------|-------------| -| [x] | Dataset write | Create / append / overwrite from ArrowArrayStream via `lance_dataset_write()`; tunable variant `lance_dataset_write_with_params()` for file/row-group sizing, Lance format version, and stable row IDs | -| [x] | Fragment writer | Batch-at-a-time fragment file writing (no commit) via `lance_write_fragments()` | -| [x] | Delete operations | Predicate-based deletion via `lance_dataset_delete()` | -| [x] | Update operations | Expression-based row updates via `lance_dataset_update()` | -| [x] | Merge-insert | Upsert via configurable when-matched / when-not-matched policies with `lance_dataset_merge_insert()` | -| [x] | Schema evolution | Add columns via SQL / all-null / `ArrowArrayStream` with `lance_dataset_add_columns_*()`, drop via `lance_dataset_drop_columns()`, rename / retype / set nullability via `lance_dataset_alter_columns()` | -| [x] | Version management | List via `lance_dataset_versions()`, rollback via `lance_dataset_restore()`, checkout via `lance_dataset_open(uri, opts, version)` | - -### Phase 4: Advanced Features - -| Status | Component | Description | -|--------|-----------|-------------| -| [x] | Fragment-level access | Fragment enumeration, ID listing, scanner fragment filtering | -| [x] | Compaction | Fragment consolidation via `lance_dataset_compact_files()` | -| [x] | Statistics export | Per-field compressed on-disk size for query planning via `lance_dataset_calculate_data_stats()` | -| [x] | Cloud storage | S3, GCS, Azure via storage options pass-through | -| [x] | Package distribution | vcpkg and Conan recipe packaging | - -### Additional (not in RFC) - -| Status | Component | Description | -|--------|-----------|-------------| -| [x] | Async scan | Callback-based `lance_scanner_scan_async()` for non-blocking scans | -| [x] | Dataset metadata | `lance_dataset_version()`, `lance_dataset_count_rows()`, `lance_dataset_latest_version()` | -| [x] | Substrait filter pushdown | `lance_scanner_set_substrait_filter()` accepts a serialized Substrait `ExtendedExpression` (preferred over SQL strings for query engines) | +## Supported build targets + +| Component | Target / minimum | +|-----------|------------------| +| CMake | 3.22+ | +| C++ | C++20 | +| GLIBC baseline | 2.28 | +| Clang | Clang 20; Apple Clang 15 | +| macOS | `.github/baseline.env` (`MACOSX_DEPLOYMENT_TARGET`) | ## Building @@ -81,11 +29,11 @@ cargo build --release Produces `target/release/liblance_c.{so,dylib,dll}` and a `liblance_c.a`. Headers stay in `include/lance/`. -### From source via CMake (C/C++ developers) +### From source via CMake (C/C++ developers and packagers) ```bash cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -cmake --build build +cmake --build build --parallel cmake --install build --prefix /your/prefix ``` @@ -100,6 +48,8 @@ target_link_libraries(myapp PRIVATE LanceC::lance_c) See [`examples/cmake-consumer/`](examples/cmake-consumer/) for a minimal working example. +Baseline values are defined once in [`.github/baseline.env`](.github/baseline.env) and loaded by CI, release workflows, and CMake. + ### vcpkg ```bash diff --git a/cmake/baseline.cmake b/cmake/baseline.cmake new file mode 100644 index 0000000..4768feb --- /dev/null +++ b/cmake/baseline.cmake @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +include_guard(GLOBAL) + +function(lance_c_load_baseline_env env_file) + if(NOT EXISTS "${env_file}") + message(FATAL_ERROR "Missing baseline env file: ${env_file}") + endif() + + file(STRINGS "${env_file}" _baseline_lines) + foreach(_line IN LISTS _baseline_lines) + string(STRIP "${_line}" _line) + if(_line STREQUAL "" OR _line MATCHES "^#") + continue() + endif() + + if(NOT _line MATCHES "^([A-Za-z_][A-Za-z0-9_]*)=(.*)$") + message(FATAL_ERROR "Invalid baseline line: ${_line}") + endif() + + set("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}" PARENT_SCOPE) + endforeach() +endfunction() diff --git a/examples/cmake-consumer/CMakeLists.txt b/examples/cmake-consumer/CMakeLists.txt index aef3535..8d51bd0 100644 --- a/examples/cmake-consumer/CMakeLists.txt +++ b/examples/cmake-consumer/CMakeLists.txt @@ -7,5 +7,5 @@ project(lance_c_consumer LANGUAGES CXX) find_package(LanceC 0.1 REQUIRED) add_executable(consumer main.cpp) -target_compile_features(consumer PRIVATE cxx_std_17) +target_compile_features(consumer PRIVATE cxx_std_20) target_link_libraries(consumer PRIVATE LanceC::lance_c) diff --git a/recipes/lance-c/all/test_package/CMakeLists.txt b/recipes/lance-c/all/test_package/CMakeLists.txt index 34f50e2..8030dd8 100644 --- a/recipes/lance-c/all/test_package/CMakeLists.txt +++ b/recipes/lance-c/all/test_package/CMakeLists.txt @@ -4,5 +4,5 @@ project(test_package LANGUAGES CXX) find_package(LanceC CONFIG REQUIRED) add_executable(test_package test_package.cpp) -target_compile_features(test_package PRIVATE cxx_std_17) +target_compile_features(test_package PRIVATE cxx_std_20) target_link_libraries(test_package PRIVATE LanceC::lance_c) diff --git a/tests/compile_and_run_test.rs b/tests/compile_and_run_test.rs index d736d61..7e7626f 100644 --- a/tests/compile_and_run_test.rs +++ b/tests/compile_and_run_test.rs @@ -124,7 +124,7 @@ fn compile_cpp_test(source: &Path, output: &Path, include_dir: &Path, lib_path: let status = Command::new("c++") .args([ - "-std=c++17", + "-std=c++20", "-Wall", "-Wextra", "-o",