ci.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: CI

on:
  push:
    branches:
      - main
      - release-*
  pull_request:
    branches:
      - main
      - release-*

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Validate .asf.yaml
        run: python3 scripts/validate_asf_yaml.py

      - name: Check License Header
        uses: apache/skywalking-eyes/header@v0.8.0

      - name: Install cargo-deny
        uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0
        with:
          tool: cargo-deny@0.19.6

      - name: Fetch locked dependencies
        run: cargo fetch --locked

      - name: Check dependency policy
        run: cargo deny --locked --all-features check --warn unmaintained advisories licenses

      - name: Verify dependency reports
        run: python3 scripts/dependencies.py verify

      - name: Format
        run: cargo fmt --all -- --check

      - name: Clippy
        run: cargo clippy --locked --all-targets --workspace --features fulltext,vortex -- -D warnings

  msrv:
    runs-on: ubuntu-latest
    env:
      PYO3_PYTHON: python3.11
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Install Rust 1.91.0
        run: rustup toolchain install 1.91.0 --profile minimal

      - name: Validate workspace MSRV metadata
        run: |
          cargo +1.91.0 metadata --locked --format-version 1 --no-deps |
            jq -e '.packages | all(.rust_version == "1.91.0")' >/dev/null

      - name: Check workspace at MSRV
        run: cargo +1.91.0 check --locked --workspace --all-targets --all-features

  build:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
          - windows-latest
    steps:
      - uses: actions/checkout@v7
      - name: Build
        run: cargo build --locked --features fulltext,vortex

  unit:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
          - windows-latest
    steps:
      - uses: actions/checkout@v7

      - name: Test
        run: cargo test --locked -p paimon --all-targets --features fulltext,vortex
        env:
          RUST_LOG: DEBUG
          RUST_BACKTRACE: full

      # The e2e tests round-trip through a real FileSystemCatalog, which has
      # known Windows path-compatibility issues (the `paimon` filesystem
      # catalog tests are likewise gated with `#[cfg(not(windows))]`).
      - name: Test paimon-rest-server
        if: runner.os != 'Windows'
        run: cargo test --locked -p paimon-rest-server --all-targets
        env:
          RUST_LOG: DEBUG
          RUST_BACKTRACE: full

  integration:
    name: integration (${{ matrix.suite }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - suite: rust
            needs_docker: true
            needs_lumina: false
            needs_uv: false
          - suite: datafusion
            needs_docker: true
            needs_lumina: true
            needs_uv: false
          - suite: lumina
            needs_docker: false
            needs_lumina: true
            needs_uv: false
          - suite: python
            needs_docker: true
            needs_lumina: false
            needs_uv: true
          - suite: go
            needs_docker: true
            needs_lumina: false
            needs_uv: false
    env:
      RUST_LOG: DEBUG
      RUST_BACKTRACE: full
    steps:
      - uses: actions/checkout@v7

      - name: Rust Cache
        uses: actions/cache@v6
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: integration-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            integration-${{ runner.os }}-

      - name: Start Docker containers
        if: matrix.needs_docker == true
        run: make docker-up

      - name: Rust Integration Test
        if: matrix.suite == 'rust'
        run: cargo test --locked -p paimon-integration-tests --all-targets

      - name: Install lumina native library
        if: matrix.needs_lumina == true
        run: |
          pip install lumina-data
          echo "LUMINA_LIB_PATH=$(python3 -c 'import lumina_data; print(lumina_data.__path__[0])')/lib/liblumina_py.so" >> $GITHUB_ENV

      - name: Core Lumina Native Build Test
        if: matrix.suite == 'lumina'
        run: >
          cargo test --locked -p paimon
          table::lumina_index_build_builder::tests::test_execute_writes_lumina_index_manifest
          --features fulltext,vortex
          -- --ignored --exact

      - name: DataFusion Lumina Build Query E2E Test
        if: matrix.suite == 'lumina'
        run: >
          cargo test --locked -p paimon-datafusion
          --features vortex
          vector_search_tests::test_lumina_build_then_vector_search_query
          -- --ignored --exact

      - name: DataFusion Integration Test
        if: matrix.suite == 'datafusion'
        run: cargo test --locked -p paimon-datafusion --all-targets

      - name: DataFusion Vortex Integration Test
        if: matrix.suite == 'datafusion'
        run: cargo test --locked -p paimon-datafusion --features vortex --test vortex_tables

      - name: Install uv
        if: matrix.needs_uv == true
        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          version: "0.9.3"
          enable-cache: true

      - name: Python Binding Integration Test
        if: matrix.suite == 'python'
        working-directory: bindings/python
        run: make install && make test

      - name: Go Integration Test
        if: matrix.suite == 'go'
        working-directory: bindings/go
        run: make test

      - name: Stop Docker containers
        if: ${{ always() && matrix.needs_docker }}
        run: make docker-down