ci.yml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: CI

on:
  workflow_call:
    # Called from ci-main.yml and ci-pr.yml, which define different concurrency groups.
    secrets:
      DEVELOCITY_ACCESS_KEY:
        required: false

jobs:
  build-checks:
    name: Gradle Build Checks
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: &setup-java-vars
          java-version: '21'
          distribution: 'temurin'
      - name: Setup test environment
        uses: ./.github/actions/setup-test-env
      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
      - name: Run Checks
        env: &gradle_env_vars
          GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }}
          DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }}
          DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }}
          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} # zizmor: ignore[secrets-outside-env]
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # zizmor: ignore[secrets-outside-env]
        run: |
          ./gradlew check sourceTarball distTar distZip publishToMavenLocal \
            -x :polaris-runtime-service:test \
            -x :polaris-admin:test \
            -x intTest \
            --continue
      - name: Verify configuration reference is up to date
        run: |
          ./gradlew :polaris-config-docs-site:copyConfigSectionsToSite
          if ! git diff --exit-code site/content/in-dev/unreleased/configuration/config-sections/; then
            echo "ERROR: Configuration reference is out of date. Please run './gradlew :polaris-config-docs-site:copyConfigSectionsToSite' and commit the changes."
            exit 1
          fi
      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
      - name: Archive test results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: failure()
        with:
          name: upload-${{ github.job }}-artifacts
          path: &test-archive-path |
            **/build/test-results/**
            **/build/reports/tests/**
      #
      # The following are not strictly "Gradle build checks", but run quite fast.
      #
      # Adding separate jobs for these requires a new GH runner instance, which requires even more resources.
      #
      - name: Verify Copilot instructions
        run: .github/scripts/check-copilot-instructions.sh
      - name: Renovate Configuration Validation
        run: npx --yes --package renovate renovate-config-validator --strict --no-global .github/renovate.json5
      # Intentionally unpinned to always use the latest allowlist from the ASF.
      - uses: apache/infrastructure-actions/allowlist-check@main # zizmor: ignore[unpinned-uses]

  runtime-service-tests:
    name: Runtime Service Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: *setup-java-vars
      - name: Setup test environment
        uses: ./.github/actions/setup-test-env
      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
      - name: Run Quarkus runtime tests
        env: *gradle_env_vars
        run: ./gradlew :polaris-runtime-service:test --continue
      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
      - name: Archive test results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: failure()
        with:
          name: upload-${{ github.job }}-artifacts
          path: *test-archive-path

  runtime-service-int-tests:
    name: Runtime Service Integration Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: *setup-java-vars
      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
      - name: Run Quarkus runtime tests
        env: *gradle_env_vars
        run: ./gradlew :polaris-runtime-service:intTest --continue
      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
      - name: Archive test results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: failure()
        with:
          name: upload-${{ github.job }}-artifacts
          path: *test-archive-path

  admin-tool-tests:
    name: Admin Tool Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: *setup-java-vars
      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
      - name: Run Quarkus admin tests
        env: *gradle_env_vars
        run: ./gradlew :polaris-admin:test --continue
      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
      - name: Archive test results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: failure()
        with:
          name: upload-${{ github.job }}-artifacts
          path: *test-archive-path

  integration-tests:
    name: Other Integration Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: *setup-java-vars
      - name: Setup test environment
        uses: ./.github/actions/setup-test-env
      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
      - name: Run integration tests
        env: *gradle_env_vars
        run: |
          ./gradlew \
            intTest \
            -x :polaris-runtime-service:intTest \
            --continue
      - name: Save partial Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-save
      - name: Archive test results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: failure()
        with:
          name: upload-${{ github.job }}-artifacts
          path: *test-archive-path

  docker-image-build:
    name: Polaris Docker Image Build
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          fetch-depth: 0
          persist-credentials: false
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: *setup-java-vars
      - name: Setup test environment
        uses: ./.github/actions/setup-test-env
      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
      - name: Image build
        env: *gradle_env_vars
        run: ./gradlew :polaris-server:assemble -Dquarkus.container-image.build=true
      - name: Save image to tar
        run: |
          docker save apache/polaris:latest | zstd -T0 > polaris-image.tar.zst
      - name: Upload image artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: polaris-image
          path: polaris-image.tar.zst
          retention-days: 1

  docker-image-scan:
    name: Polaris Docker Image Scan
    if: github.repository == 'apache/polaris'
    runs-on: ubuntu-latest
    needs:
      - docker-image-build
    timeout-minutes: 20
    permissions:
      contents: read
      security-events: write
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Download image artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: polaris-image
      - name: Load image
        run: zstd -d -c polaris-image.tar.zst | docker load
      - name: Cache Trivy vulnerability DB
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        with:
          path: ${{ github.workspace }}/.cache/trivy
          key: trivy-db-${{ runner.os }}
      # lhotari/sandboxed-trivy-action is the ASF-approved fork of
      # aquasecurity/trivy-action; it runs Trivy inside a hardened container.
      - name: Run Trivy scan (table, logged)
        uses: lhotari/sandboxed-trivy-action@f01374b6cc3bf7264ab238293e94f6db7ada6dd0 # v1.0.2
        with:
          scan-type: image
          scan-ref: apache/polaris:latest
          format: table
          severity: HIGH,CRITICAL
          ignore-unfixed: 'true'
          exit-code: '0'
      - name: Run Trivy scan (SARIF)
        uses: lhotari/sandboxed-trivy-action@f01374b6cc3bf7264ab238293e94f6db7ada6dd0 # v1.0.2
        with:
          scan-type: image
          scan-ref: apache/polaris:latest
          format: sarif
          output: trivy-results.sarif
          severity: HIGH,CRITICAL
          ignore-unfixed: 'true'
          exit-code: '0'
      - name: Upload SARIF to GitHub Code Scanning
        if: ${{ startsWith(github.ref, 'refs/heads/') }}
        uses: github/codeql-action/upload-sarif@87557b9c84dde89fdd9b10e88954ac2f4248e463 # v4.36.1
        with:
          sarif_file: trivy-results.sarif
          category: trivy-image
      - name: Upload SARIF as workflow artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: trivy-results-sarif
          path: trivy-results.sarif
          retention-days: 7

  helm-tests:
    name: Helm Tests (Helm ${{ matrix.helm-version }} - K8s ${{ matrix.kubernetes-version }})
    runs-on: ubuntu-latest
    needs:
      - docker-image-build
    timeout-minutes: 60
    permissions:
      contents: read
    strategy:
      matrix:
        include:
          - helm-version: 'v3.20.0'
            kubernetes-version: 'v1.33.8'
          - helm-version: 'v4.0.5'
            kubernetes-version: 'v1.34.4'
          - helm-version: 'v4.1.1'
            kubernetes-version: 'v1.35.1'
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          fetch-depth: 0
          persist-credentials: false
      - name: Set up Helm
        uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
        with:
          version: ${{ matrix.helm-version }}
      - name: Set up chart-testing
        run: |
          CT_VERSION=3.14.0
          ARCH=$(uname -m)
          case "$ARCH" in
            x86_64|amd64) ARCH=amd64 ;;
            aarch64|arm64) ARCH=arm64 ;;
          esac
          curl -sSLo ct.tar.gz "https://github.com/helm/chart-testing/releases/download/v${CT_VERSION}/chart-testing_${CT_VERSION}_linux_${ARCH}.tar.gz"
          sudo tar -xzf ct.tar.gz -C /usr/local/bin ct
          mkdir -p .ct
          tar -xzf ct.tar.gz --strip-components=1 -C .ct etc
          rm ct.tar.gz
          pip install yamllint yamale
      - name: Set up helm-docs
        run: |
          go install github.com/norwoodj/helm-docs/cmd/helm-docs@latest
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
      - name: Install Helm plugins
        run: make helm-install-plugins
      - name: Verify Helm schema is up to date
        run: make helm-schema-verify
      - name: Verify Helm documentation is up to date
        run: make helm-doc-verify
      - name: Run 'helm template' validation
        run: |
          cd helm/polaris
          for f in values.yaml ci/*.yaml; do
            echo "::group::helm template $f"
            helm template --debug --namespace polaris-ns --values $f .
            echo "::endgroup::"
          done
      - name: Run Helm unit tests
        run: make helm-unittest
      - name: Run chart-testing (lint)
        env:
          DEFAULT_BRANCH: ${{ github.base_ref || github.event.repository.default_branch }}
        run: ct lint --target-branch "${DEFAULT_BRANCH}" --debug --charts ./helm/polaris --validate-maintainers=false
      - name: Set up Minikube
        uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21
        with:
          kubernetes-version: ${{ matrix.kubernetes-version }}
      - name: Download image artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: polaris-image
      - name: Load image into Minikube
        run: |
          zstd -d -c polaris-image.tar.zst | minikube image load -
          APP_VERSION=$(grep '^appVersion:' helm/polaris/Chart.yaml | awk '{print $2}')
          minikube image tag apache/polaris:latest apache/polaris:${APP_VERSION}
          echo "Available images:"
          minikube image ls
      - name: Install fixtures
        run: |
          kubectl create namespace polaris-ns
          kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures
      - name: Run chart-testing (install)
        env:
          DEFAULT_BRANCH: ${{ github.base_ref || github.event.repository.default_branch }}
        run: |
          ct install --target-branch "${DEFAULT_BRANCH}" \
            --namespace polaris-ns \
            --debug --charts ./helm/polaris

  python-client:
    name: Polaris Python Client Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: *setup-java-vars
      - name: Setup test environment
        uses: ./.github/actions/setup-test-env
      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Lint
        run: make client-lint
      - name: License Compliance Check
        run: make client-license-check
      - name: Generated Client Tests
        run: make client-unit-test
      - name: Integration Tests
        run: make client-integration-test
      - name: Run Polaris Client help manual
        run: ./polaris --help

  regtest:
    name: Regression Tests (${{ matrix.s3-backend }})
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        s3-backend: ["minio", "rustfs"]
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Free disk space
        uses: ./.github/actions/free-disk-space
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: *setup-java-vars
      - name: Setup test environment
        uses: ./.github/actions/setup-test-env
      - name: Prepare Gradle build cache
        uses: ./.github/actions/ci-incr-build-cache-prepare
      - name: Fix permissions
        run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
      - name: Image build
        env: *gradle_env_vars
        run: |
          ./gradlew \
              publishToMavenLocal \
              :polaris-server:assemble \
              :polaris-server:quarkusAppPartsBuild --rerun \
              -Dquarkus.container-image.build=true
      - name: Regression Test
        env:
          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} # zizmor: ignore[secrets-outside-env]
          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} # zizmor: ignore[secrets-outside-env]
          S3_TEST_BACKEND: ${{ matrix.s3-backend }}
        run: docker compose --profile ${{ matrix.s3-backend }} -f regtests/docker-compose.yml up --build --exit-code-from regtest
      - name: Spark Plugin Regression Test
        # NOTE: the regression test runs with spark 3.5.6 and scala 2.12 in Java 17. We also have integration
        # tests runs with the existing gradle.yml, which only runs on Java 21. Since spark Java compatibility
        # for 3.5 is 8, 11, and 17, we should run spark client with those compatible java versions.
        # TODO: add separate spark client CI and run with Java 8, 11 and 17.
        run: docker compose -f plugins/spark/v3.5/regtests/docker-compose.yml up --build --exit-code-from regtest

  markdown-link-check:
    name: "Markdown Link Check"
    if: github.repository == 'apache/polaris'
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Setup test environment
        uses: ./.github/actions/setup-test-env
      - name: Markdown link check
        uses: tcort/github-action-markdown-link-check@e7c7a18363c842693fadde5d41a3bd3573a7a225
        with:
          use-quiet-mode: 'yes'
          config-file: '.github/workflows/check-md-link-config.json'
          folder-path: 'regtests, .github, build-logic, polaris-core, runtime, persistence, spec, getting-started, helm'
          file-path: 'CHAT_BYLAWS.md, CODE_OF_CONDUCT.md, CONTRIBUTING.md, README.md, SECURITY.md'

  site:
    name: "Site Generation"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Checkout Versioned Docs
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: "versioned-docs"
          path: site/content/releases
          persist-credentials: false
        # Do not let this step fail when the versioned-docs branch does not exist, as on most forks
        continue-on-error: true
      - name: Setup test environment
        uses: ./.github/actions/setup-test-env
      - name: Install docker-compose
        run: |
          sudo curl --fail-with-body -SL \
            https://github.com/docker/compose/releases/download/v2.29.2/docker-compose-linux-x86_64 \
            -o /usr/local/bin/docker-compose
          sudo chmod 755 /usr/local/bin/docker-compose
          sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
      - name: Verify release-doc link policy
        run: site/bin/check-release-doc-links.sh
      - name: Build Apache Site
        run: site/bin/create-static-site.sh

  store-gradle-cache:
    name: Store Gradle Cache
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    permissions:
      contents: read
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    needs:
      - build-checks
      - runtime-service-tests
      - runtime-service-int-tests
      - admin-tool-tests
      - integration-tests
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
      - name: Set up JDK 21
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
        with: *setup-java-vars
      - name: Collect partial Gradle build caches
        uses: ./.github/actions/ci-incr-build-cache-prepare
        with:
          cache-read-only: false

  required-checks:
    # Do not rename this job, this is referenced in .asf.yaml!
    name: "Required Checks"
    needs:
      - build-checks
      - runtime-service-tests
      - runtime-service-int-tests
      - admin-tool-tests
      - integration-tests
      - docker-image-build
      - docker-image-scan
      - helm-tests
      - python-client
      - regtest
      - markdown-link-check
      - site
    # Always run this job, even if a "needed" job fails. Without this one, GitHub will not run
    # this job and that yields "Success" to the branch protection rule, which is wrong.
    # Potential results for each "needed" job are: `success`, `failure`, `cancelled`, `skipped`.
    # We consider `success` and `skipped` as "ok", `failure` and `cancelled` as "not ok".
    if: ${{ always() }}
    runs-on: ubuntu-24.04
    permissions: {}
    steps:
      - env:
          NEEDS_RESULTS: ${{ join(needs.*.result, ',') }}
        run: |
          if [[ "${NEEDS_RESULTS}" =~ .*(failure|cancelled).* ]]; then
            echo "At least one check failed"
            exit 1
          else
            echo "All checks passed"
          fi