spark_sql_test_reusable.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Reusable Spark SQL test workflow. Invoked once per Spark version by the
# spark_sql_test_<version>.yml caller workflows. Keep all job logic here so
# the per-version callers stay thin.

name: Spark SQL Tests (reusable)

on:
  workflow_call:
    inputs:
      spark-short:
        description: 'Spark minor version, e.g. 3.5'
        required: true
        type: string
      spark-full:
        description: 'Spark full version, e.g. 3.5.8'
        required: true
        type: string
      java:
        description: 'JDK major version, e.g. 17'
        required: true
        type: number
      collect-fallback-logs:
        description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
        required: false
        default: false
        type: boolean

env:
  RUST_VERSION: stable
  RUST_BACKTRACE: 1
  # Force GNU ld on Linux: recent Rust stable defaults to rust-lld on
  # x86_64-unknown-linux-gnu, and rust-lld cannot resolve -ljvm against the
  # Zulu JDK layout installed by setup-java. Keep bfd for all cargo invocations.
  RUSTFLAGS: "-Clink-arg=-fuse-ld=bfd"

jobs:

  # Build native library once and share with all test jobs
  build-native:
    name: Build Native Library
    runs-on: ubuntu-24.04
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v6

      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ env.RUST_VERSION }}
          jdk-version: 17

      - name: Restore Cargo cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            native/target
          key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
          restore-keys: |
            ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-

      - name: Build native library (CI profile)
        run: |
          cd native
          cargo build --profile ci
        env:
          RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd"

      - name: Upload native library
        uses: actions/upload-artifact@v7
        with:
          name: native-lib-linux
          path: native/target/ci/libcomet.so
          retention-days: 1

      - name: Save Cargo cache
        uses: actions/cache/save@v5
        if: github.ref == 'refs/heads/main'
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            native/target
          key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}

  spark-sql-test:
    needs: build-native
    strategy:
      matrix:
        module:
          - {name: "catalyst", args1: "catalyst/test", args2: ""}
          - {name: "sql_core-1", args1: "", args2: "sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest"}
          - {name: "sql_core-2", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.ExtendedSQLTest"}
          - {name: "sql_core-3", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest"}
          - {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
          - {name: "sql_hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"}
          - {name: "sql_hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
      fail-fast: false
    name: spark-sql-${{ matrix.module.name }}/spark-${{ inputs.spark-full }}-jdk${{ inputs.java }}
    runs-on: ubuntu-24.04
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v6
      - name: Free up disk space (container)
        # Mirror apache/spark's dev/free_disk_space_container. The amd64/rust
        # container inherits GitHub's tool-cache via the /__t mount; strip the
        # entries we never use (CodeQL, Go, Node) to free ~5-10 GB before the
        # Spark clone, Maven dep tree, sbt build cache, and per-test-suite
        # warehouse dirs all start fighting for the runner's disk.
        shell: bash
        run: |
          echo "Disk usage before cleanup:"
          df -h || true
          rm -rf /__t/CodeQL || true
          rm -rf /__t/go     || true
          rm -rf /__t/node   || true
          echo "Disk usage after cleanup:"
          df -h || true
      - name: Setup Rust & Java toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{env.RUST_VERSION}}
          jdk-version: ${{ inputs.java }}
      - name: Download native library
        uses: actions/download-artifact@v8
        with:
          name: native-lib-linux
          path: native/target/release/
      - name: Setup Spark
        uses: ./.github/actions/setup-spark-builder
        with:
          spark-version: ${{ inputs.spark-full }}
          spark-short-version: ${{ inputs.spark-short }}
          skip-native-build: true
      - name: Run Spark tests
        run: |
          cd apache-spark
          rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
          # SERIAL_SBT_TESTS gates SparkParallelTestGrouping in
          # project/SparkBuild.scala. For Spark 4.0.2 on JDK 21 we
          # leave it unset so the grouping is installed and
          # DEDICATED_JVM_SBT_TESTS below actually forks a dedicated
          # JVM per listed suite, working around the V1/V2 Parquet and
          # Orc source-suite cross-suite file-stream leak under JDK 21
          # (issue #4327). For other rows we keep it set to reduce
          # peak memory on standard 7 GB runners.
          if [ "${{ inputs.spark-short }}" != "4.0" ] || [ "${{ inputs.java }}" != "21" ]; then
            export SERIAL_SBT_TESTS=1
          fi
          # Cap parallel forked test JVMs at 1: on a 7 GB runner the 3 GB SBT
          # controller and a single 5 GB forked test JVM (HEAP_SIZE override
          # below) already over-commit RAM; any concurrency on top would push
          # us straight into OOMs.
          NOLINT_ON_COMPILE=true ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ENABLE_COMET_LOG_FALLBACK_REASONS=${{ inputs.collect-fallback-logs }} \
            build/sbt -Dsbt.log.noformat=true -mem $SBT_MEM \
              'set Global / concurrentRestrictions := Seq(Tags.limit(Tags.ForkedTestGroup, 1))' \
              ${{ matrix.module.args1 }} "${{ matrix.module.args2 }}"
          if [ "${{ inputs.collect-fallback-logs }}" = "true" ]; then
            find . -type f -name "unit-tests.log" -print0 | xargs -0 grep -h "Comet cannot accelerate" | sed 's/.*Comet cannot accelerate/Comet cannot accelerate/' | sort -u > fallback.log
          fi
        env:
          LC_ALL: "C.UTF-8"
          SBT_MEM: "3072"
          HEAP_SIZE: "3g"
          # Cap forked test JVM direct memory. Without this, -XX:MaxDirectMemorySize
          # defaults to ≈ -Xmx (5 GB), letting Netty / DirectByteBuffer / shuffle
          # buffers grow alongside the heap and push host RSS past the 7 GB
          # runner ceiling. JAVA_TOOL_OPTIONS is honoured by every child JVM;
          # SBT picks it up too but allocates negligible direct memory itself.
          JAVA_TOOL_OPTIONS: "-XX:MaxDirectMemorySize=1g"
          # Force these suites into their own forked JVMs so per-suite state
          # (codegen classes, native pools, shuffle buffers) is reclaimed on
          # JVM exit instead of accumulating across a single testOnly run:
          #   * AdaptiveQueryExecSuite — long, codegen- and shuffle-heavy;
          #     observed to SIGKILL the host on 7 GB runners after ~10 tests.
          #   * Parquet/Orc V1/V2 source suites — Spark 4.0 only; mirrors
          #     apache/spark's build_java21.yml / build_java25.yml workaround
          #     for cross-suite file-stream / thread leaks under newer JDKs.
          # SparkBuild.scala reads this env var and installs the testGrouping
          # regardless of SERIAL_SBT_TESTS, so listed suites still get their
          # own JVM in serial-test mode (only run policy changes).
          DEDICATED_JVM_SBT_TESTS: ${{ inputs.spark-short == '4.0' && 'org.apache.spark.sql.execution.adaptive.AdaptiveQueryExecSuite,org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV1Suite,org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV2Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV1Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV2Suite' || 'org.apache.spark.sql.execution.adaptive.AdaptiveQueryExecSuite' }}
      - name: Dump kernel + memory state on failure
        if: failure()
        shell: bash
        run: |
          echo '== /proc/meminfo =='; cat /proc/meminfo 2>/dev/null || true
          echo '== /proc/swaps ==';   cat /proc/swaps   2>/dev/null || true
          echo '== top RSS ==';       ps -eo pid,rss,cmd --sort=-rss 2>/dev/null | head -20 || true
          echo '== dmesg (best effort) ==';
          dmesg -T 2>/dev/null | tail -200 || cat /var/log/kern.log 2>/dev/null | tail -200 || echo 'kernel log unavailable inside container'
      - name: Upload fallback log
        if: ${{ inputs.collect-fallback-logs }}
        uses: actions/upload-artifact@v7
        with:
          name: fallback-log-spark-sql-${{ matrix.module.name }}-spark-${{ inputs.spark-full }}-jdk${{ inputs.java }}
          path: "**/fallback.log"

  merge-fallback-logs:
    if: ${{ inputs.collect-fallback-logs }}
    name: merge-fallback-logs
    needs: [spark-sql-test]
    runs-on: ubuntu-24.04
    steps:
      - name: Download fallback log artifacts
        uses: actions/download-artifact@v8
        with:
          path: fallback-logs/
      - name: Merge fallback logs
        run: |
          find ./fallback-logs/ -type f -name "fallback.log" -print0 | xargs -0 cat | sort -u > all_fallback.log
      - name: Upload merged fallback log
        uses: actions/upload-artifact@v7
        with:
          name: all-fallback-log-spark-${{ inputs.spark-full }}-jdk${{ inputs.java }}
          path: all_fallback.log