datafusion-comet/.github/workflows/spark_sql_test_4_1.yml at 8abbff2ee4349972f252d63c52edfcfae219566e · apache/datafusion-comet · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Runs on main only. Forward-looking coverage for the in-development Spark line.
name: Spark SQL Tests (Spark 4.1)

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

on:
  push:
    branches:
      - main
    paths:
      - "native/**/src/**"
      - "native/**/Cargo.toml"
      - "native/Cargo.lock"
      - "!native/hdfs/**"
      - "!native/fs-hdfs/**"
      - "common/src/main/**"
      - "common/pom.xml"
      - "spark/src/main/**"
      - "!spark/src/main/spark-3.4/**"
      - "!spark/src/main/spark-3.5/**"
      - "!spark/src/main/spark-3.x/**"
      - "!spark/src/main/spark-4.0/**"
      - "!spark/src/main/spark-4.2/**"
      - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
      - "spark/pom.xml"
      - "dev/diffs/4.1.1.diff"
      - "pom.xml"
      - "rust-toolchain.toml"
      - ".github/workflows/spark_sql_test_4_1.yml"
      - ".github/workflows/spark_sql_test_reusable.yml"
      - ".github/actions/setup-builder/**"
      - ".github/actions/setup-spark-builder/**"
  # On-demand PR runs: a committer adds the `run-spark-4.1-tests` label
  # and the workflow runs against the PR's merge ref. Works for forks.
  pull_request:
    types: [labeled]
  workflow_dispatch:
    inputs:
      collect-fallback-logs:
        description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
        required: false
        default: false
        type: boolean

jobs:
  spark-sql:
    if: github.event_name != 'pull_request' || github.event.label.name == 'run-spark-4.1-tests'
    uses: ./.github/workflows/spark_sql_test_reusable.yml
    with:
      spark-short: '4.1'
      spark-full: '4.1.1'
      java: 17
      collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}