Skip to content

Commit bc76e61

Browse files
andygroveandygrove
committed
ci: split iceberg_spark_test workflow per Iceberg version
Move job logic into iceberg_spark_test_reusable.yml and add three thin caller workflows (1.8, 1.9, 1.10). Each caller runs against a single Spark version: Iceberg 1.8 with Spark 3.4 / JDK 11; Iceberg 1.9 and 1.10 with Spark 3.5 / JDK 17. Only the latest (1.10) runs on pull requests; 1.8 and 1.9 run on pushes to main only.
1 parent 48f7b03 commit bc76e61

5 files changed

Lines changed: 251 additions & 108 deletions

File tree

‎.github/workflows/iceberg_spark_test_1_10.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on every PR and on main. Latest supported Iceberg version, paired with
19+
# Spark 3.5.
20+
name: Iceberg Spark SQL Tests (Iceberg 1.10)
21+
22+
concurrency:
23+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
24+
cancel-in-progress: true
25+
26+
on:
27+
push:
28+
branches:
29+
- main
30+
paths:
31+
- "native/**/src/**"
32+
- "native/**/Cargo.toml"
33+
- "native/Cargo.lock"
34+
- "!native/hdfs/**"
35+
- "!native/fs-hdfs/**"
36+
- "common/src/main/**"
37+
- "common/pom.xml"
38+
- "spark/src/main/**"
39+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
40+
- "spark/pom.xml"
41+
- "dev/diffs/iceberg/**"
42+
- "pom.xml"
43+
- "rust-toolchain.toml"
44+
- ".github/workflows/iceberg_spark_test_1_10.yml"
45+
- ".github/workflows/iceberg_spark_test_reusable.yml"
46+
- ".github/actions/setup-builder/**"
47+
- ".github/actions/setup-iceberg-builder/**"
48+
pull_request:
49+
paths:
50+
- "native/**/src/**"
51+
- "native/**/Cargo.toml"
52+
- "native/Cargo.lock"
53+
- "!native/hdfs/**"
54+
- "!native/fs-hdfs/**"
55+
- "common/src/main/**"
56+
- "common/pom.xml"
57+
- "spark/src/main/**"
58+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
59+
- "spark/pom.xml"
60+
- "dev/diffs/iceberg/**"
61+
- "pom.xml"
62+
- "rust-toolchain.toml"
63+
- ".github/workflows/iceberg_spark_test_1_10.yml"
64+
- ".github/workflows/iceberg_spark_test_reusable.yml"
65+
- ".github/actions/setup-builder/**"
66+
- ".github/actions/setup-iceberg-builder/**"
67+
workflow_dispatch:
68+
69+
jobs:
70+
iceberg-spark:
71+
uses: ./.github/workflows/iceberg_spark_test_reusable.yml
72+
with:
73+
iceberg-short: '1.10'
74+
iceberg-full: '1.10.0'
75+
spark-short: '3.5'
76+
spark-full: '3.5.8'
77+
java: 17

‎.github/workflows/iceberg_spark_test_1_8.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on main only. Oldest supported Iceberg version, paired with Spark 3.4.
19+
name: Iceberg Spark SQL Tests (Iceberg 1.8)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/iceberg/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/iceberg_spark_test_1_8.yml"
44+
- ".github/workflows/iceberg_spark_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-iceberg-builder/**"
47+
workflow_dispatch:
48+
49+
jobs:
50+
iceberg-spark:
51+
uses: ./.github/workflows/iceberg_spark_test_reusable.yml
52+
with:
53+
iceberg-short: '1.8'
54+
iceberg-full: '1.8.1'
55+
spark-short: '3.4'
56+
spark-full: '3.4.3'
57+
java: 11

‎.github/workflows/iceberg_spark_test_1_9.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on main only. Iceberg 1.9 paired with Spark 3.5.
19+
name: Iceberg Spark SQL Tests (Iceberg 1.9)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/iceberg/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/iceberg_spark_test_1_9.yml"
44+
- ".github/workflows/iceberg_spark_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-iceberg-builder/**"
47+
workflow_dispatch:
48+
49+
jobs:
50+
iceberg-spark:
51+
uses: ./.github/workflows/iceberg_spark_test_reusable.yml
52+
with:
53+
iceberg-short: '1.9'
54+
iceberg-full: '1.9.1'
55+
spark-short: '3.5'
56+
spark-full: '3.5.8'
57+
java: 17

0 commit comments

Comments
 (0)