Skip to content

Commit 06f8b59

Browse files
andygroveandygrove
committed
Merge remote-tracking branch 'apache/main' into feat/count-mixed-partial-final
2 parents e30d4e8 + 43cf3ec commit 06f8b59

55 files changed

Lines changed: 1122 additions & 1208 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.claude/skills/bug-triage/SKILL.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,7 @@ For each issue, review the title and body and determine:
6565
2. **Area labels** (zero or more): from the area table in the guide
6666
(`area:writer`, `area:shuffle`, `area:aggregation`, `area:scan`,
6767
`area:expressions`, `area:ffi`, `area:ci`) plus the pre-existing area
68-
indicators (`native_datafusion`, `native_iceberg_compat`, `spark 4`,
69-
`spark sql tests`).
68+
indicators (`spark 4`, `spark sql tests`).
7069
3. **Escalation note**: if the issue matches an escalation trigger from the
7170
guide (e.g., a `priority:high` crash that may also produce wrong results),
7271
note it in the summary.

‎.github/workflows/codeql.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,11 @@ jobs:
4949
persist-credentials: false
5050

5151
- name: Initialize CodeQL
52-
uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
52+
uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
5353
with:
5454
languages: actions
5555

5656
- name: Perform CodeQL Analysis
57-
uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
57+
uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
5858
with:
5959
category: "/language:actions"

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,27 +25,41 @@ on:
2525
push:
2626
branches:
2727
- main
28-
paths-ignore:
29-
- "benchmarks/**"
30-
- "doc/**"
31-
- "docs/**"
32-
- "**.md"
33-
- "native/core/benches/**"
34-
- "native/spark-expr/benches/**"
35-
- "spark/src/test/**"
36-
- "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
37-
- "spark-integration/**"
28+
paths:
29+
- "native/**/src/**"
30+
- "native/**/Cargo.toml"
31+
- "native/Cargo.lock"
32+
- "!native/hdfs/**"
33+
- "!native/fs-hdfs/**"
34+
- "common/src/main/**"
35+
- "common/pom.xml"
36+
- "spark/src/main/**"
37+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
38+
- "spark/pom.xml"
39+
- "dev/diffs/iceberg/**"
40+
- "pom.xml"
41+
- "rust-toolchain.toml"
42+
- ".github/workflows/iceberg_spark_test.yml"
43+
- ".github/actions/setup-builder/**"
44+
- ".github/actions/setup-iceberg-builder/**"
3845
pull_request:
39-
paths-ignore:
40-
- "benchmarks/**"
41-
- "doc/**"
42-
- "docs/**"
43-
- "**.md"
44-
- "native/core/benches/**"
45-
- "native/spark-expr/benches/**"
46-
- "spark/src/test/**"
47-
- "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
48-
- "spark-integration/**"
46+
paths:
47+
- "native/**/src/**"
48+
- "native/**/Cargo.toml"
49+
- "native/Cargo.lock"
50+
- "!native/hdfs/**"
51+
- "!native/fs-hdfs/**"
52+
- "common/src/main/**"
53+
- "common/pom.xml"
54+
- "spark/src/main/**"
55+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
56+
- "spark/pom.xml"
57+
- "dev/diffs/iceberg/**"
58+
- "pom.xml"
59+
- "rust-toolchain.toml"
60+
- ".github/workflows/iceberg_spark_test.yml"
61+
- ".github/actions/setup-builder/**"
62+
- ".github/actions/setup-iceberg-builder/**"
4963
# manual trigger
5064
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
5165
workflow_dispatch:

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -136,18 +136,6 @@ jobs:
136136
java_version: "17"
137137
maven_opts: "-Pspark-4.0 -Pscala-2.13"
138138

139-
- name: "Spark 4.1, JDK 17, Scala 2.13"
140-
java_version: "17"
141-
# The spark-4.1 profile already pins Scala to 2.13.17 to match Spark 4.1.1's
142-
# runtime; the scala-2.13 profile would override it back to 2.13.16 and break.
143-
maven_opts: "-Pspark-4.1"
144-
145-
- name: "Spark 4.2, JDK 17, Scala 2.13"
146-
java_version: "17"
147-
# The spark-4.2 profile pins Scala to 2.13.18 to match Spark 4.2.0-preview4's
148-
# runtime; the scala-2.13 profile would override it back to 2.13.16 and break.
149-
maven_opts: "-Pspark-4.2"
150-
151139
suite:
152140
- name: "fuzz"
153141
value: |

‎.github/workflows/spark_sql_test_3_4.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on main only. Extra coverage for the oldest supported Spark.
19+
name: Spark SQL Tests (Spark 3.4)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/spark_sql_test_3_4.yml"
44+
- ".github/workflows/spark_sql_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-spark-builder/**"
47+
# On-demand PR runs: a committer adds the `run-spark-3.4-tests` label
48+
# and the workflow runs against the PR's merge ref. Works for forks.
49+
pull_request:
50+
types: [labeled]
51+
workflow_dispatch:
52+
inputs:
53+
collect-fallback-logs:
54+
description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
55+
required: false
56+
default: false
57+
type: boolean
58+
59+
jobs:
60+
spark-sql:
61+
if: github.event_name != 'pull_request' || github.event.label.name == 'run-spark-3.4-tests'
62+
uses: ./.github/workflows/spark_sql_test_reusable.yml
63+
with:
64+
spark-short: '3.4'
65+
spark-full: '3.4.3'
66+
java: 11
67+
collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}

‎.github/workflows/spark_sql_test_3_5.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on every PR and on main. Spark 3.5 is the default supported version.
19+
name: Spark SQL Tests (Spark 3.5)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/spark_sql_test_3_5.yml"
44+
- ".github/workflows/spark_sql_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-spark-builder/**"
47+
pull_request:
48+
paths:
49+
- "native/**/src/**"
50+
- "native/**/Cargo.toml"
51+
- "native/Cargo.lock"
52+
- "!native/hdfs/**"
53+
- "!native/fs-hdfs/**"
54+
- "common/src/main/**"
55+
- "common/pom.xml"
56+
- "spark/src/main/**"
57+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
58+
- "spark/pom.xml"
59+
- "dev/diffs/**"
60+
- "pom.xml"
61+
- "rust-toolchain.toml"
62+
- ".github/workflows/spark_sql_test_3_5.yml"
63+
- ".github/workflows/spark_sql_test_reusable.yml"
64+
- ".github/actions/setup-builder/**"
65+
- ".github/actions/setup-spark-builder/**"
66+
workflow_dispatch:
67+
inputs:
68+
collect-fallback-logs:
69+
description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
70+
required: false
71+
default: false
72+
type: boolean
73+
74+
jobs:
75+
spark-sql:
76+
uses: ./.github/workflows/spark_sql_test_reusable.yml
77+
with:
78+
spark-short: '3.5'
79+
spark-full: '3.5.8'
80+
java: 11
81+
collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}

‎.github/workflows/spark_sql_test_4_0.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on every PR and on main. Spark 4.0 is the newest stable Spark line.
19+
name: Spark SQL Tests (Spark 4.0)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/spark_sql_test_4_0.yml"
44+
- ".github/workflows/spark_sql_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-spark-builder/**"
47+
pull_request:
48+
paths:
49+
- "native/**/src/**"
50+
- "native/**/Cargo.toml"
51+
- "native/Cargo.lock"
52+
- "!native/hdfs/**"
53+
- "!native/fs-hdfs/**"
54+
- "common/src/main/**"
55+
- "common/pom.xml"
56+
- "spark/src/main/**"
57+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
58+
- "spark/pom.xml"
59+
- "dev/diffs/**"
60+
- "pom.xml"
61+
- "rust-toolchain.toml"
62+
- ".github/workflows/spark_sql_test_4_0.yml"
63+
- ".github/workflows/spark_sql_test_reusable.yml"
64+
- ".github/actions/setup-builder/**"
65+
- ".github/actions/setup-spark-builder/**"
66+
workflow_dispatch:
67+
inputs:
68+
collect-fallback-logs:
69+
description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
70+
required: false
71+
default: false
72+
type: boolean
73+
74+
jobs:
75+
spark-sql:
76+
uses: ./.github/workflows/spark_sql_test_reusable.yml
77+
with:
78+
spark-short: '4.0'
79+
spark-full: '4.0.2'
80+
java: 21
81+
collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}

0 commit comments

Comments
 (0)