Skip to content

Commit 234e6e7

Browse files
andygroveclaude
andygrove
and
claude
committed
Merge remote-tracking branch 'apache/main' into add-jvm-shuffle-benchmarks
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2 parents c51c3ab + d89e50a commit 234e6e7

243 files changed

Lines changed: 10939 additions & 2947 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/java-test/action.yaml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ inputs:
3232
scan_impl:
3333
description: 'The default Parquet scan implementation'
3434
required: false
35-
default: 'native_comet'
35+
default: 'auto'
3636
upload-test-reports:
3737
description: 'Whether to upload test results including coverage to GitHub'
3838
required: false
@@ -146,7 +146,3 @@ runs:
146146
path: "**/target/surefire-reports/*.txt"
147147
retention-days: 7 # 1 week for test reports
148148
overwrite: true
149-
150-
- name: Upload coverage results
151-
if: ${{ inputs.upload-test-reports == 'true' }}
152-
uses: codecov/codecov-action@v5

‎.github/actions/setup-spark-builder/action.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ inputs:
2222
description: 'The Apache Spark short version (e.g., 3.5) to build'
2323
required: true
2424
spark-version:
25-
description: 'The Apache Spark version (e.g., 3.5.7) to build'
25+
description: 'The Apache Spark version (e.g., 3.5.8) to build'
2626
required: true
2727
skip-native-build:
2828
description: 'Skip native build (when using pre-built artifact)'

‎.github/workflows/docs.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
python-version: "3.10"
4848

4949
- name: Setup Java
50-
uses: actions/setup-java@v4
50+
uses: actions/setup-java@v5
5151
with:
5252
distribution: 'temurin'
5353
java-version: '17'

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 99 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,66 @@ env:
4646
RUST_VERSION: stable
4747

4848
jobs:
49+
# Build native library once and share with all test jobs
50+
build-native:
51+
if: contains(github.event.pull_request.title, '[iceberg]')
52+
name: Build Native Library
53+
runs-on: ubuntu-24.04
54+
container:
55+
image: amd64/rust
56+
steps:
57+
- uses: actions/checkout@v6
58+
59+
- name: Setup Rust & Java toolchain
60+
uses: ./.github/actions/setup-builder
61+
with:
62+
rust-version: ${{ env.RUST_VERSION }}
63+
jdk-version: 17
64+
65+
- name: Restore Cargo cache
66+
uses: actions/cache/restore@v5
67+
with:
68+
path: |
69+
~/.cargo/registry
70+
~/.cargo/git
71+
native/target
72+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
73+
restore-keys: |
74+
${{ runner.os }}-cargo-ci-
75+
76+
- name: Build native library
77+
# Use CI profile for faster builds (no LTO) and to share cache with pr_build_linux.yml.
78+
run: |
79+
cd native && cargo build --profile ci
80+
env:
81+
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3"
82+
83+
- name: Save Cargo cache
84+
uses: actions/cache/save@v5
85+
if: github.ref == 'refs/heads/main'
86+
with:
87+
path: |
88+
~/.cargo/registry
89+
~/.cargo/git
90+
native/target
91+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
92+
93+
- name: Upload native library
94+
uses: actions/upload-artifact@v6
95+
with:
96+
name: native-lib-iceberg
97+
path: native/target/ci/libcomet.so
98+
retention-days: 1
99+
49100
iceberg-spark:
101+
needs: build-native
50102
if: contains(github.event.pull_request.title, '[iceberg]')
51103
strategy:
52104
matrix:
53105
os: [ubuntu-24.04]
54106
java-version: [11, 17]
55107
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
56-
spark-version: [{short: '3.5', full: '3.5.7'}]
108+
spark-version: [{short: '3.5', full: '3.5.8'}]
57109
scala-version: ['2.13']
58110
fail-fast: false
59111
name: iceberg-spark/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -69,10 +121,14 @@ jobs:
69121
with:
70122
rust-version: ${{env.RUST_VERSION}}
71123
jdk-version: ${{ matrix.java-version }}
124+
- name: Download native library
125+
uses: actions/download-artifact@v7
126+
with:
127+
name: native-lib-iceberg
128+
path: native/target/release/
72129
- name: Build Comet
73-
shell: bash
74130
run: |
75-
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
131+
./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }}
76132
- name: Setup Iceberg
77133
uses: ./.github/actions/setup-iceberg-builder
78134
with:
@@ -86,13 +142,14 @@ jobs:
86142
-Pquick=true -x javadoc
87143
88144
iceberg-spark-extensions:
145+
needs: build-native
89146
if: contains(github.event.pull_request.title, '[iceberg]')
90147
strategy:
91148
matrix:
92149
os: [ubuntu-24.04]
93150
java-version: [11, 17]
94151
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
95-
spark-version: [{short: '3.5', full: '3.5.7'}]
152+
spark-version: [{short: '3.5', full: '3.5.8'}]
96153
scala-version: ['2.13']
97154
fail-fast: false
98155
name: iceberg-spark-extensions/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -108,10 +165,14 @@ jobs:
108165
with:
109166
rust-version: ${{env.RUST_VERSION}}
110167
jdk-version: ${{ matrix.java-version }}
168+
- name: Download native library
169+
uses: actions/download-artifact@v7
170+
with:
171+
name: native-lib-iceberg
172+
path: native/target/release/
111173
- name: Build Comet
112-
shell: bash
113174
run: |
114-
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
175+
./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }}
115176
- name: Setup Iceberg
116177
uses: ./.github/actions/setup-iceberg-builder
117178
with:
@@ -125,13 +186,14 @@ jobs:
125186
-Pquick=true -x javadoc
126187
127188
iceberg-spark-runtime:
189+
needs: build-native
128190
if: contains(github.event.pull_request.title, '[iceberg]')
129191
strategy:
130192
matrix:
131193
os: [ubuntu-24.04]
132194
java-version: [11, 17]
133195
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
134-
spark-version: [{short: '3.5', full: '3.5.7'}]
196+
spark-version: [{short: '3.5', full: '3.5.8'}]
135197
scala-version: ['2.13']
136198
fail-fast: false
137199
name: iceberg-spark-runtime/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -147,10 +209,14 @@ jobs:
147209
with:
148210
rust-version: ${{env.RUST_VERSION}}
149211
jdk-version: ${{ matrix.java-version }}
212+
- name: Download native library
213+
uses: actions/download-artifact@v7
214+
with:
215+
name: native-lib-iceberg
216+
path: native/target/release/
150217
- name: Build Comet
151-
shell: bash
152218
run: |
153-
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
219+
./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }}
154220
- name: Setup Iceberg
155221
uses: ./.github/actions/setup-iceberg-builder
156222
with:
@@ -164,13 +230,14 @@ jobs:
164230
-Pquick=true -x javadoc
165231
166232
iceberg-spark-rust:
233+
needs: build-native
167234
if: contains(github.event.pull_request.title, '[iceberg]')
168235
strategy:
169236
matrix:
170237
os: [ubuntu-24.04]
171238
java-version: [11, 17]
172239
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
173-
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
240+
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
174241
scala-version: ['2.13']
175242
fail-fast: false
176243
name: iceberg-spark-rust/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -186,10 +253,14 @@ jobs:
186253
with:
187254
rust-version: ${{env.RUST_VERSION}}
188255
jdk-version: ${{ matrix.java-version }}
256+
- name: Download native library
257+
uses: actions/download-artifact@v7
258+
with:
259+
name: native-lib-iceberg
260+
path: native/target/release/
189261
- name: Build Comet
190-
shell: bash
191262
run: |
192-
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
263+
./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }}
193264
- name: Setup Iceberg
194265
uses: ./.github/actions/setup-iceberg-rust-builder
195266
with:
@@ -203,13 +274,14 @@ jobs:
203274
-Pquick=true -x javadoc
204275
205276
iceberg-spark-extensions-rust:
277+
needs: build-native
206278
if: contains(github.event.pull_request.title, '[iceberg]')
207279
strategy:
208280
matrix:
209281
os: [ubuntu-24.04]
210282
java-version: [11, 17]
211283
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
212-
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
284+
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
213285
scala-version: ['2.13']
214286
fail-fast: false
215287
name: iceberg-spark-extensions-rust/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -225,10 +297,14 @@ jobs:
225297
with:
226298
rust-version: ${{env.RUST_VERSION}}
227299
jdk-version: ${{ matrix.java-version }}
300+
- name: Download native library
301+
uses: actions/download-artifact@v7
302+
with:
303+
name: native-lib-iceberg
304+
path: native/target/release/
228305
- name: Build Comet
229-
shell: bash
230306
run: |
231-
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
307+
./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }}
232308
- name: Setup Iceberg
233309
uses: ./.github/actions/setup-iceberg-rust-builder
234310
with:
@@ -242,13 +318,14 @@ jobs:
242318
-Pquick=true -x javadoc
243319
244320
iceberg-spark-runtime-rust:
321+
needs: build-native
245322
if: contains(github.event.pull_request.title, '[iceberg]')
246323
strategy:
247324
matrix:
248325
os: [ubuntu-24.04]
249326
java-version: [11, 17]
250327
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}]
251-
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
328+
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}]
252329
scala-version: ['2.13']
253330
fail-fast: false
254331
name: iceberg-spark-runtime-rust/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -264,10 +341,14 @@ jobs:
264341
with:
265342
rust-version: ${{env.RUST_VERSION}}
266343
jdk-version: ${{ matrix.java-version }}
344+
- name: Download native library
345+
uses: actions/download-artifact@v7
346+
with:
347+
name: native-lib-iceberg
348+
path: native/target/release/
267349
- name: Build Comet
268-
shell: bash
269350
run: |
270-
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
351+
./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }}
271352
- name: Setup Iceberg
272353
uses: ./.github/actions/setup-iceberg-rust-builder
273354
with:

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ jobs:
9494
# CI profile: same overflow behavior as release, but faster compilation
9595
# (no LTO, parallel codegen)
9696
cargo build --profile ci
97+
env:
98+
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3"
9799

98100
- name: Upload native library
99101
uses: actions/upload-artifact@v6
@@ -164,7 +166,7 @@ jobs:
164166
- name: "Spark 3.4, JDK 11, Scala 2.12"
165167
java_version: "11"
166168
maven_opts: "-Pspark-3.4 -Pscala-2.12"
167-
scan_impl: "native_comet"
169+
scan_impl: "auto"
168170

169171
- name: "Spark 3.5.5, JDK 17, Scala 2.13"
170172
java_version: "17"
@@ -174,7 +176,7 @@ jobs:
174176
- name: "Spark 3.5.6, JDK 17, Scala 2.13"
175177
java_version: "17"
176178
maven_opts: "-Pspark-3.5 -Dspark.version=3.5.6 -Pscala-2.13"
177-
scan_impl: "native_comet"
179+
scan_impl: "auto"
178180

179181
- name: "Spark 3.5, JDK 17, Scala 2.12"
180182
java_version: "17"
@@ -250,6 +252,7 @@ jobs:
250252
- name: "expressions"
251253
value: |
252254
org.apache.comet.CometExpressionSuite
255+
org.apache.comet.CometSqlFileTestSuite
253256
org.apache.comet.CometExpressionCoverageSuite
254257
org.apache.comet.CometHashExpressionSuite
255258
org.apache.comet.CometTemporalExpressionSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ jobs:
9494
# CI profile: same overflow behavior as release, but faster compilation
9595
# (no LTO, parallel codegen)
9696
cargo build --profile ci
97+
env:
98+
RUSTFLAGS: "-Ctarget-cpu=apple-m1"
9799

98100
- name: Upload native library
99101
uses: actions/upload-artifact@v6
@@ -193,6 +195,7 @@ jobs:
193195
- name: "expressions"
194196
value: |
195197
org.apache.comet.CometExpressionSuite
198+
org.apache.comet.CometSqlFileTestSuite
196199
org.apache.comet.CometExpressionCoverageSuite
197200
org.apache.comet.CometHashExpressionSuite
198201
org.apache.comet.CometTemporalExpressionSuite

0 commit comments

Comments
 (0)