Skip to content

Commit c6c3002

Browse files
andygroveandygrove
authored
build: build native library once and share across CI test jobs (#3249)
1 parent 5d6ed61 commit c6c3002

6 files changed

Lines changed: 230 additions & 7 deletions

File tree

‎.github/actions/java-test/action.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,16 @@ inputs:
3737
description: 'Whether to upload test results including coverage to GitHub'
3838
required: false
3939
default: 'false'
40+
skip-native-build:
41+
description: 'Skip native build (when using pre-built artifact)'
42+
required: false
43+
default: 'false'
4044

4145
runs:
4246
using: "composite"
4347
steps:
4448
- name: Run Cargo release build
49+
if: ${{ inputs.skip-native-build != 'true' }}
4550
shell: bash
4651
# it is important that we run the Scala tests against a release build rather than a debug build
4752
# to make sure that no tests are relying on overflow checks that are present only in debug builds

‎.github/actions/setup-spark-builder/action.yaml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ inputs:
2424
spark-version:
2525
description: 'The Apache Spark version (e.g., 3.5.7) to build'
2626
required: true
27+
skip-native-build:
28+
description: 'Skip native build (when using pre-built artifact)'
29+
required: false
30+
default: 'false'
2731
runs:
2832
using: "composite"
2933
steps:
@@ -51,7 +55,15 @@ runs:
5155
restore-keys: |
5256
${{ runner.os }}-spark-sql-
5357
54-
- name: Build Comet
58+
- name: Build Comet (with native)
59+
if: ${{ inputs.skip-native-build != 'true' }}
5560
shell: bash
5661
run: |
5762
PROFILES="-Pspark-${{inputs.spark-short-version}}" make release
63+
64+
- name: Build Comet (Maven only, skip native)
65+
if: ${{ inputs.skip-native-build == 'true' }}
66+
shell: bash
67+
run: |
68+
# Native library should already be in native/target/release/
69+
./mvnw install -Prelease -DskipTests -Pspark-${{inputs.spark-short-version}}

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 80 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,48 @@ env:
4646
RUST_VERSION: stable
4747

4848
jobs:
49-
50-
# Run Rust tests once per JDK version
49+
50+
# Build native library once and share with all test jobs
51+
build-native:
52+
name: Build Native Library
53+
runs-on: ubuntu-latest
54+
container:
55+
image: amd64/rust
56+
steps:
57+
- uses: actions/checkout@v6
58+
59+
- name: Setup Rust toolchain
60+
uses: ./.github/actions/setup-builder
61+
with:
62+
rust-version: ${{ env.RUST_VERSION }}
63+
jdk-version: 17 # JDK only needed for common module proto generation
64+
65+
- name: Cache Cargo
66+
uses: actions/cache@v4
67+
with:
68+
path: |
69+
~/.cargo/registry
70+
~/.cargo/git
71+
native/target
72+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
73+
restore-keys: |
74+
${{ runner.os }}-cargo-ci-
75+
76+
- name: Build native library (CI profile)
77+
run: |
78+
cd native
79+
# CI profile: same overflow behavior as release, but faster compilation
80+
# (no LTO, parallel codegen)
81+
cargo build --profile ci
82+
83+
- name: Upload native library
84+
uses: actions/upload-artifact@v4
85+
with:
86+
name: native-lib-linux
87+
path: native/target/ci/libcomet.so
88+
retention-days: 1
89+
90+
# Run Rust tests (runs in parallel with build-native, uses debug builds)
5191
linux-test-rust:
5292
strategy:
5393
matrix:
@@ -60,15 +100,29 @@ jobs:
60100
image: amd64/rust
61101
steps:
62102
- uses: actions/checkout@v6
103+
63104
- name: Setup Rust & Java toolchain
64105
uses: ./.github/actions/setup-builder
65106
with:
66-
rust-version: ${{env.RUST_VERSION}}
107+
rust-version: ${{ env.RUST_VERSION }}
67108
jdk-version: ${{ matrix.java_version }}
109+
110+
- name: Cache Cargo
111+
uses: actions/cache@v4
112+
with:
113+
path: |
114+
~/.cargo/registry
115+
~/.cargo/git
116+
native/target
117+
key: ${{ runner.os }}-cargo-debug-java${{ matrix.java_version }}-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
118+
restore-keys: |
119+
${{ runner.os }}-cargo-debug-java${{ matrix.java_version }}-
120+
68121
- name: Rust test steps
69122
uses: ./.github/actions/rust-test
70-
123+
71124
linux-test:
125+
needs: build-native
72126
strategy:
73127
matrix:
74128
os: [ubuntu-latest]
@@ -186,11 +240,31 @@ jobs:
186240

187241
steps:
188242
- uses: actions/checkout@v6
243+
189244
- name: Setup Rust & Java toolchain
190245
uses: ./.github/actions/setup-builder
191246
with:
192-
rust-version: ${{env.RUST_VERSION}}
247+
rust-version: ${{ env.RUST_VERSION }}
193248
jdk-version: ${{ matrix.profile.java_version }}
249+
250+
- name: Download native library
251+
uses: actions/download-artifact@v4
252+
with:
253+
name: native-lib-linux
254+
# Download to release/ since Maven's -Prelease expects libcomet.so there
255+
path: native/target/release/
256+
257+
# Restore cargo registry cache (for any cargo commands that might run)
258+
- name: Cache Cargo registry
259+
uses: actions/cache@v4
260+
with:
261+
path: |
262+
~/.cargo/registry
263+
~/.cargo/git
264+
key: ${{ runner.os }}-cargo-registry-${{ hashFiles('native/**/Cargo.lock') }}
265+
restore-keys: |
266+
${{ runner.os }}-cargo-registry-
267+
194268
- name: Java test steps
195269
uses: ./.github/actions/java-test
196270
with:
@@ -199,3 +273,4 @@ jobs:
199273
maven_opts: ${{ matrix.profile.maven_opts }}
200274
scan_impl: ${{ matrix.profile.scan_impl }}
201275
upload-test-reports: true
276+
skip-native-build: true

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,48 @@ env:
4747

4848
jobs:
4949

50+
# Build native library once and share with all test jobs
51+
build-native:
52+
name: Build Native Library (macOS)
53+
runs-on: macos-14
54+
steps:
55+
- uses: actions/checkout@v6
56+
57+
- name: Setup Rust & Java toolchain
58+
uses: ./.github/actions/setup-macos-builder
59+
with:
60+
rust-version: ${{ env.RUST_VERSION }}
61+
jdk-version: 17
62+
jdk-architecture: aarch64
63+
protoc-architecture: aarch_64
64+
65+
- name: Cache Cargo
66+
uses: actions/cache@v4
67+
with:
68+
path: |
69+
~/.cargo/registry
70+
~/.cargo/git
71+
native/target
72+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
73+
restore-keys: |
74+
${{ runner.os }}-cargo-ci-
75+
76+
- name: Build native library (CI profile)
77+
run: |
78+
cd native
79+
# CI profile: same overflow behavior as release, but faster compilation
80+
# (no LTO, parallel codegen)
81+
cargo build --profile ci
82+
83+
- name: Upload native library
84+
uses: actions/upload-artifact@v4
85+
with:
86+
name: native-lib-macos
87+
path: native/target/ci/libcomet.dylib
88+
retention-days: 1
89+
5090
macos-aarch64-test:
91+
needs: build-native
5192
strategy:
5293
matrix:
5394
os: [macos-14]
@@ -145,13 +186,33 @@ jobs:
145186
runs-on: ${{ matrix.os }}
146187
steps:
147188
- uses: actions/checkout@v6
189+
148190
- name: Setup Rust & Java toolchain
149191
uses: ./.github/actions/setup-macos-builder
150192
with:
151-
rust-version: ${{env.RUST_VERSION}}
193+
rust-version: ${{ env.RUST_VERSION }}
152194
jdk-version: ${{ matrix.profile.java_version }}
153195
jdk-architecture: aarch64
154196
protoc-architecture: aarch_64
197+
198+
- name: Download native library
199+
uses: actions/download-artifact@v4
200+
with:
201+
name: native-lib-macos
202+
# Download to release/ since Maven's -Prelease expects libcomet.dylib there
203+
path: native/target/release/
204+
205+
# Restore cargo registry cache (for any cargo commands that might run)
206+
- name: Cache Cargo registry
207+
uses: actions/cache@v4
208+
with:
209+
path: |
210+
~/.cargo/registry
211+
~/.cargo/git
212+
key: ${{ runner.os }}-cargo-registry-${{ hashFiles('native/**/Cargo.lock') }}
213+
restore-keys: |
214+
${{ runner.os }}-cargo-registry-
215+
155216
- name: Set thread thresholds envs for spark test on macOS
156217
# see: https://github.com/apache/datafusion-comet/issues/2965
157218
shell: bash
@@ -160,9 +221,11 @@ jobs:
160221
echo "SPARK_TEST_SQL_RESULT_QUERY_STAGE_MAX_THREAD_THRESHOLD=256" >> $GITHUB_ENV
161222
echo "SPARK_TEST_HIVE_SHUFFLE_EXCHANGE_MAX_THREAD_THRESHOLD=48" >> $GITHUB_ENV
162223
echo "SPARK_TEST_HIVE_RESULT_QUERY_STAGE_MAX_THREAD_THRESHOLD=48" >> $GITHUB_ENV
224+
163225
- name: Java test steps
164226
uses: ./.github/actions/java-test
165227
with:
166228
artifact_name: ${{ matrix.os }}-${{ matrix.profile.name }}-${{ matrix.suite.name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
167229
suites: ${{ matrix.suite.name == 'sql' && matrix.profile.name == 'Spark 3.4, JDK 11, Scala 2.12' && '' || matrix.suite.value }}
168230
maven_opts: ${{ matrix.profile.maven_opts }}
231+
skip-native-build: true

‎.github/workflows/spark_sql_test.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,47 @@ env:
5252
RUST_VERSION: stable
5353

5454
jobs:
55+
56+
# Build native library once and share with all test jobs
57+
build-native:
58+
name: Build Native Library
59+
runs-on: ubuntu-24.04
60+
container:
61+
image: amd64/rust
62+
steps:
63+
- uses: actions/checkout@v6
64+
65+
- name: Setup Rust toolchain
66+
uses: ./.github/actions/setup-builder
67+
with:
68+
rust-version: ${{ env.RUST_VERSION }}
69+
jdk-version: 17
70+
71+
- name: Cache Cargo
72+
uses: actions/cache@v4
73+
with:
74+
path: |
75+
~/.cargo/registry
76+
~/.cargo/git
77+
native/target
78+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
79+
restore-keys: |
80+
${{ runner.os }}-cargo-ci-
81+
82+
- name: Build native library (CI profile)
83+
run: |
84+
cd native
85+
cargo build --profile ci
86+
87+
- name: Upload native library
88+
uses: actions/upload-artifact@v4
89+
with:
90+
name: native-lib-linux
91+
path: native/target/ci/libcomet.so
92+
retention-days: 1
93+
5594
spark-sql-auto-scan:
95+
needs: build-native
5696
strategy:
5797
matrix:
5898
os: [ubuntu-24.04]
@@ -81,11 +121,17 @@ jobs:
81121
with:
82122
rust-version: ${{env.RUST_VERSION}}
83123
jdk-version: ${{ matrix.spark-version.java }}
124+
- name: Download native library
125+
uses: actions/download-artifact@v4
126+
with:
127+
name: native-lib-linux
128+
path: native/target/release/
84129
- name: Setup Spark
85130
uses: ./.github/actions/setup-spark-builder
86131
with:
87132
spark-version: ${{ matrix.spark-version.full }}
88133
spark-short-version: ${{ matrix.spark-version.short }}
134+
skip-native-build: true
89135
- name: Run Spark tests
90136
run: |
91137
cd apache-spark
@@ -105,6 +151,7 @@ jobs:
105151
path: "**/fallback.log"
106152

107153
spark-sql-native-native-comet:
154+
needs: build-native
108155
strategy:
109156
matrix:
110157
os: [ ubuntu-24.04 ]
@@ -130,11 +177,17 @@ jobs:
130177
with:
131178
rust-version: ${{env.RUST_VERSION}}
132179
jdk-version: ${{ matrix.java-version }}
180+
- name: Download native library
181+
uses: actions/download-artifact@v4
182+
with:
183+
name: native-lib-linux
184+
path: native/target/release/
133185
- name: Setup Spark
134186
uses: ./.github/actions/setup-spark-builder
135187
with:
136188
spark-version: ${{ matrix.spark-version.full }}
137189
spark-short-version: ${{ matrix.spark-version.short }}
190+
skip-native-build: true
138191
- name: Run Spark tests
139192
run: |
140193
cd apache-spark
@@ -154,6 +207,7 @@ jobs:
154207
path: "**/fallback.log"
155208

156209
spark-sql-native-iceberg-compat:
210+
needs: build-native
157211
strategy:
158212
matrix:
159213
os: [ubuntu-24.04]
@@ -179,11 +233,17 @@ jobs:
179233
with:
180234
rust-version: ${{env.RUST_VERSION}}
181235
jdk-version: ${{ matrix.java-version }}
236+
- name: Download native library
237+
uses: actions/download-artifact@v4
238+
with:
239+
name: native-lib-linux
240+
path: native/target/release/
182241
- name: Setup Spark
183242
uses: ./.github/actions/setup-spark-builder
184243
with:
185244
spark-version: ${{ matrix.spark-version.full }}
186245
spark-short-version: ${{ matrix.spark-version.short }}
246+
skip-native-build: true
187247
- name: Run Spark tests
188248
run: |
189249
cd apache-spark

‎native/Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,11 @@ overflow-checks = false
6262
lto = "thin"
6363
codegen-units = 1
6464
strip = "debuginfo"
65+
66+
# CI profile: faster compilation, same overflow behavior as release
67+
# Use with: cargo build --profile ci
68+
[profile.ci]
69+
inherits = "release"
70+
lto = false # Skip LTO for faster linking
71+
codegen-units = 16 # Parallel codegen (faster compile, slightly larger binary)
72+
# overflow-checks inherited as false from release

0 commit comments

Comments
 (0)