Skip to content

Commit 75bad47

Browse files
compheadcomphead
authored
chore: add optional CI flow for parquet writes (apache#4696)
1 parent b8c3695 commit 75bad47

1 file changed

Lines changed: 154 additions & 0 deletions

File tree

‎.github/workflows/spark_sql_writer_tests.yml‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Manual workflow that exercises Spark's Parquet WRITER test suites with
19+
# Comet's experimental native Parquet writer enabled (ENABLE_COMET_WRITE=true,
20+
# spark.comet.parquet.write.enabled). Self-contained — does not call
21+
# spark_sql_test_reusable.yml — so it can run on feature branches without
22+
# touching the standard CI matrix.
23+
#
24+
# Trigger: workflow_dispatch only. Pick a Spark minor version from the
25+
# selector and click "Run workflow".
26+
#
27+
# Suite list focuses on writer-side coverage: Parquet IO, committer,
28+
# encoding, compression codec, V1/V2 file format, V1/V2 query, V1/V2
29+
# partition discovery, field IDs, schema; plus the generic writer
30+
# infrastructure (FileFormatWriterSuite, PartitionedWriteSuite). Reader-only
31+
# Parquet suites (ParquetV{1,2}FilterSuite, *SchemaPruningSuite, etc.) are
32+
# intentionally excluded — they're already covered by the sql_core-* lanes.
33+
34+
name: Spark SQL Writer Tests (manual)
35+
36+
on:
37+
workflow_dispatch:
38+
inputs:
39+
spark-version:
40+
description: 'Spark minor version to test against'
41+
type: choice
42+
required: true
43+
default: '3.5'
44+
options:
45+
- '3.5'
46+
- '4.1'
47+
48+
env:
49+
RUST_VERSION: stable
50+
RUST_BACKTRACE: 1
51+
# Force GNU ld on Linux: recent Rust stable defaults to rust-lld on
52+
# x86_64-unknown-linux-gnu, and rust-lld cannot resolve -ljvm against the
53+
# Zulu JDK layout installed by setup-java. Keep bfd for all cargo invocations.
54+
RUSTFLAGS: "-Clink-arg=-fuse-ld=bfd"
55+
56+
jobs:
57+
writer-test:
58+
name: spark-sql-writer/spark-${{ inputs.spark-version }}
59+
runs-on: ubuntu-24.04
60+
container:
61+
image: amd64/rust
62+
steps:
63+
- uses: actions/checkout@v7
64+
65+
- name: Resolve Spark full version and JDK
66+
id: resolve
67+
shell: bash
68+
run: |
69+
# Map each supported Spark minor version to its full version + JDK.
70+
# Mirrors ci.yml's per-version reusable invocations (default-on PR
71+
# versions only; 3.4 and 4.0 are label-gated and not offered here).
72+
case "${{ inputs.spark-version }}" in
73+
3.5) spark_full=3.5.8; java=17 ;;
74+
4.1) spark_full=4.1.2; java=17 ;;
75+
*) echo "Unsupported spark-version: ${{ inputs.spark-version }}" >&2; exit 1 ;;
76+
esac
77+
echo "spark-full=$spark_full" >> "$GITHUB_OUTPUT"
78+
echo "java=$java" >> "$GITHUB_OUTPUT"
79+
80+
- name: Setup Rust & Java toolchain
81+
uses: ./.github/actions/setup-builder
82+
with:
83+
rust-version: ${{ env.RUST_VERSION }}
84+
jdk-version: ${{ steps.resolve.outputs.java }}
85+
86+
- name: Restore Cargo cache
87+
uses: actions/cache/restore@v5
88+
with:
89+
path: |
90+
~/.cargo/registry
91+
~/.cargo/git
92+
native/target
93+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
94+
restore-keys: |
95+
${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
96+
97+
- name: Build native library (CI profile)
98+
run: |
99+
cd native
100+
cargo build --profile ci
101+
env:
102+
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd"
103+
104+
- name: Stage native library at release path
105+
run: |
106+
# setup-spark-builder's `mvnw install -DskipTests` (skip-native-build
107+
# path) bundles native/target/release/libcomet.so into the Comet JAR.
108+
# We built with --profile ci to avoid LTO, so the file lives at
109+
# native/target/ci/. Copy it to where the Maven build expects it.
110+
mkdir -p native/target/release
111+
cp native/target/ci/libcomet.so native/target/release/libcomet.so
112+
113+
- name: Setup Spark
114+
uses: ./.github/actions/setup-spark-builder
115+
with:
116+
spark-version: ${{ steps.resolve.outputs.spark-full }}
117+
spark-short-version: ${{ inputs.spark-version }}
118+
skip-native-build: true
119+
120+
- name: Run Parquet writer tests
121+
run: |
122+
cd apache-spark
123+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
124+
# SERIAL_SBT_TESTS gates SparkParallelTestGrouping in
125+
# project/SparkBuild.scala. We always set it to reduce peak memory
126+
# on standard 7 GB runners (3.5 and 4.1 are unaffected by the
127+
# 4.0+JDK 21 file-stream-leak case the reusable workflow handles).
128+
export SERIAL_SBT_TESTS=1
129+
# Same forked-test-JVM caps as sql_core-* in spark_sql_test_reusable.yml.
130+
export HEAP_SIZE=3g
131+
export METASPACE_SIZE=1g
132+
NOLINT_ON_COMPILE=true ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ENABLE_COMET_WRITE=true \
133+
build/sbt -Dsbt.log.noformat=true -mem $SBT_MEM \
134+
'set Global / concurrentRestrictions := Seq(Tags.limit(Tags.ForkedTestGroup, 1))' \
135+
"sql/testOnly org.apache.spark.sql.execution.datasources.parquet.ParquetIOSuite org.apache.spark.sql.execution.datasources.parquet.ParquetCommitterSuite org.apache.spark.sql.execution.datasources.parquet.ParquetEncodingSuite org.apache.spark.sql.execution.datasources.parquet.ParquetCompressionCodecPrecedenceSuite org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV1Suite org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV2Suite org.apache.spark.sql.execution.datasources.parquet.ParquetV1QuerySuite org.apache.spark.sql.execution.datasources.parquet.ParquetV2QuerySuite org.apache.spark.sql.execution.datasources.parquet.ParquetV1PartitionDiscoverySuite org.apache.spark.sql.execution.datasources.parquet.ParquetV2PartitionDiscoverySuite org.apache.spark.sql.execution.datasources.parquet.ParquetFieldIdIOSuite org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaSuite org.apache.spark.sql.execution.datasources.FileFormatWriterSuite org.apache.spark.sql.sources.PartitionedWriteSuite"
136+
env:
137+
LC_ALL: "C.UTF-8"
138+
# Cap SBT orchestrator heap so the freed RAM goes to the forked test
139+
# JVM and OS/container overhead, fixing cgroup-OOM SIGKILLs under
140+
# 7 GB runners.
141+
SBT_MEM: "1024"
142+
# G1GC + tuning for the SBT orchestrator JVM. -Xss4m replaces the
143+
# launcher's -Xss64m default (no compile here, deep recursion not
144+
# needed). UseStringDeduplication and MaxMetaspaceSize cap real and
145+
# ceiling footprint. ExitOnOutOfMemoryError fails fast.
146+
SBT_OPTS: >-
147+
-Xss4m
148+
-XX:+UseG1GC
149+
-XX:+UseStringDeduplication
150+
-XX:MaxMetaspaceSize=384m
151+
-XX:G1HeapRegionSize=2m
152+
-XX:InitiatingHeapOccupancyPercent=35
153+
-XX:+ParallelRefProcEnabled
154+
-XX:+ExitOnOutOfMemoryError

0 commit comments

Comments
 (0)