Skip to content

Commit c4fe785

Browse files
compheadclaude
comphead
and
claude
committed
ci: switch to Spark-style fork-delegated CI
Comment out all existing CI workflows (originals preserved as inline comments) and introduce four new ones modelled on Apache Spark's CI delegation pattern, where the heavy build runs on the contributor's fork and a bridge workflow syncs check status back to the upstream PR: - build_main.yml ("Build"): triggers on push to any branch so contributor pushes to forks run CI on fork-owned Actions minutes rather than apache/datafusion-comet. - notify_test_workflow.yml ("On pull request update"): runs as pull_request_target, locates the fork's Build run, and creates a "Build" check run on the upstream PR pointing at it. - update_build_status.yml: 15-min cron that PATCHes each PR's Build check with the latest status from the fork's run. - build_and_test.yml: reusable workflow invoked by build_main.yml, currently runs only the sql_core-1 shard (sql/testOnly * -- -l ExtendedSQLTest -l SlowSQLTest). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 48f7b03 commit c4fe785

24 files changed

Lines changed: 2194 additions & 1740 deletions

‎.github/workflows/build_and_test.yml‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Reusable workflow invoked by build_main.yml. Modelled on Apache Spark's
19+
# build_and_test.yml: the heavy work runs wherever the calling push lands,
20+
# i.e. on the contributor's fork. Currently runs only the sql_core-1 shard.
21+
22+
name: Build and test
23+
24+
on:
25+
workflow_call:
26+
inputs:
27+
spark-short:
28+
description: 'Spark minor version, e.g. 3.5'
29+
required: false
30+
type: string
31+
default: '3.5'
32+
spark-full:
33+
description: 'Spark full version, e.g. 3.5.8'
34+
required: false
35+
type: string
36+
default: '3.5.8'
37+
java:
38+
description: 'JDK major version'
39+
required: false
40+
type: number
41+
default: 11
42+
43+
env:
44+
RUST_VERSION: stable
45+
RUST_BACKTRACE: 1
46+
RUSTFLAGS: "-Clink-arg=-fuse-ld=bfd"
47+
48+
jobs:
49+
build-native:
50+
name: Build Native Library
51+
runs-on: ubuntu-24.04
52+
container:
53+
image: amd64/rust
54+
steps:
55+
- uses: actions/checkout@v6
56+
57+
- name: Setup Rust toolchain
58+
uses: ./.github/actions/setup-builder
59+
with:
60+
rust-version: ${{ env.RUST_VERSION }}
61+
jdk-version: 17
62+
63+
- name: Restore Cargo cache
64+
uses: actions/cache/restore@v5
65+
with:
66+
path: |
67+
~/.cargo/registry
68+
~/.cargo/git
69+
native/target
70+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
71+
restore-keys: |
72+
${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
73+
74+
- name: Build native library (CI profile)
75+
run: |
76+
cd native
77+
cargo build --profile ci
78+
env:
79+
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd"
80+
81+
- name: Upload native library
82+
uses: actions/upload-artifact@v7
83+
with:
84+
name: native-lib-linux
85+
path: native/target/ci/libcomet.so
86+
retention-days: 1
87+
88+
- name: Save Cargo cache
89+
uses: actions/cache/save@v5
90+
if: github.ref == 'refs/heads/main'
91+
with:
92+
path: |
93+
~/.cargo/registry
94+
~/.cargo/git
95+
native/target
96+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
97+
98+
sql_core-1:
99+
needs: build-native
100+
name: "Build modules: sql_core-1"
101+
runs-on: ubuntu-24.04
102+
container:
103+
image: amd64/rust
104+
steps:
105+
- uses: actions/checkout@v6
106+
- name: Setup Rust & Java toolchain
107+
uses: ./.github/actions/setup-builder
108+
with:
109+
rust-version: ${{env.RUST_VERSION}}
110+
jdk-version: ${{ inputs.java }}
111+
- name: Download native library
112+
uses: actions/download-artifact@v8
113+
with:
114+
name: native-lib-linux
115+
path: native/target/release/
116+
- name: Setup Spark
117+
uses: ./.github/actions/setup-spark-builder
118+
with:
119+
spark-version: ${{ inputs.spark-full }}
120+
spark-short-version: ${{ inputs.spark-short }}
121+
skip-native-build: true
122+
- name: Run Spark sql_core-1 tests
123+
run: |
124+
cd apache-spark
125+
rm -rf /root/.m2/repository/org/apache/parquet
126+
export SERIAL_SBT_TESTS=1
127+
NOLINT_ON_COMPILE=true ENABLE_COMET=true ENABLE_COMET_ONHEAP=true \
128+
build/sbt -Dsbt.log.noformat=true -mem $SBT_MEM \
129+
'set Global / concurrentRestrictions := Seq(Tags.limit(Tags.ForkedTestGroup, 1))' \
130+
"sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest"
131+
env:
132+
LC_ALL: "C.UTF-8"
133+
SBT_MEM: "3072"

‎.github/workflows/build_main.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Modelled on Apache Spark's build_main.yml. Triggers on push to any branch,
19+
# which means contributor pushes to a fork run CI on the fork's Actions
20+
# minutes/runners rather than on apache/datafusion-comet. The bridge to the
21+
# upstream PR check is notify_test_workflow.yml.
22+
23+
name: "Build"
24+
25+
on:
26+
push:
27+
branches:
28+
- '**'
29+
30+
jobs:
31+
call-build-and-test:
32+
permissions:
33+
packages: write
34+
name: Run
35+
uses: ./.github/workflows/build_and_test.yml

‎.github/workflows/codeql.yml‎

Lines changed: 42 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -16,44 +16,45 @@
1616
# under the License.
1717
#
1818

19-
name: "CodeQL"
20-
21-
concurrency:
22-
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23-
cancel-in-progress: true
24-
25-
on:
26-
push:
27-
branches: [ "main" ]
28-
pull_request:
29-
branches: [ "main" ]
30-
schedule:
31-
- cron: '16 4 * * 1'
32-
33-
permissions:
34-
contents: read
35-
36-
jobs:
37-
analyze:
38-
name: Analyze Actions
39-
runs-on: ubuntu-slim
40-
permissions:
41-
contents: read
42-
security-events: write
43-
packages: read
44-
45-
steps:
46-
- name: Checkout repository
47-
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
48-
with:
49-
persist-credentials: false
50-
51-
- name: Initialize CodeQL
52-
uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
53-
with:
54-
languages: actions
55-
56-
- name: Perform CodeQL Analysis
57-
uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
58-
with:
59-
category: "/language:actions"
19+
# Disabled: replaced by build_main.yml / build_and_test.yml (Spark-style fork-delegated CI). Preserved as comments pending migration.
20+
# name: "CodeQL"
21+
22+
# concurrency:
23+
# group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
24+
# cancel-in-progress: true
25+
26+
# on:
27+
# push:
28+
# branches: [ "main" ]
29+
# pull_request:
30+
# branches: [ "main" ]
31+
# schedule:
32+
# - cron: '16 4 * * 1'
33+
34+
# permissions:
35+
# contents: read
36+
37+
# jobs:
38+
# analyze:
39+
# name: Analyze Actions
40+
# runs-on: ubuntu-slim
41+
# permissions:
42+
# contents: read
43+
# security-events: write
44+
# packages: read
45+
46+
# steps:
47+
# - name: Checkout repository
48+
# uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
49+
# with:
50+
# persist-credentials: false
51+
52+
# - name: Initialize CodeQL
53+
# uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
54+
# with:
55+
# languages: actions
56+
57+
# - name: Perform CodeQL Analysis
58+
# uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
59+
# with:
60+
# category: "/language:actions"

‎.github/workflows/docs.yaml‎

Lines changed: 65 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -15,75 +15,76 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
on:
19-
push:
20-
branches:
21-
- main
22-
paths:
23-
- .asf.yaml
24-
- .github/workflows/docs.yaml
25-
- docs/**
18+
# Disabled: replaced by build_main.yml / build_and_test.yml (Spark-style fork-delegated CI). Preserved as comments pending migration.
19+
# on:
20+
# push:
21+
# branches:
22+
# - main
23+
# paths:
24+
# - .asf.yaml
25+
# - .github/workflows/docs.yaml
26+
# - docs/**
2627

27-
name: Deploy DataFusion Comet site
28+
# name: Deploy DataFusion Comet site
2829

29-
jobs:
30-
build-docs:
31-
name: Build docs
32-
if: ${{ startsWith(github.repository, 'apache/') }}
33-
runs-on: ubuntu-latest
34-
steps:
35-
- name: Checkout docs sources
36-
uses: actions/checkout@v6
30+
# jobs:
31+
# build-docs:
32+
# name: Build docs
33+
# if: ${{ startsWith(github.repository, 'apache/') }}
34+
# runs-on: ubuntu-latest
35+
# steps:
36+
# - name: Checkout docs sources
37+
# uses: actions/checkout@v6
3738

38-
- name: Checkout asf-site branch
39-
uses: actions/checkout@v6
40-
with:
41-
ref: asf-site
42-
path: asf-site
39+
# - name: Checkout asf-site branch
40+
# uses: actions/checkout@v6
41+
# with:
42+
# ref: asf-site
43+
# path: asf-site
4344

44-
- name: Setup Python
45-
uses: actions/setup-python@v6
46-
with:
47-
python-version: "3.10"
45+
# - name: Setup Python
46+
# uses: actions/setup-python@v6
47+
# with:
48+
# python-version: "3.10"
4849

49-
- name: Setup Java
50-
uses: actions/setup-java@v5
51-
with:
52-
distribution: 'temurin'
53-
java-version: '17'
54-
cache: 'maven'
50+
# - name: Setup Java
51+
# uses: actions/setup-java@v5
52+
# with:
53+
# distribution: 'temurin'
54+
# java-version: '17'
55+
# cache: 'maven'
5556

56-
- name: Install dependencies
57-
run: |
58-
set -x
59-
python3 -m venv venv
60-
source venv/bin/activate
61-
pip install -r docs/requirements.txt
57+
# - name: Install dependencies
58+
# run: |
59+
# set -x
60+
# python3 -m venv venv
61+
# source venv/bin/activate
62+
# pip install -r docs/requirements.txt
6263

63-
- name: Build docs
64-
run: |
65-
set -x
66-
source venv/bin/activate
67-
cd docs
68-
./build.sh
64+
# - name: Build docs
65+
# run: |
66+
# set -x
67+
# source venv/bin/activate
68+
# cd docs
69+
# ./build.sh
6970

70-
- name: Copy & push the generated HTML
71-
run: |
72-
set -x
73-
cd asf-site/
74-
rsync \
75-
-a \
76-
--delete \
77-
--exclude '/.git/' \
78-
../docs/build/html/ \
79-
./
80-
cp ../.asf.yaml .
81-
touch .nojekyll
82-
git status --porcelain
83-
if [ "$(git status --porcelain)" != "" ]; then
84-
git config user.name "github-actions[bot]"
85-
git config user.email "github-actions[bot]@users.noreply.github.com"
86-
git add --all
87-
git commit -m 'Publish built docs triggered by ${{ github.sha }}'
88-
git push || git push --force
89-
fi
71+
# - name: Copy & push the generated HTML
72+
# run: |
73+
# set -x
74+
# cd asf-site/
75+
# rsync \
76+
# -a \
77+
# --delete \
78+
# --exclude '/.git/' \
79+
# ../docs/build/html/ \
80+
# ./
81+
# cp ../.asf.yaml .
82+
# touch .nojekyll
83+
# git status --porcelain
84+
# if [ "$(git status --porcelain)" != "" ]; then
85+
# git config user.name "github-actions[bot]"
86+
# git config user.email "github-actions[bot]@users.noreply.github.com"
87+
# git add --all
88+
# git commit -m 'Publish built docs triggered by ${{ github.sha }}'
89+
# git push || git push --force
90+
# fi

0 commit comments

Comments
 (0)