Skip to content

chore(ci): workflow cleanup and fixes #7448

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
0ax1 merged 10 commits into from
Apr 15, 2026
Merged

chore(ci): workflow cleanup and fixes #7448

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e11649f
fix(ci): correct fuzz corpus restore path in run-fuzzer.yml
0ax1 Apr 15, 2026
ac174fc
fix(ci): correct stale PR message to match 14-day threshold
0ax1 Apr 15, 2026
5712be8
chore(ci): remove dead cargo-sweep steps from setup-rust
0ax1 Apr 15, 2026
889f551
fix(ci): use --bin instead of --package in bench-pr.yml
0ax1 Apr 15, 2026
a8632e8
chore(ci): right-size job timeouts
0ax1 Apr 15, 2026
9363328
refactor(ci): deduplicate SQL benchmark matrix across workflows
0ax1 Apr 15, 2026
009a9dc
chore(ci): set CARGO_TERM_COLOR to always in ci.yml
0ax1 Apr 15, 2026
38c0379
chore(ci): hoist R2 env vars to job level in fuzz workflows
0ax1 Apr 15, 2026
770a6f2
fixup
0ax1 Apr 15, 2026
50cef37
fix more
0ax1 Apr 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions .github/actions/setup-rust/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,3 @@ runs:
- name: Install Protoc (for lance-encoding build step)
if: runner.os != 'Windows'
uses: ./.github/actions/setup-protoc

- name: Install Sweep
shell: bash
if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}
Comment thread
0ax1 marked this conversation as resolved.
run: cargo install cargo-sweep

- name: Timestamp Cache
shell: bash
if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}
run: cargo sweep --stamp
79 changes: 0 additions & 79 deletions .github/workflows/bench-pr.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,82 +137,3 @@ jobs:
secrets: inherit
with:
mode: "pr"
benchmark_matrix: |
[
{
"id": "clickbench-nvme",
"subcommand": "clickbench",
"name": "Clickbench on NVME",
"targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb",
"extra_data_formats": "vortex-compact"
},
{
"id": "tpch-nvme",
"subcommand": "tpch",
"name": "TPC-H SF=1 on NVME",
"targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
"scale_factor": "1.0"
},
{
"id": "tpch-s3",
"subcommand": "tpch",
"name": "TPC-H SF=1 on S3",
"local_dir": "vortex-bench/data/tpch/1.0",
"remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
"scale_factor": "1.0"
},
{
"id": "tpch-nvme-10",
"subcommand": "tpch",
"name": "TPC-H SF=10 on NVME",
"targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
"scale_factor": "10.0"
},
{
"id": "tpch-s3-10",
"subcommand": "tpch",
"name": "TPC-H SF=10 on S3",
"local_dir": "vortex-bench/data/tpch/10.0",
"remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
"scale_factor": "10.0"
},
{
"id": "tpcds-nvme",
"subcommand": "tpcds",
"name": "TPC-DS SF=1 on NVME",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
"scale_factor": "1.0"
},
{
"id": "statpopgen",
"subcommand": "statpopgen",
"name": "Statistical and Population Genetics",
"targets": "duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
"scale_factor": "100"
},
{
"id": "fineweb",
"subcommand": "fineweb",
"name": "FineWeb NVMe",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
"scale_factor": "100"
},
{
"id": "fineweb-s3",
"subcommand": "fineweb",
"name": "FineWeb S3",
"local_dir": "vortex-bench/data/fineweb",
"remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
"scale_factor": "100"
},
{
"id": "polarsignals",
"subcommand": "polarsignals",
"name": "PolarSignals Profiling",
"targets": "datafusion:vortex",
"scale_factor": "1"
},
]
82 changes: 1 addition & 81 deletions .github/workflows/bench.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ permissions:
jobs:
commit-metadata:
runs-on: ubuntu-latest
timeout-minutes: 120
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
- name: Setup AWS CLI
Expand Down Expand Up @@ -118,83 +118,3 @@ jobs:
secrets: inherit
with:
mode: "develop"
benchmark_matrix: |
[
{
"id": "clickbench-nvme",
"subcommand": "clickbench",
"name": "Clickbench on NVME",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
"build_lance": true
},
{
"id": "tpch-nvme",
"subcommand": "tpch",
"name": "TPC-H SF=1 on NVME",
"targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
"scale_factor": "1.0",
"build_lance": true
},
{
"id": "tpch-s3",
"subcommand": "tpch",
"name": "TPC-H SF=1 on S3",
"local_dir": "vortex-bench/data/tpch/1.0",
"remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
"scale_factor": "1.0"
},
{
"id": "tpch-nvme-10",
"subcommand": "tpch",
"name": "TPC-H SF=10 on NVME",
"targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
"scale_factor": "10.0",
"build_lance": true
},
{
"id": "tpch-s3-10",
"subcommand": "tpch",
"name": "TPC-H SF=10 on S3",
"local_dir": "vortex-bench/data/tpch/10.0",
"remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
"scale_factor": "10.0"
},
{
"id": "tpcds-nvme",
"subcommand": "tpcds",
"name": "TPC-DS SF=1 on NVME",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
"scale_factor": "1.0"
},
{
"id": "statpopgen",
"subcommand": "statpopgen",
"name": "Statistical and Population Genetics",
"local_dir": "vortex-bench/data/statpopgen",
"targets": "duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
"scale_factor": "100"
},
{
"id": "fineweb",
"subcommand": "fineweb",
"name": "FineWeb NVMe",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact"
},
{
"id": "fineweb-s3",
"subcommand": "fineweb",
"name": "FineWeb S3",
"local_dir": "vortex-bench/data/fineweb",
"remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/",
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact"
},
{
"id": "polarsignals",
"subcommand": "polarsignals",
"name": "PolarSignals Profiling",
"targets": "datafusion:vortex",
"scale_factor": "1"
},
]
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ permissions:
issues: write # audit-check creates issues

env:
CARGO_TERM_COLOR: auto
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
NIGHTLY_TOOLCHAIN: nightly-2026-02-05

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/docs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ jobs:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
runs-on: ubuntu-latest
timeout-minutes: 120
timeout-minutes: 10
Comment thread
0ax1 marked this conversation as resolved.
needs: build
steps:
# Note, since we provide the job with a CloudFlare scoped API token, we run it in a separate job that doesn't
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/fuzz-coverage.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ env:
jobs:
coverage:
name: "Coverage: ${{ matrix.fuzz_target }}"
env:
AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -56,11 +61,6 @@ jobs:

- name: Download corpus from R2
shell: bash
env:
AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
run: |
CORPUS_KEY="${{ matrix.fuzz_target }}_corpus.tar.zst"
CORPUS_DIR="fuzz/corpus/${{ matrix.fuzz_target }}"
Expand Down
15 changes: 5 additions & 10 deletions .github/workflows/minimize_fuzz_corpus_workflow.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ env:
jobs:
minimize:
name: "Minimize ${{ inputs.fuzz_target }}"
env:
AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=arm64-medium/disk=large/tag={1}-minimize', github.run_id, inputs.fuzz_target)
Expand Down Expand Up @@ -64,11 +69,6 @@ jobs:

- name: Restore corpus
shell: bash
env:
AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
run: |
CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst"
CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}"
Expand Down Expand Up @@ -98,11 +98,6 @@ jobs:

- name: Persist corpus
shell: bash
env:
AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
run: |
CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst"
CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}"
Expand Down
16 changes: 5 additions & 11 deletions .github/workflows/run-fuzzer.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ jobs:
name: "Run ${{ inputs.fuzz_name || inputs.fuzz_target }}"
env:
FUZZ_NAME: ${{ inputs.fuzz_name || inputs.fuzz_target }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
timeout-minutes: 230 # almost 4 hours
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
Expand Down Expand Up @@ -95,14 +99,9 @@ jobs:

- name: Restore corpus
shell: bash
env:
AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
run: |
CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst"
CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}-${{ inputs.extra_features }}"
CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}"

# Try to download corpus
if python3 scripts/s3-download.py "s3://vortex-fuzz-corpus/$CORPUS_KEY" "$CORPUS_KEY"; then
Expand Down Expand Up @@ -189,11 +188,6 @@ jobs:

- name: Persist corpus
shell: bash
env:
AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }}
AWS_REGION: "us-east-1"
AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
run: |
CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst"
CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}"
Expand Down
Loading
Loading