Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 31 additions & 78 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
name: "Python (lint)"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/tag=python-lint', github.run_id)
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 120
steps:
Expand All @@ -59,14 +59,7 @@ jobs:
with:
sccache: s3
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-rust
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install uv
uses: spiraldb/actions/.github/actions/setup-uv@0.18.5
with:
sync: false
prune-cache: false
- uses: ./.github/actions/setup-prebuild
# Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min)
- name: Python Lint - Format
run: uvx ruff format --check .
Expand All @@ -82,7 +75,7 @@ jobs:
name: "Python (test)"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-large/tag=python-test', github.run_id)
&& format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 120
env:
Expand All @@ -94,16 +87,7 @@ jobs:
with:
sccache: s3
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-rust
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install uv
uses: spiraldb/actions/.github/actions/setup-uv@0.18.5
with:
sync: false
prune-cache: false
- name: Install Doxygen
uses: ssciwr/doxygen-install@v1
- uses: ./.github/actions/setup-prebuild

- name: Pytest - Vortex
run: |
Expand Down Expand Up @@ -174,7 +158,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre/tag=rust-docs', github.run_id)
&& format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre-v2/tag=rust-docs', github.run_id)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand All @@ -194,7 +178,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre/tag={2}', github.run_id, matrix.config.runner, matrix.config.name)
&& format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/tag={2}', github.run_id, matrix.config.runner, matrix.config.name)
|| 'ubuntu-latest' }}
env:
# disable lints for build, they will be caught in Rust lint job.
Expand Down Expand Up @@ -229,8 +213,6 @@ jobs:
- name: Install wasm32 target
if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }}
run: rustup target add wasm32-unknown-unknown
- name: Install cargo-hack
uses: taiki-e/install-action@cargo-hack
- uses: ./.github/actions/check-rebuild
with:
command: "${{matrix.config.env.rustflags}} cargo hack build --locked ${{matrix.config.args}} --ignore-private"
Expand All @@ -244,7 +226,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre/tag=rust-min-deps', github.run_id)
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-min-deps', github.run_id)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand All @@ -253,16 +235,14 @@ jobs:
sccache: s3
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-prebuild
- uses: taiki-e/install-action@cargo-hack
- uses: taiki-e/install-action@cargo-minimal-versions
- run: cargo minimal-versions check --direct --workspace --ignore-private

rust-lint:
name: "Rust (lint)"
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre/tag=rust-lint', github.run_id)
&& format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-lint', github.run_id)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand Down Expand Up @@ -302,7 +282,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre/tag=rust-lint-no-default', github.run_id)
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-lint-no-default', github.run_id)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand All @@ -311,8 +291,6 @@ jobs:
sccache: s3
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-prebuild
- name: Install cargo-hack
uses: taiki-e/install-action@cargo-hack
- name: Rust Lint - Clippy No Default Features
shell: bash
run: |
Expand All @@ -323,7 +301,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre/tag=public-api', github.run_id)
&& format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=public-api', github.run_id)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand Down Expand Up @@ -363,7 +341,7 @@ jobs:
- suite: tests
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite)
&& format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite)
|| 'ubuntu-latest' }}
env:
RUSTFLAGS: "-Cinstrument-coverage -A warnings"
Expand All @@ -377,18 +355,10 @@ jobs:
sccache: s3
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-prebuild
- name: Install grcov
uses: taiki-e/install-action@grcov
- name: Install nextest
uses: taiki-e/install-action@v2
with:
tool: nextest
- name: Rust Tests
if: ${{ matrix.suite == 'tests' }}
run: |
cargo nextest run --locked --workspace --all-features --no-fail-fast
- name: Install llvm-tools-preview
run: rustup component add llvm-tools-preview
- name: Generate coverage report
run: |
grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
Expand All @@ -414,7 +384,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre/tag=rust-test-sanitizer', github.run_id)
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-test-sanitizer', github.run_id)
|| 'ubuntu-latest' }}
env:
# Add debug symbols and enable ASAN/LSAN with better output
Expand All @@ -434,20 +404,11 @@ jobs:
with:
sccache: s3
- uses: actions/checkout@v6
- name: Install llvm
uses: aminya/setup-cpp@v1
with:
compiler: llvm
cache-tools: true
- uses: ./.github/actions/setup-prebuild
- name: Install nightly for sanitizer
run: |
rustup toolchain install $NIGHTLY_TOOLCHAIN
rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
- name: Install nextest
uses: taiki-e/install-action@v2
with:
tool: nextest
- name: Rust Tests
env:
RUSTFLAGS: "-A warnings -Zsanitizer=address -Zsanitizer=leak --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none"
Expand Down Expand Up @@ -604,7 +565,7 @@ jobs:
runner: runs-on=${{ github.run_id }}/pool=windows-x64-pre
fallback_runner: windows-latest
- os: linux-arm64
runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre/tag=rust-test-linux-arm64
runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=rust-test-linux-arm64
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& matrix.runner
Expand All @@ -614,7 +575,7 @@ jobs:
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- name: Setup (Windows)
if: matrix.os == 'windows-x64'
run: |
Expand Down Expand Up @@ -653,7 +614,7 @@ jobs:
name: "Java"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/tag=java', github.run_id)
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=java', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 120
steps:
Expand All @@ -662,13 +623,7 @@ jobs:
with:
sccache: s3
- uses: actions/checkout@v6
- uses: actions/setup-java@v5
with:
distribution: "corretto"
java-version: "17"
- uses: ./.github/actions/setup-rust
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: ./.github/actions/setup-prebuild
- run: ./gradlew test --parallel
working-directory: ./java

Expand All @@ -688,7 +643,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre/tag=bench-codspeed-{1}', github.run_id, matrix.shard)
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand Down Expand Up @@ -737,7 +692,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre/tag=cxx-build', github.run_id)
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand Down Expand Up @@ -765,7 +720,7 @@ jobs:
name: "SQL logic tests"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre/tag=sql-logic-test', github.run_id)
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=sql-logic-test', github.run_id)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand All @@ -774,10 +729,6 @@ jobs:
sccache: s3
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-prebuild
- name: Install uv
uses: spiraldb/actions/.github/actions/setup-uv@0.18.5
with:
sync: false
- name: Run sqllogictest tests
run: |
./vortex-sqllogictest/slt/tpch/generate_data.sh
Expand Down Expand Up @@ -805,22 +756,24 @@ jobs:

miri:
name: "Rust tests (miri)"
runs-on: ubuntu-latest
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 120
env:
MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE
RUSTFLAGS: "-A warnings --cfg vortex_nightly"
RUST_BACKTRACE: full
steps:
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-rust
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
toolchain: ${{ env.NIGHTLY_TOOLCHAIN }}
components: "rust-src, rustfmt, clippy, miri"
- uses: taiki-e/install-action@v2
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
tool: nextest
sccache: s3
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-prebuild
- name: Install nightly with miri
run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri
- name: Run Miri
run: cargo +$NIGHTLY_TOOLCHAIN miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi

Expand Down Expand Up @@ -891,7 +844,7 @@ jobs:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre/tag=rust-publish-dry-run', github.run_id)
&& format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=rust-publish-dry-run', github.run_id)
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
Expand Down
Loading