Skip to content

Commit cd2fbee

Browse files
joseph-isaacsjoseph-isaacs
committed
chore[ci]: break up files
Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
1 parent 82a5c5a commit cd2fbee

4 files changed

Lines changed: 351 additions & 282 deletions

File tree

‎.github/workflows/ci-cuda.yml‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
name: CI / CUDA
2+
3+
# Concurrency control:
4+
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
5+
# - Push to develop: runs queue sequentially, never cancelled.
6+
# - `workflow_dispatch`: groups by branch and queues if run on develop.
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
10+
on:
11+
push:
12+
branches: [develop]
13+
pull_request: { }
14+
workflow_dispatch: { }
15+
16+
permissions:
17+
actions: read
18+
contents: read
19+
20+
env:
21+
CARGO_TERM_COLOR: auto
22+
RUST_BACKTRACE: 1
23+
24+
jobs:
25+
cuda-build-lint:
26+
if: github.repository == 'vortex-data/vortex'
27+
name: "CUDA build & lint"
28+
timeout-minutes: 40
29+
runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build
30+
steps:
31+
- uses: runs-on/action@v2
32+
with:
33+
sccache: s3
34+
- uses: actions/checkout@v6
35+
- uses: ./.github/actions/setup-rust
36+
with:
37+
repo-token: ${{ secrets.GITHUB_TOKEN }}
38+
- uses: ./.github/actions/check-rebuild
39+
with:
40+
command: >-
41+
cargo build --locked --all-features --all-targets
42+
-p vortex-cuda -p vortex-cub -p vortex-nvcomp
43+
-p gpu-scan-cli -p vortex-test-e2e-cuda
44+
- name: Clippy CUDA crates
45+
run: |
46+
cargo clippy --locked --all-features --all-targets \
47+
-p vortex-cuda \
48+
-p vortex-cub \
49+
-p vortex-nvcomp \
50+
-p gpu-scan-cli \
51+
-p vortex-test-e2e-cuda \
52+
-- -D warnings
53+
54+
cuda-test:
55+
if: github.repository == 'vortex-data/vortex'
56+
name: "CUDA tests"
57+
timeout-minutes: 30
58+
runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests
59+
steps:
60+
- uses: runs-on/action@v2
61+
with:
62+
sccache: s3
63+
- name: Display NVIDIA SMI details
64+
run: |
65+
nvidia-smi
66+
nvidia-smi -L
67+
nvidia-smi -q -d Memory
68+
- uses: actions/checkout@v6
69+
- uses: ./.github/actions/setup-rust
70+
with:
71+
repo-token: ${{ secrets.GITHUB_TOKEN }}
72+
- name: Install nextest
73+
uses: taiki-e/install-action@v2
74+
with:
75+
tool: nextest
76+
- name: Rust Tests
77+
env:
78+
FLAT_LAYOUT_INLINE_ARRAY_NODE: true
79+
run: |
80+
cargo nextest run \
81+
--locked \
82+
-p vortex-file \
83+
-p vortex-cuda \
84+
-p vortex-cub \
85+
-p vortex-nvcomp \
86+
-p vortex-test-e2e-cuda \
87+
--all-features \
88+
--no-fail-fast \
89+
--target x86_64-unknown-linux-gnu \
90+
--verbose
91+
92+
cuda-test-sanitizer:
93+
if: github.repository == 'vortex-data/vortex'
94+
name: "CUDA tests (${{ matrix.sanitizer }})"
95+
timeout-minutes: 30
96+
runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer
97+
strategy:
98+
fail-fast: false
99+
matrix:
100+
include:
101+
- sanitizer: memcheck
102+
runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1"
103+
# TODO(joe): try to re-enable racecheck, it is hanging in CI.
104+
# - sanitizer: racecheck
105+
# runner_flags: "--tool racecheck --error-exitcode 1"
106+
- sanitizer: synccheck
107+
runner_flags: "--tool synccheck --error-exitcode 1"
108+
- sanitizer: initcheck
109+
runner_flags: "--tool initcheck --error-exitcode 1"
110+
steps:
111+
- uses: runs-on/action@v2
112+
with:
113+
sccache: s3
114+
- name: Display NVIDIA SMI details
115+
run: |
116+
nvidia-smi
117+
nvidia-smi -L
118+
nvidia-smi -q -d Memory
119+
- uses: actions/checkout@v6
120+
- uses: ./.github/actions/setup-rust
121+
with:
122+
repo-token: ${{ secrets.GITHUB_TOKEN }}
123+
- name: Build tests
124+
run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run
125+
- name: "CUDA - ${{ matrix.sanitizer }}"
126+
env:
127+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}"
128+
run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu
129+
130+
cuda-test-cudf:
131+
if: github.repository == 'vortex-data/vortex'
132+
name: "CUDA tests (cudf)"
133+
timeout-minutes: 30
134+
runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf
135+
steps:
136+
- uses: runs-on/action@v2
137+
with:
138+
sccache: s3
139+
- name: Display NVIDIA SMI details
140+
run: |
141+
nvidia-smi
142+
nvidia-smi -L
143+
nvidia-smi -q -d Memory
144+
- uses: actions/checkout@v6
145+
- uses: ./.github/actions/setup-rust
146+
with:
147+
repo-token: ${{ secrets.GITHUB_TOKEN }}
148+
- name: Build cudf test library
149+
run: cargo build --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu
150+
- name: Download and run cudf-test-harness
151+
run: |
152+
curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz
153+
cd cudf-test-harness-x86_64
154+
compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/debug/libvortex_test_e2e_cuda.so

‎.github/workflows/ci-java.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: CI / Java
2+
3+
# Concurrency control:
4+
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
5+
# - Push to develop: runs queue sequentially, never cancelled.
6+
# - `workflow_dispatch`: groups by branch and queues if run on develop.
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
10+
on:
11+
push:
12+
branches: [develop]
13+
pull_request: { }
14+
workflow_dispatch: { }
15+
16+
permissions:
17+
actions: read
18+
contents: read
19+
20+
env:
21+
CARGO_TERM_COLOR: auto
22+
RUST_BACKTRACE: 1
23+
24+
jobs:
25+
build-java:
26+
name: "Java"
27+
runs-on: >-
28+
${{ github.repository == 'vortex-data/vortex'
29+
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=java', github.run_id)
30+
|| 'ubuntu-latest' }}
31+
timeout-minutes: 40
32+
steps:
33+
- uses: runs-on/action@v2
34+
if: github.repository == 'vortex-data/vortex'
35+
with:
36+
sccache: s3
37+
- uses: actions/checkout@v6
38+
- uses: ./.github/actions/setup-prebuild
39+
- run: ./gradlew test --parallel
40+
working-directory: ./java
41+
42+
check-java-publish-build:
43+
runs-on: ${{ matrix.target.runs-on }}
44+
container:
45+
image: "ubuntu:20.04"
46+
timeout-minutes: 40
47+
strategy:
48+
fail-fast: false
49+
matrix:
50+
target:
51+
- { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu }
52+
- { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu }
53+
steps:
54+
- uses: actions/checkout@v6
55+
with:
56+
fetch-depth: 0
57+
- uses: ./.github/actions/prepare-java-linux
58+
- uses: actions/setup-java@v5
59+
with:
60+
distribution: "corretto"
61+
java-version: "17"
62+
- uses: ./.github/actions/setup-rust
63+
with:
64+
targets: ${{ matrix.target.target }}
65+
repo-token: ${{ secrets.GITHUB_TOKEN }}
66+
enable-sccache: "false"
67+
- run: cargo build --package vortex-jni

‎.github/workflows/ci-python.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: CI / Python
2+
3+
# Concurrency control:
4+
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
5+
# - Push to develop: runs queue sequentially, never cancelled.
6+
# - `workflow_dispatch`: groups by branch and queues if run on develop.
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
10+
on:
11+
push:
12+
branches: [develop]
13+
pull_request: { }
14+
workflow_dispatch: { }
15+
16+
permissions:
17+
actions: read
18+
contents: read
19+
20+
env:
21+
CARGO_TERM_COLOR: auto
22+
RUST_BACKTRACE: 1
23+
24+
jobs:
25+
python-lint:
26+
name: "Python (lint)"
27+
runs-on: >-
28+
${{ github.repository == 'vortex-data/vortex'
29+
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id)
30+
|| 'ubuntu-latest' }}
31+
timeout-minutes: 40
32+
steps:
33+
- uses: runs-on/action@v2
34+
if: github.repository == 'vortex-data/vortex'
35+
with:
36+
sccache: s3
37+
- uses: actions/checkout@v6
38+
- uses: ./.github/actions/setup-prebuild
39+
# Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min)
40+
- name: Python Lint - Format
41+
run: uvx ruff format --check .
42+
- name: Python Lint - Ruff
43+
run: uvx ruff check .
44+
# PyRight needs the project for type information, so use uv run
45+
- name: Python Lint - PyRight
46+
env:
47+
MATURIN_PEP517_ARGS: "--profile dev"
48+
run: uv run basedpyright vortex-python
49+
50+
python-test:
51+
name: "Python (test)"
52+
runs-on: >-
53+
${{ github.repository == 'vortex-data/vortex'
54+
&& format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id)
55+
|| 'ubuntu-latest' }}
56+
timeout-minutes: 40
57+
env:
58+
RUST_LOG: "info,uv=debug"
59+
MATURIN_PEP517_ARGS: "--profile dev"
60+
steps:
61+
- uses: runs-on/action@v2
62+
if: github.repository == 'vortex-data/vortex'
63+
with:
64+
sccache: s3
65+
- uses: actions/checkout@v6
66+
- uses: ./.github/actions/setup-prebuild
67+
68+
- name: Pytest - Vortex
69+
run: |
70+
uv run --all-packages pytest --benchmark-disable -n auto test/
71+
working-directory: vortex-python/
72+
73+
- name: Setup benchmark environment
74+
run: sudo bash scripts/setup-benchmark.sh
75+
76+
- name: Pytest Benchmarks - Vortex
77+
run: |
78+
bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/
79+
working-directory: vortex-python/
80+
81+
- name: Doctest - PyVortex
82+
run: |
83+
uv run --all-packages make doctest
84+
working-directory: docs/
85+
86+
- name: Ensure docs build - PyVortex
87+
run: |
88+
uv run --all-packages make html
89+
working-directory: docs/
90+
91+
python-wheel-build:
92+
name: "Python (wheel build)"
93+
runs-on: ubuntu-latest
94+
timeout-minutes: 40
95+
steps:
96+
- uses: actions/checkout@v6
97+
- name: Rust Dependency Cache
98+
uses: Swatinem/rust-cache@v2
99+
with:
100+
save-if: ${{ github.ref_name == 'develop' }}
101+
- uses: ./.github/actions/setup-rust
102+
with:
103+
repo-token: ${{ secrets.GITHUB_TOKEN }}
104+
enable-sccache: "false"
105+
- uses: mlugg/setup-zig@v2.2.1
106+
- name: Install uv
107+
uses: spiraldb/actions/.github/actions/setup-uv@0.18.5
108+
with:
109+
sync: false
110+
prune-cache: false
111+
- name: Ensure wheel and sdist can be built on Linux - PyVortex
112+
shell: bash
113+
run: |
114+
echo "Clearing wheel target directory"
115+
rm -rf ../target/wheels/
116+
117+
uv venv
118+
uv tool run maturin@1.10 build --interpreter python3.11 --zig
119+
uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist
120+
121+
file_count=$(ls -1 ../target/wheels/ | wc -l)
122+
123+
if [[ $file_count -ne 2 ]]; then
124+
echo "Unexpected number of files detected ${file_count}:"
125+
ls ../target/wheels/
126+
exit 1
127+
else
128+
echo "Generated two files"
129+
fi
130+
working-directory: vortex-python/

0 commit comments

Comments
 (0)