ci: test prover on CUDA

Author: JuArce

.github/workflows/gpu-tests.yml

@@ -1,11 +1,12 @@
 name: GPU Tests (merge queue)
 
-# Run the CUDA-only test groups (which CPU CI can't, since GitHub runners have no GPU) on a
-# rented Vast.ai RTX 5090 when a PR is in the merge queue, and block the merge if they fail.
+# Run the GPU test suite (which CPU CI can't, since GitHub runners have no GPU) on a rented
+# Vast.ai RTX 5090 when a PR is in the merge queue, and block the merge if it fails.
 # Groups (see scripts/gpu_test.sh): math-cuda kernel parity, cuda_path_integration (GPU proof
-# verifies), cuda_fallback (CPU fallback verifies). Orchestration runs on a GitHub-hosted
-# runner; all GPU work happens on the rented box (provisioned by the template onstart). The
-# box is ALWAYS destroyed at the end.
+# verifies), cuda_fallback (CPU fallback verifies), the prover/stark/crypto/ecsm suite on the
+# GPU path, and the comprehensive all-instructions prove. Orchestration runs on a GitHub-hosted
+# runner; all GPU work happens on the rented box (provisioned by the template onstart). The box
+# is ALWAYS destroyed at the end.
 #
 # Triggered on `merge_group` (one rental per merge, not per push) + `workflow_dispatch` for
 # manual runs. To gate merges, add the job name `gpu-tests` to the branch-protection required
@@ -31,8 +32,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # Vast offer search: RTX 5090, >=16 cores, >=32GB RAM (workloads are small), >=64GB disk,
-  # verified + rentable, Blackwell-capable driver, <= cap.
+  # Vast offer search: RTX 5090, >=16 cores, >=64GB RAM (the prover suite proves real ELFs,
+  # so allow headroom), >=64GB disk, verified + rentable, Blackwell-capable driver, <= cap.
   GPU_NAME: RTX_5090
   PRICE_CAP: "1"
   VAST_IMAGE_DISK: "64"
@@ -49,9 +50,9 @@ env:
 jobs:
   gpu-tests:
     runs-on: ubuntu-latest
-    # Provisioning + dual-feature cuda build (~25 min) + the three test groups (~15 min) +
-    # image-pull slack. Generous ceiling; teardown still always destroys the box.
-    timeout-minutes: 120
+    # Provisioning + cuda builds + 5 test groups; the prover suite (single-threaded, real
+    # ELF proves) dominates. Generous ceiling; teardown still always destroys the box.
+    timeout-minutes: 240
     steps:
       - name: Install Vast CLI
         # No secrets in this step's env: install-time code can't read the API key during pip
@@ -89,7 +90,7 @@ jobs:
           # 13.1 because the template's nvcc is 13.1 and build.rs JIT-compiles its PTX at load —
           # a 13.0 driver rejects 13.1 PTX (CUDA_ERROR_UNSUPPORTED_PTX_VERSION). Bump this in
           # lockstep if the base image's CUDA toolkit changes.
-          QUERY="gpu_name=${GPU_NAME} num_gpus=1 cpu_cores_effective>=16 cpu_ram>=32 disk_space>=64 verified=true rentable=true cuda_max_good>=13.1 dph_total<=${PRICE_CAP}"
+          QUERY="gpu_name=${GPU_NAME} num_gpus=1 cpu_cores_effective>=16 cpu_ram>=64 disk_space>=64 verified=true rentable=true cuda_max_good>=13.1 dph_total<=${PRICE_CAP}"
           echo "Query: $QUERY (+ client-side driver_version major >= $MIN_DRIVER)"
           # Keep only offers whose driver major >= MIN_DRIVER, then most expensive first (within
           # the cap) — premium hosts have faster disks/network and better reliability; cheapest

Makefile

@@ -2,6 +2,7 @@
 compile-programs clean-asm clean-rust clean-bench clean-shared clean test test-asm \
 test-rust test-executor test-flamegraph flamegraph-prover \
 test-fast test-prover test-prover-all test-disk-spill test-math-cuda test-cuda-integration test-cuda-fallback \
+test-prover-cuda test-prover-comprehensive-cuda \
 bench-math-cuda bench-prover bench-prover-cuda build check clippy fmt lint regen-ethrex-fixtures \
 update-ethrex-fixture-checksums check-ethrex-fixture-checksums
 
@@ -254,6 +255,20 @@ test-cuda-fallback:
 	cargo test -p lambda-vm-prover --release --features test-cuda-faults \
 	    --test cuda_fallback_tests -- --ignored --nocapture
 
+# The prover/stark/crypto/ecsm test suite with the GPU (cuda) path enabled (requires NVIDIA
+# GPU + nvcc). The GPU CI counterpart of CPU CI's sharded prover tests. Single-threaded: the
+# GPU serializes proves and the dispatch counters are process-global. cuda on prover cascades
+# to stark; crypto/ecsm build without it (they have no GPU path).
+test-prover-cuda:
+	cargo test --release -p lambda-vm-prover -p stark -p crypto -p ecsm \
+	    --features lambda-vm-prover/cuda -- --test-threads=1
+
+# The comprehensive all-instructions prove (ignored by default) on the GPU path (requires
+# NVIDIA GPU + nvcc). GPU counterpart of CPU CI's merge-queue-only comprehensive job.
+test-prover-comprehensive-cuda:
+	cargo test --release -p lambda-vm-prover --features cuda \
+	    test_prove_elfs_all_instructions_64_full -- --ignored --test-threads=1 --nocapture
+
 # math-cuda quick microbench (median of 10 runs)
 bench-math-cuda:
 	cargo test -p math-cuda --release --test bench_quick -- --ignored --nocapture

README.md

@@ -188,6 +188,8 @@ See [`spec/README.md`](./spec/README.md) for full setup instructions.
 | `make test-math-cuda` | math-cuda GPU kernel parity tests (requires NVIDIA GPU + nvcc; see GPU Tests) |
 | `make test-cuda-integration` | End-to-end GPU dispatch + proof verification (requires NVIDIA GPU + nvcc) |
 | `make test-cuda-fallback` | GPU error-path / CPU-fallback tests (requires NVIDIA GPU + nvcc) |
+| `make test-prover-cuda` | Prover/stark/crypto/ecsm suite on the GPU path (requires NVIDIA GPU + nvcc) |
+| `make test-prover-comprehensive-cuda` | Comprehensive all-instructions prove on the GPU path (requires NVIDIA GPU + nvcc) |
 | `make build` | Build all workspace crates |
 | `make check` | Check all crates (faster than build, no codegen) |
 | `make clippy` | Run clippy on all crates |
@@ -228,6 +230,8 @@ The CUDA test groups run only on a machine with an NVIDIA GPU and `nvcc`:
 - `make test-math-cuda` — GPU-vs-CPU kernel parity (NTT, LDE, barycentric, FRI, …)
 - `make test-cuda-integration` — proves a guest on GPU and checks every dispatch fired + the proof verifies
 - `make test-cuda-fallback` — forces GPU dispatch errors and checks the CPU fallback still verifies
+- `make test-prover-cuda` — the prover/stark/crypto/ecsm suite with the GPU path enabled
+- `make test-prover-comprehensive-cuda` — the comprehensive all-instructions prove on the GPU path
 
 **Requirement: an NVIDIA driver supporting CUDA ≥ 13.1.** The kernels are compiled with the
 toolkit's `nvcc` (currently CUDA 13.1) into PTX that the driver JIT-compiles at load; a driver

scripts/gpu_test.sh

@@ -2,10 +2,12 @@
 #
 # gpu_test.sh — run the CUDA-only test groups on a GPU box.
 #
-# These groups can't run in CPU CI (GitHub runners have no GPU):
-#   1. math-cuda kernel parity        (make test-math-cuda)
-#   2. end-to-end GPU dispatch + proof (make test-cuda-integration)
-#   3. GPU error-path / CPU fallback   (make test-cuda-fallback)
+# Exercises the CUDA path, which CPU CI can't (GitHub runners have no GPU):
+#   1. math-cuda kernel parity         (make test-math-cuda)
+#   2. end-to-end GPU dispatch + proof  (make test-cuda-integration)
+#   3. GPU error-path / CPU fallback    (make test-cuda-fallback)
+#   4. prover/stark/crypto/ecsm suite   (make test-prover-cuda) — CPU CI's prover tests on GPU
+#   5. comprehensive all-instructions   (make test-prover-comprehensive-cuda)
 #
 # Runs on the rented Vast box from the gpu-tests.yml merge-queue workflow. All three groups
 # run even if one fails (so the log shows every failure); the script exits non-zero if ANY
@@ -42,12 +44,14 @@ log "pinning cudarc to $CUDARC_PIN"
 sed -i "s/\"cuda-version-from-build-system\"/\"${CUDARC_PIN}\"/; /\"fallback-latest\"/d" \
     crypto/math-cuda/Cargo.toml
 
-# --- Build the asm guest ELFs used by Groups 2 & 3 (clang on .s; fast) ----------
-# (math-cuda parity tests need no ELF; cuda_path_integration / cuda_fallback prove an asm ELF.)
-log "compiling asm guest programs"
+# --- Build the guest ELFs the tests prove ---------------------------------------
+# math-cuda parity needs none; cuda_path_integration / cuda_fallback prove an asm ELF; the
+# prover suite (Groups 4 & 5) proves asm AND rust guests. Build both up front.
+log "compiling guest programs (asm + rust)"
 make compile-programs-asm
+make compile-programs-rust
 
-# --- Run the three CUDA test groups via the Makefile targets --------------------
+# --- Run the CUDA test groups via the Makefile targets --------------------------
 fail=0
 run() {  # $1 = make target
     log "make $1"
@@ -56,9 +60,11 @@ run() {  # $1 = make target
         fail=1
     fi
 }
-run test-math-cuda         # Group 1: kernel parity
-run test-cuda-integration  # Group 2: end-to-end GPU dispatch + proof verifies
-run test-cuda-fallback     # Group 3: GPU error -> CPU fallback still verifies
+run test-math-cuda                  # Group 1: kernel parity
+run test-cuda-integration           # Group 2: end-to-end GPU dispatch + proof verifies
+run test-cuda-fallback              # Group 3: GPU error -> CPU fallback still verifies
+run test-prover-cuda                # Group 4: prover/stark/crypto/ecsm suite on the GPU path
+run test-prover-comprehensive-cuda  # Group 5: comprehensive all-instructions prove on GPU
 
 if [ "$fail" -ne 0 ]; then
     log "FAILED — one or more GPU test groups failed"