From da4c3c5337e2c3bf4294e4c399dbafd6a0bab982 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Tue, 30 Jun 2026 15:32:57 -0300 Subject: [PATCH 1/6] test(recursion): memory-bounded execute/prove smoke tests; wire CI Stream the execute-only path via Executor::resume() and read the committed marker straight off Memory (Executor::finish), dropping the run()+Traces build that OOM'd. Prove the outer recursion guest memory-bounded via prove_and_verify_continuation (2^20-cycle epochs) instead of a monolithic ~125 GB prove. Un-ignore the three execute-only tests (now sub-minute); the full-prove tests stay #[ignore]d (>60s) and run in the comprehensive CI job. CI: build the recursion guest ELFs (cache + conditional setup-rust + make compile-recursion-elfs) in both the per-PR and comprehensive prover jobs, and add test(test_recursion_prove) to the comprehensive filter. --- .github/workflows/pr_main.yaml | 48 +++++++- prover/src/tests/recursion_smoke_test.rs | 138 ++++++++++++++--------- 2 files changed, 131 insertions(+), 55 deletions(-) diff --git a/.github/workflows/pr_main.yaml b/.github/workflows/pr_main.yaml index ae675d770..e527f79a2 100644 --- a/.github/workflows/pr_main.yaml +++ b/.github/workflows/pr_main.yaml @@ -308,6 +308,29 @@ jobs: with: name: prover-tests + # The un-ignored recursion execute-only smoke tests read prebuilt guest + # ELFs from executor/program_artifacts/recursion/ at runtime, so build + # them before the test run (cached on the guest sources + toolchain spec). + - name: Cache compiled recursion guest ELF artifacts + id: cache-recursion-elfs + uses: actions/cache@v4 + with: + path: executor/program_artifacts/recursion + key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} + restore-keys: | + recursion-elf-artifacts- + + # Only needed when the recursion cache missed but the rust-elf cache hit + # (so the rust-elf compile step's Setup Rust did not run). + - name: Setup Rust Environment (recursion ELFs) + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true' + uses: ./.github/actions/setup-rust + + - name: Compile recursion guest ELFs + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' + run: | + make compile-recursion-elfs + - name: Run prover tests (shard ${{ matrix.partition }}/4) run: | cargo nextest run \ @@ -370,12 +393,35 @@ jobs: with: name: prover-tests + # The #[ignore]d recursion full-prove smoke tests (run only here, via + # --run-ignored) read prebuilt guest ELFs from + # executor/program_artifacts/recursion/ at runtime, so build them first. + - name: Cache compiled recursion guest ELF artifacts + id: cache-recursion-elfs + uses: actions/cache@v4 + with: + path: executor/program_artifacts/recursion + key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} + restore-keys: | + recursion-elf-artifacts- + + # Only needed when the recursion cache missed but the rust-elf cache hit + # (so the rust-elf compile step's Setup Rust did not run). + - name: Setup Rust Environment (recursion ELFs) + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true' + uses: ./.github/actions/setup-rust + + - name: Compile recursion guest ELFs + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' + run: | + make compile-recursion-elfs + - name: Run comprehensive prover tests run: | cargo nextest run \ --archive-file prover-tests.tar.zst \ --test-threads=1 \ - -E 'test(test_prove_elfs_all_instructions_64_full)' \ + -E 'test(test_prove_elfs_all_instructions_64_full) | test(test_recursion_prove)' \ --run-ignored ignored-only # Seed ELF caches on refs/heads/main so merge-queue runs can restore them. diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index a4f9fb7b0..3bc53d859 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -4,17 +4,20 @@ //! 1. Proves an inner program on the host. //! 2. Serializes `(VmProof, inner_elf, opts)` with postcard. //! 3. Hands that as private input to the recursion guest. -//! 4. Either **proves** the recursion guest's execution and verifies the outer -//! proof (`OuterMode::Prove`), or merely **executes** the guest in-VM and -//! reads the committed marker off the trace (`OuterMode::ExecuteOnly`) — a -//! cheaper tier that skips the LDE/FRI that dominate the full pipeline. +//! 4. Either **proves** the recursion guest's execution (memory-bounded via +//! continuations) and verifies the outer proof (`OuterMode::Prove`), or +//! merely **executes** the guest in-VM and reads the committed marker +//! straight off the executor's memory (`OuterMode::ExecuteOnly`) — a cheaper +//! tier that skips the LDE/FRI that dominate the full pipeline. //! //! The guest ELFs are built by `make compile-recursion-elfs` (which the //! `test-prover-all` make target depends on) and read from //! `executor/program_artifacts/recursion/`, like every other program test. //! -//! Tests are `#[ignore]`d because the outer proof runs the full STARK verifier -//! inside the VM (minutes per run, large memory footprint). +//! Both tiers stream/epoch the work, so neither holds the whole execution at +//! once. The execute-only tests run in well under a minute and are not ignored; +//! the full-prove tests still run the STARK verifier in-VM (minutes per run) and +//! stay `#[ignore]`d, run by the comprehensive CI job and `make test-prover-all`. use std::path::PathBuf; @@ -83,64 +86,91 @@ fn prove_inner_and_encode_blob( /// whether we also prove the guest's own execution. #[derive(Clone, Copy, Debug)] enum OuterMode { - /// Execute the guest in-VM and read the committed marker off the trace. - /// Skips the LDE blowup + FRI commit that dominate the full pipeline's - /// footprint, so it needs materially less RAM than `Prove`. - /// - /// "Less" is not "little": `Executor::run` retains a per-instruction log - /// and `Traces` materializes the full execution trace, so verifying even a - /// 1-query inner proof still needs tens of GB — it OOMs on a 36 GB box. + /// Execute the guest in-VM and read the committed marker straight off the + /// executor's memory. Streams logs via `Executor::resume()` and never + /// builds a `Traces`, so footprint stays bounded to the VM's touched + /// memory + instruction cache — runs end-to-end where the old `run()` + + /// `Traces` path OOM'd. Skips the LDE/FRI of the full pipeline entirely. ExecuteOnly, - /// Prove the guest's execution and verify the outer proof on the host. The - /// full STARK verifier inside the VM — minutes per run, ~125 GB. + /// Prove the guest's execution **memory-bounded** via continuations, then + /// verify the outer proof on the host. The run is split into + /// `2^OUTER_EPOCH_SIZE_LOG2`-cycle epochs, each proved independently, plus + /// the one cross-epoch global-memory linkage. Peak RAM is a single epoch's + /// trace+LDE (the accumulated per-epoch proofs are small), not the + /// monolithic ~125 GB a single-shot prove of the whole verifier would + /// build — so the outer prove runs end-to-end on a normal box. Prove, } /// Execute the recursion guest in-VM on `blob` and return the bytes it /// committed (the success marker the in-VM verifier emits). +/// +/// Streams execution via `Executor::resume()`: each chunk reuses one +/// `CHUNK_SIZE`-bounded log buffer (cleared per chunk), so logs never +/// accumulate, and we never build a `Traces`. The committed marker is read +/// directly off the executor's memory — `Memory::commit_public_output` +/// accumulates the same byte stream the trace builder's `public_output_bytes` +/// would, exposed via `Executor::finish()`. This avoids both OOM sources of +/// the old `run()` + `Traces::from_elf_and_logs` path (the full `Vec` and +/// the materialized execution trace), neither of which the marker needs. fn execute_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec { use executor::elf::Elf; use executor::vm::execution::Executor; - eprintln!("[{label}] executing outer (recursion guest, in-VM verify) ..."); + eprintln!("[{label}] executing outer (recursion guest, in-VM verify, streaming) ..."); let program = Elf::load(recursion_elf_bytes).expect("load recursion elf"); - let result = Executor::new(&program, blob.to_vec()) - .expect("executor new") - .run() - .expect("recursion guest execution failed (verify panicked in-VM?)"); - - let traces = crate::tables::trace_builder::Traces::from_elf_and_logs( - &program, - &result.logs, - &crate::MaxRowsConfig::default(), - blob, - #[cfg(feature = "disk-spill")] - stark::storage_mode::StorageMode::Ram, - ) - .expect("trace build"); + let mut executor = Executor::new(&program, blob.to_vec()).expect("executor new"); + + // Drain chunks to completion without retaining logs or building a trace. + while executor + .resume() + .expect("recursion guest execution failed (verify panicked in-VM?)") + .is_some() + {} + + let committed = executor + .finish() + .expect("read committed output after execution") + .memory_values; eprintln!( "[{label}] committed {} bytes: {:?} (as str: {:?})", - traces.public_output_bytes.len(), - traces.public_output_bytes, - String::from_utf8_lossy(&traces.public_output_bytes), + committed.len(), + committed, + String::from_utf8_lossy(&committed), ); - traces.public_output_bytes + committed } -/// Prove the recursion guest's execution on `blob`, verify the outer proof on -/// the host, and return the bytes the guest committed. +/// Epoch size for the memory-bounded outer prove: 2^20 ≈ 1M cycles per epoch. +/// Each epoch's trace+LDE is what bounds peak RAM, so this trades per-epoch +/// footprint against epoch count — small enough to fit a normal box, large +/// enough to keep epoch count (and thus the cross-epoch overhead) modest. Well +/// under `local_to_global::MAX_EPOCHS` (2^20) for any guest we run here. +const OUTER_EPOCH_SIZE_LOG2: u32 = 20; + +/// Prove the recursion guest's execution on `blob` **memory-bounded** via +/// continuations and verify the bundle on the host, returning the bytes the +/// guest committed. `prove_and_verify_continuation` proves epoch-by-epoch +/// (holding one epoch's trace at a time) and reconstructs the committed output +/// from the per-epoch bound slices, so a `Some(output)` already means every +/// epoch proof, the cross-epoch linkage, and the L2G binding verified. fn prove_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec { - eprintln!("[{label}] proving outer (recursion guest) ..."); - let outer_proof = - crate::prove_with_inputs(recursion_elf_bytes, blob).expect("outer prove should succeed"); - eprintln!("[{label}] outer proof generated"); - - assert!( - crate::verify(&outer_proof, recursion_elf_bytes).expect("outer verify errored"), - "outer proof must verify on host" + let opts = crate::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is always valid"); + eprintln!( + "[{label}] proving outer (recursion guest) via continuations \ + (epoch=2^{OUTER_EPOCH_SIZE_LOG2} cycles) ..." ); - outer_proof.public_output + let committed = crate::continuation::prove_and_verify_continuation( + recursion_elf_bytes, + blob, + OUTER_EPOCH_SIZE_LOG2, + &opts, + ) + .expect("outer continuation prove/verify errored") + .expect("outer continuation proof must verify on host"); + eprintln!("[{label}] outer continuation proof generated and verified"); + committed } /// Core pipeline: prove an inner program with the given options, hand the @@ -250,14 +280,16 @@ fn test_recursion_blob_decodes_and_verifies_on_host() { // === Execute-only tier ======================================================== // Mirrors the proving tests below, but stops at `OuterMode::ExecuteOnly`: the -// guest runs in-VM and we read the committed marker off the trace, skipping the -// outer STARK prove. Needs tens of GB (execution trace), not the ~125 GB the -// full outer prove wants — but still OOMs on a 36 GB box. +// guest runs in-VM and we read the committed marker straight off the executor's +// memory, skipping the outer STARK prove. Streams execution (no trace, no +// buffered logs), so the footprint is bounded to the VM's touched memory. +// +// Not `#[ignore]`d: each runs in well under a minute, so they execute in the +// regular per-PR prover test job (which builds the recursion guest ELFs first). /// Execute-only mirror of `test_recursion_prove_empty`: verify a `blowup=8` /// proof of the empty program in-VM. #[test] -#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"] fn test_recursion_execute_empty() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -272,7 +304,6 @@ fn test_recursion_execute_empty() { /// Execute-only mirror of `test_recursion_prove_1query`: smallest possible /// inner proof (blowup=2, 1 query) → least guest work. #[test] -#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"] fn test_recursion_execute_1query() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -288,7 +319,6 @@ fn test_recursion_execute_1query() { /// Execute-only mirror of `test_recursion_prove`: verify a `blowup=8` proof of /// fibonacci(10) in-VM. #[test] -#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"] fn test_recursion_execute() { let root = workspace_root(); let fib_elf_bytes = read_guest_elf(&root, "fibonacci"); @@ -310,7 +340,7 @@ fn test_recursion_execute() { /// lambda-vm verifier's intrinsic recursion overhead — i.e. what it costs /// to verify the smallest possible lambda-vm proof, with no inner workload. #[test] -#[ignore = "slow: runs the full STARK verifier inside the VM"] +#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"] fn test_recursion_prove_empty() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -327,7 +357,7 @@ fn test_recursion_prove_empty() { /// run end-to-end on a 125 GB box" experiment — security is intentionally /// terrible. Use only for capacity probing. #[test] -#[ignore = "slow: runs the full STARK verifier inside the VM"] +#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"] fn test_recursion_prove_1query() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -343,7 +373,7 @@ fn test_recursion_prove_1query() { /// Inner program: fibonacci(10). #[test] -#[ignore = "slow: runs the full STARK verifier inside the VM"] +#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"] fn test_recursion_prove() { let root = workspace_root(); let fib_elf_bytes = read_guest_elf(&root, "fibonacci"); From 032ac99e5392e5df60924fa584d814aded09107b Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Tue, 30 Jun 2026 15:49:41 -0300 Subject: [PATCH 2/6] cargo fmt --- prover/src/tests/recursion_smoke_test.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index 3bc53d859..a1bc9ae50 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -156,7 +156,8 @@ const OUTER_EPOCH_SIZE_LOG2: u32 = 20; /// from the per-epoch bound slices, so a `Some(output)` already means every /// epoch proof, the cross-epoch linkage, and the L2G binding verified. fn prove_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec { - let opts = crate::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is always valid"); + let opts = + crate::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is always valid"); eprintln!( "[{label}] proving outer (recursion guest) via continuations \ (epoch=2^{OUTER_EPOCH_SIZE_LOG2} cycles) ..." From e6f070f3967509e92c2a637d882e297d9c485c56 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Tue, 30 Jun 2026 16:09:59 -0300 Subject: [PATCH 3/6] a bit of brevity --- prover/src/tests/recursion_smoke_test.rs | 63 +++++------------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index a1bc9ae50..a52858228 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -10,14 +10,7 @@ //! straight off the executor's memory (`OuterMode::ExecuteOnly`) — a cheaper //! tier that skips the LDE/FRI that dominate the full pipeline. //! -//! The guest ELFs are built by `make compile-recursion-elfs` (which the -//! `test-prover-all` make target depends on) and read from -//! `executor/program_artifacts/recursion/`, like every other program test. -//! -//! Both tiers stream/epoch the work, so neither holds the whole execution at -//! once. The execute-only tests run in well under a minute and are not ignored; -//! the full-prove tests still run the STARK verifier in-VM (minutes per run) and -//! stay `#[ignore]`d, run by the comprehensive CI job and `make test-prover-all`. +//! The guest ELFs are assumed built by `make compile-recursion-elfs`. use std::path::PathBuf; @@ -89,30 +82,18 @@ enum OuterMode { /// Execute the guest in-VM and read the committed marker straight off the /// executor's memory. Streams logs via `Executor::resume()` and never /// builds a `Traces`, so footprint stays bounded to the VM's touched - /// memory + instruction cache — runs end-to-end where the old `run()` + - /// `Traces` path OOM'd. Skips the LDE/FRI of the full pipeline entirely. + /// memory + instruction cache. Skips the LDE/FRI of the full pipeline entirely. ExecuteOnly, - /// Prove the guest's execution **memory-bounded** via continuations, then - /// verify the outer proof on the host. The run is split into - /// `2^OUTER_EPOCH_SIZE_LOG2`-cycle epochs, each proved independently, plus - /// the one cross-epoch global-memory linkage. Peak RAM is a single epoch's - /// trace+LDE (the accumulated per-epoch proofs are small), not the - /// monolithic ~125 GB a single-shot prove of the whole verifier would - /// build — so the outer prove runs end-to-end on a normal box. + /// Prove the guest's execution memory-bounded via continuations, then + /// verify the outer proof on the host. Peak RAM is a single epoch's proof. Prove, } /// Execute the recursion guest in-VM on `blob` and return the bytes it /// committed (the success marker the in-VM verifier emits). /// -/// Streams execution via `Executor::resume()`: each chunk reuses one -/// `CHUNK_SIZE`-bounded log buffer (cleared per chunk), so logs never -/// accumulate, and we never build a `Traces`. The committed marker is read -/// directly off the executor's memory — `Memory::commit_public_output` -/// accumulates the same byte stream the trace builder's `public_output_bytes` -/// would, exposed via `Executor::finish()`. This avoids both OOM sources of -/// the old `run()` + `Traces::from_elf_and_logs` path (the full `Vec` and -/// the materialized execution trace), neither of which the marker needs. +/// Streams execution via `Executor::resume()`. The committed marker is +/// read directly off the executor's memory. This avoids OOMs. fn execute_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec { use executor::elf::Elf; use executor::vm::execution::Executor; @@ -142,19 +123,12 @@ fn execute_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8] committed } -/// Epoch size for the memory-bounded outer prove: 2^20 ≈ 1M cycles per epoch. -/// Each epoch's trace+LDE is what bounds peak RAM, so this trades per-epoch -/// footprint against epoch count — small enough to fit a normal box, large -/// enough to keep epoch count (and thus the cross-epoch overhead) modest. Well -/// under `local_to_global::MAX_EPOCHS` (2^20) for any guest we run here. +/// Epoch size for the outer prove: 2^20 ≈ 1M cycles per epoch. const OUTER_EPOCH_SIZE_LOG2: u32 = 20; -/// Prove the recursion guest's execution on `blob` **memory-bounded** via +/// Prove the recursion guest's execution on `blob` memory-bounded via /// continuations and verify the bundle on the host, returning the bytes the -/// guest committed. `prove_and_verify_continuation` proves epoch-by-epoch -/// (holding one epoch's trace at a time) and reconstructs the committed output -/// from the per-epoch bound slices, so a `Some(output)` already means every -/// epoch proof, the cross-epoch linkage, and the L2G binding verified. +/// guest committed. fn prove_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec { let opts = crate::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is always valid"); @@ -245,10 +219,7 @@ fn run_recursion_pipeline( /// Reproduce the recursion guest's EXACT path on the host — decode the postcard /// blob into `(VmProof, Vec, ProofOptions)` and call `verify_with_options`. -/// The cheapest regression guard in this file: no VM execution, just the -/// encode/decode contract plus a host verify, so it catches drift in the proof -/// format or the blob layout in seconds. Unlike the guest, a failure here -/// surfaces the actual error instead of an infinite abort loop. +/// Cheap regression guard. #[test] #[ignore = "needs prebuilt guest ELF (make compile-recursion-elfs)"] fn test_recursion_blob_decodes_and_verifies_on_host() { @@ -280,13 +251,6 @@ fn test_recursion_blob_decodes_and_verifies_on_host() { } // === Execute-only tier ======================================================== -// Mirrors the proving tests below, but stops at `OuterMode::ExecuteOnly`: the -// guest runs in-VM and we read the committed marker straight off the executor's -// memory, skipping the outer STARK prove. Streams execution (no trace, no -// buffered logs), so the footprint is bounded to the VM's touched memory. -// -// Not `#[ignore]`d: each runs in well under a minute, so they execute in the -// regular per-PR prover test job (which builds the recursion guest ELFs first). /// Execute-only mirror of `test_recursion_prove_empty`: verify a `blowup=8` /// proof of the empty program in-VM. @@ -338,8 +302,7 @@ fn test_recursion_execute() { // === Full-prove tier ========================================================== /// Inner program: empty (halt immediately). Useful for measuring the -/// lambda-vm verifier's intrinsic recursion overhead — i.e. what it costs -/// to verify the smallest possible lambda-vm proof, with no inner workload. +/// verifier's intrinsic recursion overhead. #[test] #[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"] fn test_recursion_prove_empty() { @@ -354,9 +317,7 @@ fn test_recursion_prove_empty() { } /// Inner program: empty, but with the absolute-minimum FRI parameters -/// (blowup=2, **fri_number_of_queries=1**). This is a "can the pipeline even -/// run end-to-end on a 125 GB box" experiment — security is intentionally -/// terrible. Use only for capacity probing. +/// (blowup=2, **fri_number_of_queries=1**). For quick profiling only. #[test] #[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"] fn test_recursion_prove_1query() { From 10cbbf02edf4273d0f5ab8235197efce73815e14 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Tue, 30 Jun 2026 16:12:31 -0300 Subject: [PATCH 4/6] less verbose --- .github/workflows/pr_main.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/pr_main.yaml b/.github/workflows/pr_main.yaml index e527f79a2..b5ed26306 100644 --- a/.github/workflows/pr_main.yaml +++ b/.github/workflows/pr_main.yaml @@ -308,9 +308,6 @@ jobs: with: name: prover-tests - # The un-ignored recursion execute-only smoke tests read prebuilt guest - # ELFs from executor/program_artifacts/recursion/ at runtime, so build - # them before the test run (cached on the guest sources + toolchain spec). - name: Cache compiled recursion guest ELF artifacts id: cache-recursion-elfs uses: actions/cache@v4 @@ -393,9 +390,6 @@ jobs: with: name: prover-tests - # The #[ignore]d recursion full-prove smoke tests (run only here, via - # --run-ignored) read prebuilt guest ELFs from - # executor/program_artifacts/recursion/ at runtime, so build them first. - name: Cache compiled recursion guest ELF artifacts id: cache-recursion-elfs uses: actions/cache@v4 From 5f6181f40abb625f6ac53e1353b34ce34e52e572 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Tue, 30 Jun 2026 16:44:23 -0300 Subject: [PATCH 5/6] re-ignore slow tests, fix cache keys --- .github/workflows/pr_main.yaml | 43 +++++++++++------------- prover/src/tests/recursion_smoke_test.rs | 3 ++ 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/.github/workflows/pr_main.yaml b/.github/workflows/pr_main.yaml index b5ed26306..96bb2f39a 100644 --- a/.github/workflows/pr_main.yaml +++ b/.github/workflows/pr_main.yaml @@ -308,26 +308,6 @@ jobs: with: name: prover-tests - - name: Cache compiled recursion guest ELF artifacts - id: cache-recursion-elfs - uses: actions/cache@v4 - with: - path: executor/program_artifacts/recursion - key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} - restore-keys: | - recursion-elf-artifacts- - - # Only needed when the recursion cache missed but the rust-elf cache hit - # (so the rust-elf compile step's Setup Rust did not run). - - name: Setup Rust Environment (recursion ELFs) - if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true' - uses: ./.github/actions/setup-rust - - - name: Compile recursion guest ELFs - if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' - run: | - make compile-recursion-elfs - - name: Run prover tests (shard ${{ matrix.partition }}/4) run: | cargo nextest run \ @@ -395,12 +375,10 @@ jobs: uses: actions/cache@v4 with: path: executor/program_artifacts/recursion - key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'syscalls/**', 'Makefile') }} + key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }} restore-keys: | recursion-elf-artifacts- - # Only needed when the recursion cache missed but the rust-elf cache hit - # (so the rust-elf compile step's Setup Rust did not run). - name: Setup Rust Environment (recursion ELFs) if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true' uses: ./.github/actions/setup-rust @@ -415,7 +393,7 @@ jobs: cargo nextest run \ --archive-file prover-tests.tar.zst \ --test-threads=1 \ - -E 'test(test_prove_elfs_all_instructions_64_full) | test(test_recursion_prove)' \ + -E 'test(test_prove_elfs_all_instructions_64_full) | test(test_recursion)' \ --run-ignored ignored-only # Seed ELF caches on refs/heads/main so merge-queue runs can restore them. @@ -464,3 +442,20 @@ jobs: - name: Compile Rust programs to ELF if: steps.cache-rust-elfs.outputs.cache-hit != 'true' run: make compile-programs-rust + + - name: Cache compiled recursion guest ELF artifacts + id: cache-recursion-elfs + uses: actions/cache@v4 + with: + path: executor/program_artifacts/recursion + key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }} + restore-keys: | + recursion-elf-artifacts- + + - name: Setup Rust Environment (recursion ELFs) + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true' + uses: ./.github/actions/setup-rust + + - name: Compile recursion guest ELFs + if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' + run: make compile-recursion-elfs diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index a52858228..7bcd4bd3d 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -255,6 +255,7 @@ fn test_recursion_blob_decodes_and_verifies_on_host() { /// Execute-only mirror of `test_recursion_prove_empty`: verify a `blowup=8` /// proof of the empty program in-VM. #[test] +#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"] fn test_recursion_execute_empty() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -269,6 +270,7 @@ fn test_recursion_execute_empty() { /// Execute-only mirror of `test_recursion_prove_1query`: smallest possible /// inner proof (blowup=2, 1 query) → least guest work. #[test] +#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"] fn test_recursion_execute_1query() { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); @@ -284,6 +286,7 @@ fn test_recursion_execute_1query() { /// Execute-only mirror of `test_recursion_prove`: verify a `blowup=8` proof of /// fibonacci(10) in-VM. #[test] +#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"] fn test_recursion_execute() { let root = workspace_root(); let fib_elf_bytes = read_guest_elf(&root, "fibonacci"); From 307e74755e0c2f90627515d967b7f754c26987c1 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Wed, 1 Jul 2026 18:03:44 -0300 Subject: [PATCH 6/6] test(recursion): CI runs execute-only tier; manual prove uses smaller epochs The prove tier's continuation bundle retains every epoch's STARK proof in RAM, so it OOMs CI runners. Narrow the CI selector to test_recursion_execute (streaming, bounded) and leave the prove tier manual-only with a 2^16 epoch to cap the per-epoch trace+LDE spike. --- .github/workflows/pr_main.yaml | 2 +- prover/src/tests/recursion_smoke_test.rs | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr_main.yaml b/.github/workflows/pr_main.yaml index 96bb2f39a..7b6258179 100644 --- a/.github/workflows/pr_main.yaml +++ b/.github/workflows/pr_main.yaml @@ -393,7 +393,7 @@ jobs: cargo nextest run \ --archive-file prover-tests.tar.zst \ --test-threads=1 \ - -E 'test(test_prove_elfs_all_instructions_64_full) | test(test_recursion)' \ + -E 'test(test_prove_elfs_all_instructions_64_full) | test(test_recursion_execute)' \ --run-ignored ignored-only # Seed ELF caches on refs/heads/main so merge-queue runs can restore them. diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index 7bcd4bd3d..f072eec53 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -84,8 +84,11 @@ enum OuterMode { /// builds a `Traces`, so footprint stays bounded to the VM's touched /// memory + instruction cache. Skips the LDE/FRI of the full pipeline entirely. ExecuteOnly, - /// Prove the guest's execution memory-bounded via continuations, then - /// verify the outer proof on the host. Peak RAM is a single epoch's proof. + /// Prove the guest's execution via continuations, then verify the outer + /// proof on the host. `prove_and_verify_continuation` retains every epoch's + /// STARK proof in the bundle before verifying, so peak RAM grows with epoch + /// count. Heavy — excluded from CI, run manually. A future verify-one-and- + /// discard API extension would make this memory-friendlier. Prove, } @@ -123,8 +126,9 @@ fn execute_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8] committed } -/// Epoch size for the outer prove: 2^20 ≈ 1M cycles per epoch. -const OUTER_EPOCH_SIZE_LOG2: u32 = 20; +/// Epoch size for the outer prove: 2^16 ≈ 65K cycles. Small so one epoch's +/// trace+LDE stays under the ~16GiB CI runners. +const OUTER_EPOCH_SIZE_LOG2: u32 = 16; /// Prove the recursion guest's execution on `blob` memory-bounded via /// continuations and verify the bundle on the host, returning the bytes the