test(recursion): memory-bounded execute/prove smoke tests; wire CI (#750)

* test(recursion): memory-bounded execute/prove smoke tests; wire CI

Stream the execute-only path via Executor::resume() and read the committed
marker straight off Memory (Executor::finish), dropping the run()+Traces
build that OOM'd. Prove the outer recursion guest memory-bounded via
prove_and_verify_continuation (2^20-cycle epochs) instead of a monolithic
~125 GB prove.

Un-ignore the three execute-only tests (now sub-minute); the full-prove
tests stay #[ignore]d (>60s) and run in the comprehensive CI job.

CI: build the recursion guest ELFs (cache + conditional setup-rust +
make compile-recursion-elfs) in both the per-PR and comprehensive prover
jobs, and add test(test_recursion_prove) to the comprehensive filter.

* cargo fmt

* a bit of brevity

* less verbose

* re-ignore slow tests, fix cache keys

* test(recursion): CI runs execute-only tier; manual prove uses smaller epochs

The prove tier's continuation bundle retains every epoch's STARK proof in
RAM, so it OOMs CI runners. Narrow the CI selector to test_recursion_execute
(streaming, bounded) and leave the prove tier manual-only with a 2^16 epoch
to cap the per-epoch trace+LDE spike.

Author: Oppen

.github/workflows/pr_main.yaml

@@ -370,12 +370,30 @@ jobs:
         with:
           name: prover-tests
 
+      - name: Cache compiled recursion guest ELF artifacts
+        id: cache-recursion-elfs
+        uses: actions/cache@v4
+        with:
+          path: executor/program_artifacts/recursion
+          key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }}
+          restore-keys: |
+            recursion-elf-artifacts-
+
+      - name: Setup Rust Environment (recursion ELFs)
+        if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true'
+        uses: ./.github/actions/setup-rust
+
+      - name: Compile recursion guest ELFs
+        if: steps.cache-recursion-elfs.outputs.cache-hit != 'true'
+        run: |
+          make compile-recursion-elfs
+
       - name: Run comprehensive prover tests
         run: |
           cargo nextest run \
             --archive-file prover-tests.tar.zst \
             --test-threads=1 \
-            -E 'test(test_prove_elfs_all_instructions_64_full)' \
+            -E 'test(test_prove_elfs_all_instructions_64_full) | test(test_recursion_execute)' \
             --run-ignored ignored-only
 
   # Seed ELF caches on refs/heads/main so merge-queue runs can restore them.
@@ -424,3 +442,20 @@ jobs:
       - name: Compile Rust programs to ELF
         if: steps.cache-rust-elfs.outputs.cache-hit != 'true'
         run: make compile-programs-rust
+
+      - name: Cache compiled recursion guest ELF artifacts
+        id: cache-recursion-elfs
+        uses: actions/cache@v4
+        with:
+          path: executor/program_artifacts/recursion
+          key: recursion-elf-artifacts-${{ hashFiles('bench_vs/lambda/**', 'prover/src/**', 'prover/Cargo.toml', 'crypto/**/src/**', 'crypto/**/Cargo.toml', 'executor/src/**', 'executor/Cargo.toml', 'syscalls/**', 'executor/programs/riscv64im-lambda-vm-elf.json', 'Makefile') }}
+          restore-keys: |
+            recursion-elf-artifacts-
+
+      - name: Setup Rust Environment (recursion ELFs)
+        if: steps.cache-recursion-elfs.outputs.cache-hit != 'true' && steps.cache-rust-elfs.outputs.cache-hit == 'true'
+        uses: ./.github/actions/setup-rust
+
+      - name: Compile recursion guest ELFs
+        if: steps.cache-recursion-elfs.outputs.cache-hit != 'true'
+        run: make compile-recursion-elfs

prover/src/tests/recursion_smoke_test.rs

@@ -4,17 +4,13 @@
 //! 1. Proves an inner program on the host.
 //! 2. Serializes `(VmProof, inner_elf, opts)` with postcard.
 //! 3. Hands that as private input to the recursion guest.
-//! 4. Either **proves** the recursion guest's execution and verifies the outer
-//!    proof (`OuterMode::Prove`), or merely **executes** the guest in-VM and
-//!    reads the committed marker off the trace (`OuterMode::ExecuteOnly`) — a
-//!    cheaper tier that skips the LDE/FRI that dominate the full pipeline.
+//! 4. Either **proves** the recursion guest's execution (memory-bounded via
+//!    continuations) and verifies the outer proof (`OuterMode::Prove`), or
+//!    merely **executes** the guest in-VM and reads the committed marker
+//!    straight off the executor's memory (`OuterMode::ExecuteOnly`) — a cheaper
+//!    tier that skips the LDE/FRI that dominate the full pipeline.
 //!
-//! The guest ELFs are built by `make compile-recursion-elfs` (which the
-//! `test-prover-all` make target depends on) and read from
-//! `executor/program_artifacts/recursion/`, like every other program test.
-//!
-//! Tests are `#[ignore]`d because the outer proof runs the full STARK verifier
-//! inside the VM (minutes per run, large memory footprint).
+//! The guest ELFs are assumed built by `make compile-recursion-elfs`.
 
 use std::path::PathBuf;
 
@@ -83,64 +79,77 @@ fn prove_inner_and_encode_blob(
 /// whether we also prove the guest's own execution.
 #[derive(Clone, Copy, Debug)]
 enum OuterMode {
-    /// Execute the guest in-VM and read the committed marker off the trace.
-    /// Skips the LDE blowup + FRI commit that dominate the full pipeline's
-    /// footprint, so it needs materially less RAM than `Prove`.
-    ///
-    /// "Less" is not "little": `Executor::run` retains a per-instruction log
-    /// and `Traces` materializes the full execution trace, so verifying even a
-    /// 1-query inner proof still needs tens of GB — it OOMs on a 36 GB box.
+    /// Execute the guest in-VM and read the committed marker straight off the
+    /// executor's memory. Streams logs via `Executor::resume()` and never
+    /// builds a `Traces`, so footprint stays bounded to the VM's touched
+    /// memory + instruction cache. Skips the LDE/FRI of the full pipeline entirely.
     ExecuteOnly,
-    /// Prove the guest's execution and verify the outer proof on the host. The
-    /// full STARK verifier inside the VM — minutes per run, ~125 GB.
+    /// Prove the guest's execution via continuations, then verify the outer
+    /// proof on the host. `prove_and_verify_continuation` retains every epoch's
+    /// STARK proof in the bundle before verifying, so peak RAM grows with epoch
+    /// count. Heavy — excluded from CI, run manually. A future verify-one-and-
+    /// discard API extension would make this memory-friendlier.
     Prove,
 }
 
 /// Execute the recursion guest in-VM on `blob` and return the bytes it
 /// committed (the success marker the in-VM verifier emits).
+///
+/// Streams execution via `Executor::resume()`. The committed marker is
+/// read directly off the executor's memory. This avoids OOMs.
 fn execute_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec<u8> {
     use executor::elf::Elf;
     use executor::vm::execution::Executor;
 
-    eprintln!("[{label}] executing outer (recursion guest, in-VM verify) ...");
+    eprintln!("[{label}] executing outer (recursion guest, in-VM verify, streaming) ...");
     let program = Elf::load(recursion_elf_bytes).expect("load recursion elf");
-    let result = Executor::new(&program, blob.to_vec())
-        .expect("executor new")
-        .run()
-        .expect("recursion guest execution failed (verify panicked in-VM?)");
-
-    let traces = crate::tables::trace_builder::Traces::from_elf_and_logs(
-        &program,
-        &result.logs,
-        &crate::MaxRowsConfig::default(),
-        blob,
-        #[cfg(feature = "disk-spill")]
-        stark::storage_mode::StorageMode::Ram,
-    )
-    .expect("trace build");
+    let mut executor = Executor::new(&program, blob.to_vec()).expect("executor new");
+
+    // Drain chunks to completion without retaining logs or building a trace.
+    while executor
+        .resume()
+        .expect("recursion guest execution failed (verify panicked in-VM?)")
+        .is_some()
+    {}
+
+    let committed = executor
+        .finish()
+        .expect("read committed output after execution")
+        .memory_values;
 
     eprintln!(
         "[{label}] committed {} bytes: {:?} (as str: {:?})",
-        traces.public_output_bytes.len(),
-        traces.public_output_bytes,
-        String::from_utf8_lossy(&traces.public_output_bytes),
+        committed.len(),
+        committed,
+        String::from_utf8_lossy(&committed),
     );
-    traces.public_output_bytes
+    committed
 }
 
-/// Prove the recursion guest's execution on `blob`, verify the outer proof on
-/// the host, and return the bytes the guest committed.
-fn prove_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec<u8> {
-    eprintln!("[{label}] proving outer (recursion guest) ...");
-    let outer_proof =
-        crate::prove_with_inputs(recursion_elf_bytes, blob).expect("outer prove should succeed");
-    eprintln!("[{label}] outer proof generated");
+/// Epoch size for the outer prove: 2^16 ≈ 65K cycles. Small so one epoch's
+/// trace+LDE stays under the ~16GiB CI runners.
+const OUTER_EPOCH_SIZE_LOG2: u32 = 16;
 
-    assert!(
-        crate::verify(&outer_proof, recursion_elf_bytes).expect("outer verify errored"),
-        "outer proof must verify on host"
+/// Prove the recursion guest's execution on `blob` memory-bounded via
+/// continuations and verify the bundle on the host, returning the bytes the
+/// guest committed.
+fn prove_outer_and_commit(label: &str, recursion_elf_bytes: &[u8], blob: &[u8]) -> Vec<u8> {
+    let opts =
+        crate::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is always valid");
+    eprintln!(
+        "[{label}] proving outer (recursion guest) via continuations \
+         (epoch=2^{OUTER_EPOCH_SIZE_LOG2} cycles) ..."
     );
-    outer_proof.public_output
+    let committed = crate::continuation::prove_and_verify_continuation(
+        recursion_elf_bytes,
+        blob,
+        OUTER_EPOCH_SIZE_LOG2,
+        &opts,
+    )
+    .expect("outer continuation prove/verify errored")
+    .expect("outer continuation proof must verify on host");
+    eprintln!("[{label}] outer continuation proof generated and verified");
+    committed
 }
 
 /// Core pipeline: prove an inner program with the given options, hand the
@@ -214,10 +223,7 @@ fn run_recursion_pipeline(
 
 /// Reproduce the recursion guest's EXACT path on the host — decode the postcard
 /// blob into `(VmProof, Vec<u8>, ProofOptions)` and call `verify_with_options`.
-/// The cheapest regression guard in this file: no VM execution, just the
-/// encode/decode contract plus a host verify, so it catches drift in the proof
-/// format or the blob layout in seconds. Unlike the guest, a failure here
-/// surfaces the actual error instead of an infinite abort loop.
+/// Cheap regression guard.
 #[test]
 #[ignore = "needs prebuilt guest ELF (make compile-recursion-elfs)"]
 fn test_recursion_blob_decodes_and_verifies_on_host() {
@@ -249,15 +255,11 @@ fn test_recursion_blob_decodes_and_verifies_on_host() {
 }
 
 // === Execute-only tier ========================================================
-// Mirrors the proving tests below, but stops at `OuterMode::ExecuteOnly`: the
-// guest runs in-VM and we read the committed marker off the trace, skipping the
-// outer STARK prove. Needs tens of GB (execution trace), not the ~125 GB the
-// full outer prove wants — but still OOMs on a 36 GB box.
 
 /// Execute-only mirror of `test_recursion_prove_empty`: verify a `blowup=8`
 /// proof of the empty program in-VM.
 #[test]
-#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"]
+#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"]
 fn test_recursion_execute_empty() {
     let root = workspace_root();
     let empty_elf_bytes = read_guest_elf(&root, "empty");
@@ -272,7 +274,7 @@ fn test_recursion_execute_empty() {
 /// Execute-only mirror of `test_recursion_prove_1query`: smallest possible
 /// inner proof (blowup=2, 1 query) → least guest work.
 #[test]
-#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"]
+#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"]
 fn test_recursion_execute_1query() {
     let root = workspace_root();
     let empty_elf_bytes = read_guest_elf(&root, "empty");
@@ -288,7 +290,7 @@ fn test_recursion_execute_1query() {
 /// Execute-only mirror of `test_recursion_prove`: verify a `blowup=8` proof of
 /// fibonacci(10) in-VM.
 #[test]
-#[ignore = "needs prebuilt recursion guest ELF + tens of GB RAM (execution trace)"]
+#[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"]
 fn test_recursion_execute() {
     let root = workspace_root();
     let fib_elf_bytes = read_guest_elf(&root, "fibonacci");
@@ -307,10 +309,9 @@ fn test_recursion_execute() {
 // === Full-prove tier ==========================================================
 
 /// Inner program: empty (halt immediately). Useful for measuring the
-/// lambda-vm verifier's intrinsic recursion overhead — i.e. what it costs
-/// to verify the smallest possible lambda-vm proof, with no inner workload.
+/// verifier's intrinsic recursion overhead.
 #[test]
-#[ignore = "slow: runs the full STARK verifier inside the VM"]
+#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"]
 fn test_recursion_prove_empty() {
     let root = workspace_root();
     let empty_elf_bytes = read_guest_elf(&root, "empty");
@@ -323,11 +324,9 @@ fn test_recursion_prove_empty() {
 }
 
 /// Inner program: empty, but with the absolute-minimum FRI parameters
-/// (blowup=2, **fri_number_of_queries=1**). This is a "can the pipeline even
-/// run end-to-end on a 125 GB box" experiment — security is intentionally
-/// terrible. Use only for capacity probing.
+/// (blowup=2, **fri_number_of_queries=1**). For quick profiling only.
 #[test]
-#[ignore = "slow: runs the full STARK verifier inside the VM"]
+#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"]
 fn test_recursion_prove_1query() {
     let root = workspace_root();
     let empty_elf_bytes = read_guest_elf(&root, "empty");
@@ -343,7 +342,7 @@ fn test_recursion_prove_1query() {
 
 /// Inner program: fibonacci(10).
 #[test]
-#[ignore = "slow: runs the full STARK verifier inside the VM"]
+#[ignore = "slow: memory-bounded continuation prove of the verifier-in-VM"]
 fn test_recursion_prove() {
     let root = workspace_root();
     let fib_elf_bytes = read_guest_elf(&root, "fibonacci");