feat(recursion-v2): add leaf verifier prover wiring (#1367)

Closes #1365. Stacked on #1363.

## Summary
- Add `AggProver<LEAF_FANIN, INTERNAL_FANIN>` orchestrating the leaf →
internal → root recursion pipeline with configurable fanin and system
params per layer
- Add `rebuild_circuit_index()` on `ZKVMVerifyingKey` for
post-deserialization reconstruction of the serde-skipped index map
- Fix `PvsAirConsistencyBus` LogUp imbalance (uncomment
`add_key_with_lookups` in VerifierPvsAir)
- Use `--pcs=basefold` in fixture generation to match the test's
expected PCS type
- Rewrite e2e tests to use `AggProver` API

## Test plan
- `bash ceno_recursion_v2/scripts/run_e2e_test.sh` (generates fixtures +
runs all prover_integration tests)

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: kunxian-xia

.github/workflows/integration.yml

@@ -189,11 +189,11 @@ jobs:
           CENO_CROSS_SHARD_LIMIT: 32
         run: cargo run --release --package ceno_zkvm --features sanity-check --bin e2e -- --platform=ceno --max-cycle-per-shard=20000 --hints=10 --public-io=4191 examples/target/riscv32im-ceno-zkvm-elf/release/examples/fibonacci
 
-      - name: Run aggregation e2e
+      - name: Run recursion v2 aggregation e2e
         env:
           RUSTFLAGS: "-C opt-level=3"
-        run: |
-          cargo run --release --package ceno_recursion --bin e2e_aggregate -- --platform=ceno --max-cycle-per-shard=1600 examples/target/riscv32im-ceno-zkvm-elf/release/examples/keccak_syscall
+          RUST_MIN_STACK: "33554432"
+        run: bash ceno_recursion_v2/scripts/run_e2e_test.sh
 
       - name: Install cargo make
         run: |

.github/workflows/tests.yml

@@ -54,17 +54,11 @@ jobs:
         run: |
           cargo make --version || cargo install cargo-make
 
-      #      - name: Run aggregation tests
+      #      - name: Run recursion v2 aggregation tests
       #        env:
       #          RUSTFLAGS: "" # cleanup RUSTFLAGS and avoid target-cpu=native
-      #        run: |
-      #          cargo run --release --package ceno_zkvm --bin e2e -- \
-      #            --platform=ceno --max-cycle-per-shard=1600 \
-      #            examples/target/riscv32im-ceno-zkvm-elf/release/examples/keccak_syscall
-      #          mkdir -p ceno_recursion/src/imported
-      #          mv proof.bin ceno_recursion/src/imported/
-      #          mv vk.bin ceno_recursion/src/imported/
-      #          cargo test --release --package ceno_recursion --tests -- --ignored
+      #          RUST_MIN_STACK: "33554432"
+      #        run: bash ceno_recursion_v2/scripts/run_e2e_test.sh
 
       - name: run test
         env:

ceno_recursion_v2/scripts/run_e2e_test.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Run the recursion v2 e2e integration test.
+# Generates base-layer proof fixtures if they don't exist.
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+FIXTURE_DIR="${CENO_RECURSION_V2_FIXTURE_DIR:-$REPO_ROOT/ceno_recursion_v2/src/imported}"
+PROOF_FILE="$FIXTURE_DIR/proof.bin"
+VK_FILE="$FIXTURE_DIR/vk.bin"
+FORCE_REGEN="${CENO_RECURSION_V2_FORCE_REGEN:-0}"
+MAX_CYCLE_PER_SHARD="${CENO_RECURSION_V2_MAX_CYCLE_PER_SHARD:-16000}"
+HINTS="${CENO_RECURSION_V2_HINTS:-10}"
+PUBLIC_IO="${CENO_RECURSION_V2_PUBLIC_IO:-4191}"
+
+# --- Step 1: ensure fixtures exist ---
+
+if [[ "$FORCE_REGEN" != "1" && -f "$PROOF_FILE" && -f "$VK_FILE" ]]; then
+    echo "[fixtures] found:"
+    ls -lh "$PROOF_FILE" "$VK_FILE"
+else
+    echo "[fixtures] generating base-layer proofs (fibonacci, --max-cycle-per-shard=$MAX_CYCLE_PER_SHARD) ..."
+
+    ELF="$REPO_ROOT/examples/target/riscv32im-ceno-zkvm-elf/release/examples/fibonacci"
+    if [[ ! -f "$ELF" ]]; then
+        echo "[fixtures] building example ELFs..."
+        cargo build --release --manifest-path "$REPO_ROOT/examples/Cargo.toml" --examples
+    fi
+
+    mkdir -p "$FIXTURE_DIR"
+    cargo run --release --package ceno_zkvm --bin e2e -- \
+        --platform=ceno \
+        --pcs=basefold \
+        --max-cycle-per-shard="$MAX_CYCLE_PER_SHARD" \
+        --hints="$HINTS" \
+        --public-io="$PUBLIC_IO" \
+        "$ELF" \
+        "$PROOF_FILE" \
+        "$VK_FILE"
+
+    echo "[fixtures] generated:"
+    ls -lh "$PROOF_FILE" "$VK_FILE"
+fi
+
+# --- Step 2: run the e2e integration test ---
+
+echo ""
+echo "[test] running recursion v2 e2e tests ..."
+cd "$REPO_ROOT/ceno_recursion_v2"
+CENO_RECURSION_V2_FIXTURE_DIR="$FIXTURE_DIR" \
+RUST_MIN_STACK=33554432 \
+cargo test --release \
+    'continuation::tests::prover_integration' \
+    -- --nocapture

ceno_recursion_v2/src/batch_constraint/expr_eval/symbolic_expression/trace.rs

@@ -227,10 +227,7 @@ impl RowMajorChip<F> for SymbolicExpressionTraceGenerator {
 
                 let mut node_idx = constraints.nodes.len();
                 for unused_var in &vk.unused_variables {
-                    if matches!(
-                        unused_var.entry,
-                        Entry::Public | Entry::Challenge
-                    ) {
+                    if matches!(unused_var.entry, Entry::Public | Entry::Challenge) {
                         continue;
                     }
                     let mut args = [F::ZERO; 2 * D_EF];

ceno_recursion_v2/src/batch_constraint/mod.rs

@@ -5,11 +5,11 @@ use openvm_cpu_backend::CpuBackend;
 use openvm_poseidon2_air::POSEIDON2_WIDTH;
 use openvm_stark_backend::{
     AirRef, FiatShamirTranscript, StarkEngine, StarkProtocolConfig, TranscriptHistory,
-    prover::{CommittedTraceData, TraceCommitter},
+    prover::{AirProvingContext, CommittedTraceData, TraceCommitter},
 };
 use openvm_stark_sdk::config::baby_bear_poseidon2::{BabyBearPoseidon2Config, EF, F};
 use p3_field::PrimeCharacteristicRing;
-use p3_matrix::dense::RowMajorMatrix;
+use p3_matrix::{Matrix, dense::RowMajorMatrix};
 
 use crate::{
     batch_constraint::{
@@ -19,10 +19,7 @@ use crate::{
         },
         expr_eval::{
             ConstraintsFoldingAir, ConstraintsFoldingCols,
-            symbolic_expression::{
-                CachedSymbolicExpressionColumns, SingleMainSymbolicExpressionColumns,
-                SymbolicExpressionAir,
-            },
+            symbolic_expression::{SingleMainSymbolicExpressionColumns, SymbolicExpressionAir},
         },
         expression_claim::{ExpressionClaimAir, ExpressionClaimCols},
     },
@@ -184,14 +181,47 @@ impl BatchConstraintModule {
         };
     }
 
-    fn placeholder_air_widths(&self) -> [usize; 3] {
+    fn placeholder_common_widths(&self) -> [usize; 3] {
         [
-            CachedSymbolicExpressionColumns::<u8>::width()
-                + SingleMainSymbolicExpressionColumns::<u8>::width() * self.max_num_proofs,
+            SingleMainSymbolicExpressionColumns::<u8>::width() * self.max_num_proofs,
             ConstraintsFoldingCols::<u8>::width(),
             ExpressionClaimCols::<u8>::width(),
         ]
     }
+
+    pub(crate) fn generate_placeholder_proving_ctxs<SC: StarkProtocolConfig<F = F>>(
+        &self,
+        cached_symbolic_expr_trace: CommittedTraceData<CpuBackend<SC>>,
+        required_heights: Option<&[usize]>,
+    ) -> Option<Vec<AirProvingContext<CpuBackend<SC>>>> {
+        let widths = self.placeholder_common_widths();
+        let air_count = required_heights
+            .map(|heights| heights.len())
+            .unwrap_or(self.num_airs());
+        let cached_height = cached_symbolic_expr_trace.trace.height();
+
+        (0..air_count)
+            .map(|idx| {
+                let height = required_heights
+                    .and_then(|heights| heights.get(idx).copied())
+                    .unwrap_or_else(|| if idx == 0 { cached_height } else { 2 });
+                if height < 2 {
+                    return None;
+                }
+                let width = widths.get(idx).copied().unwrap_or(1).max(1);
+                let matrix = RowMajorMatrix::new(vec![F::ZERO; height * width], width);
+                Some(if idx == 0 {
+                    AirProvingContext {
+                        cached_mains: vec![cached_symbolic_expr_trace.clone()],
+                        common_main: matrix,
+                        public_values: vec![],
+                    }
+                } else {
+                    AirProvingContext::simple_no_pis(matrix)
+                })
+            })
+            .collect::<Option<Vec<_>>>()
+    }
 }
 
 impl AirModule for BatchConstraintModule {
@@ -251,7 +281,7 @@ impl<SC: StarkProtocolConfig<F = F>> TraceGenModule<GlobalCtxCpu, CpuBackend<SC>
         _ctx: &Self::ModuleSpecificCtx<'_>,
         required_heights: Option<&[usize]>,
     ) -> Option<Vec<openvm_stark_backend::prover::AirProvingContext<CpuBackend<SC>>>> {
-        let widths = self.placeholder_air_widths();
+        let widths = self.placeholder_common_widths();
         let air_count = required_heights
             .map(|heights| heights.len())
             .unwrap_or(self.num_airs());

ceno_recursion_v2/src/circuit/inner/verifier/air.rs

@@ -246,17 +246,15 @@ impl<AB: AirBuilder + InteractionBuilder + AirBuilderWithPublicValues> Air<AB> f
         //     local.is_valid * is_internal,
         // );
 
-        // We provide proof metadata for lookup here to ensure consistency between AIRs that
-        // process public values.
-        // self.pvs_air_consistency_bus.add_key_with_lookups(
-        //     builder,
-        //     local.proof_idx,
-        //     PvsAirConsistencyMessage {
-        //         deferral_flag,
-        //         has_verifier_pvs: local.has_verifier_pvs.into(),
-        //     },
-        //     local.is_valid * consistency_mult,
-        // );
+        self.pvs_air_consistency_bus.add_key_with_lookups(
+            builder,
+            local.proof_idx,
+            PvsAirConsistencyMessage {
+                deferral_flag,
+                has_verifier_pvs: local.has_verifier_pvs.into(),
+            },
+            local.is_valid * consistency_mult,
+        );
 
         // Finally, we need to constrain that the public values this AIR produces are consistent
         // with the child's. Note that we only impose constraints for layers below the current

ceno_recursion_v2/src/continuation/prover/inner/mod.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::{sync::Arc, time::Instant};
 
 use ceno_zkvm::scheme::ZKVMProof;
 use continuations_v2::SC;
@@ -9,7 +9,10 @@ use openvm_stark_backend::{
     StarkEngine,
     keygen::types::{MultiStarkProvingKey, MultiStarkVerifyingKey},
     proof::Proof,
-    prover::{CommittedTraceData, DeviceMultiStarkProvingKey, ProverBackend, ProvingContext},
+    prover::{
+        AirProvingContext, CommittedTraceData, DeviceMultiStarkProvingKey, MatrixDimensions,
+        ProverBackend, ProvingContext,
+    },
 };
 use openvm_stark_sdk::config::baby_bear_poseidon2::{
     Digest, EF, F, default_duplex_sponge_recorder,
@@ -183,18 +186,32 @@ where
         proofs: &[ZKVMProof<RecursionField, Basefold<RecursionField, BasefoldRSParams>>],
         child_vk_kind: ChildVkKind,
     ) -> Result<Proof<SC>> {
+        let tracegen_start = Instant::now();
         let ctx = self.generate_proving_ctx(proofs, child_vk_kind, ProofsType::Vm, None);
-        if tracing::enabled!(tracing::Level::DEBUG) {
-            // TODO enable trace height
-            //     trace_heights_tracing_info::<_, SC>(&ctx.per_trace, &self.circuit.airs());
+        tracing::info!(
+            elapsed_ms = tracegen_start.elapsed().as_secs_f64() * 1000.0,
+            num_traces = ctx.per_trace.len(),
+            "generated recursion proving context"
+        );
+        if tracing::enabled!(tracing::Level::INFO) {
+            trace_heights_tracing_info::<PB, SC>(&ctx.per_trace, &self.circuit.airs());
         }
 
         let engine = E::new(self.pk.params.clone());
         #[cfg(debug_assertions)]
         debug_constraints(&self.circuit, &ctx, &engine);
+        let prove_start = Instant::now();
         let proof = engine.prove(&self.d_pk, ctx)?;
-        #[cfg(debug_assertions)]
+        tracing::info!(
+            elapsed_ms = prove_start.elapsed().as_secs_f64() * 1000.0,
+            "proved recursion aggregation"
+        );
+        let verify_start = Instant::now();
         engine.verify(&self.vk, &proof)?;
+        tracing::info!(
+            elapsed_ms = verify_start.elapsed().as_secs_f64() * 1000.0,
+            "verified recursion aggregation proof"
+        );
         Ok(proof)
     }
 
@@ -272,10 +289,42 @@ where
         self.vk.clone()
     }
 
+    #[cfg(test)]
+    pub(crate) fn air_names(&self) -> Vec<String> {
+        <InnerCircuit<S> as Circuit<SC>>::airs(self.circuit.as_ref())
+            .iter()
+            .map(|air| air.name().to_string())
+            .collect()
+    }
+
     pub fn get_self_vk_pcs_data(&self) -> Option<CommittedTraceData<PB>>
     where
         CommittedTraceData<PB>: Clone,
     {
         self.self_vk_pcs_data.clone()
     }
 }
+
+fn trace_heights_tracing_info<PB: ProverBackend, SC: openvm_stark_backend::StarkProtocolConfig>(
+    ctxs: &[(usize, AirProvingContext<PB>)],
+    airs: &[openvm_stark_backend::AirRef<SC>],
+) {
+    let mut total_cells = 0usize;
+    let mut total_width = 0usize;
+    for ((air_id, ctx), air) in ctxs.iter().zip(airs) {
+        let height = ctx.common_main.height();
+        let width = ctx.common_main.width();
+        let cells = height * width;
+        tracing::info!(
+            air_id,
+            air_name = air.name(),
+            height,
+            width,
+            cells,
+            "recursion trace dimensions"
+        );
+        total_cells += cells;
+        total_width += width;
+    }
+    tracing::info!(total_cells, total_width, "recursion trace totals");
+}