feat: merge-train/spartan (#24911)

AztecBot · web-flow · commit 7ac944c0bf44 · 2026-07-22T19:29:28.000Z
BEGIN_COMMIT_OVERRIDE fix(bench): grant helm-sa logging.viewer for scraper log reads (#24908) feat(bench): publish block-capacity benchmark to the custom dashboard pipeline (#24913) feat(bench): publish real-proving benchmark to the custom dashboard pipeline (#24914) END_COMMIT_OVERRIDE
diff --git a/.github/workflows/nightly-spartan-bench.yml b/.github/workflows/nightly-spartan-bench.yml
@@ -349,6 +349,12 @@ jobs:
           AWS_SHUTDOWN_TIME: 180
           NO_SPOT: 1
           SKIP_NETWORK_DEPLOY: "1"
+          # Publishes to the custom network-dashboard pipeline (GCS). Fake/fixed-delay
+          # proving, so it's tagged simulated-proving (distinct from the weekly
+          # real-proving run).
+          BENCH_SWEEP_ID: simulated-proving-${{ github.run_id }}
+          BENCH_SWEEP_LABEL: simulated-proving
+          BENCH_BENCHMARK_TYPE: simulated-proving
         run: |
           ./.github/ci3.sh network-proving-bench prove-n-tps-fake prove-n-tps-fake "${{ needs.select-image.outputs.docker_image }}"
 
@@ -527,6 +533,11 @@ jobs:
           AWS_SHUTDOWN_TIME: 240
           NO_SPOT: 1
           SKIP_NETWORK_DEPLOY: "1"
+          # Publishes to the custom network-dashboard pipeline (GCS) tagged as the
+          # block-capacity benchmark, grouped as one sweep per run.
+          BENCH_SWEEP_ID: block-capacity-${{ github.run_id }}
+          BENCH_SWEEP_LABEL: block-capacity
+          BENCH_BENCHMARK_TYPE: block-capacity
         run: |
           ./.github/ci3.sh network-block-capacity-bench block-capacity nightly-block-capacity "${{ needs.select-image.outputs.docker_image }}"
 
diff --git a/.github/workflows/weekly-proving-bench.yml b/.github/workflows/weekly-proving-bench.yml
@@ -133,6 +133,11 @@ jobs:
           AWS_SHUTDOWN_TIME: 180
           NO_SPOT: 1
           SKIP_NETWORK_DEPLOY: "1"
+          # Publishes to the custom network-dashboard pipeline (GCS) tagged as the
+          # real-proving benchmark, grouped as one sweep per run.
+          BENCH_SWEEP_ID: real-proving-${{ github.run_id }}
+          BENCH_SWEEP_LABEL: real-proving
+          BENCH_BENCHMARK_TYPE: real-proving
         run: |
           ./.github/ci3.sh network-proving-bench prove-n-tps-real prove-n-tps-real
 
diff --git a/ci.sh b/ci.sh
@@ -353,7 +353,7 @@ case "$cmd" in
     export INSTANCE_POSTFIX="n-proving-bench"
     skip_network_deploy=0
     [ "${SKIP_NETWORK_DEPLOY:-0}" = "1" ] && skip_network_deploy=1
-    bootstrap_ec2 "SKIP_NETWORK_DEPLOY=$skip_network_deploy ./bootstrap.sh ci-network-proving-bench $*"
+    bootstrap_ec2 "BENCH_SWEEP_ID=${BENCH_SWEEP_ID:-} BENCH_SWEEP_LABEL=${BENCH_SWEEP_LABEL:-} BENCH_BENCHMARK_TYPE=${BENCH_BENCHMARK_TYPE:-} SKIP_NETWORK_DEPLOY=$skip_network_deploy ./bootstrap.sh ci-network-proving-bench $*"
     ;;
   network-block-capacity-bench)
     # Args: <scenario> <namespace> [docker_image]
@@ -363,7 +363,7 @@ case "$cmd" in
     export INSTANCE_POSTFIX="n-block-cap-bench"
     skip_network_deploy=0
     [ "${SKIP_NETWORK_DEPLOY:-0}" = "1" ] && skip_network_deploy=1
-    bootstrap_ec2 "SKIP_NETWORK_DEPLOY=$skip_network_deploy ./bootstrap.sh ci-network-block-capacity-bench $*"
+    bootstrap_ec2 "BENCH_SWEEP_ID=${BENCH_SWEEP_ID:-} BENCH_SWEEP_LABEL=${BENCH_SWEEP_LABEL:-block-capacity} BENCH_BENCHMARK_TYPE=${BENCH_BENCHMARK_TYPE:-block-capacity} SKIP_NETWORK_DEPLOY=$skip_network_deploy ./bootstrap.sh ci-network-block-capacity-bench $*"
     ;;
   network-bench-10tps)
     # Args: <scenario> <namespace> [docker_image]
diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh
@@ -208,7 +208,40 @@ function proving_bench {
   gcp_auth
   export_admin_api_key
   export K8S_ENRICHER=${K8S_ENRICHER:-1}
-  proving_bench_cmds | parallelize 1
+  export BENCH_RUN_ID="${BENCH_RUN_ID:-$(date -u +%Y%m%d)-proving-${COMMIT_HASH:0:10}}"
+  # real proving (prove-n-tps-real, REAL_VERIFIER=true) vs fake/simulated timings.
+  local benchmark_type="${BENCH_BENCHMARK_TYPE:-$([ "${REAL_VERIFIER:-}" = "true" ] && echo real-proving || echo simulated-proving)}"
+
+  local test_rc=0
+  proving_bench_cmds | parallelize 1 || test_rc=$?
+  if [[ "$test_rc" -ne 0 ]]; then
+    echo "[proving_bench] test exited ${test_rc}; scraping captured data anyway"
+  fi
+
+  # Publish to the custom pipeline (GCS -> network-dashboard) alongside the
+  # legacy github-action-benchmark output the workflow still uploads.
+  local metadata="/tmp/n_tps_prove_timing_data.json"
+  local run_json="bench-out/bench-${benchmark_type}-${BENCH_RUN_ID}.json"
+  if [[ -f "$metadata" ]]; then
+    local started=$(jq -r .startedAt < "$metadata")
+    local ended=$(jq -r .endedAt < "$metadata")
+    echo "Scraping ${benchmark_type} run ${BENCH_RUN_ID} (started=${started} ended=${ended})"
+    NAMESPACE="$NAMESPACE" GCP_PROJECT_ID="${GCP_PROJECT_ID:-}" ./scripts/bench_10tps/bench_scrape.ts \
+      --run-id "$BENCH_RUN_ID" \
+      --started "$started" \
+      --ended "$ended" \
+      --target-tps "${TARGET_TPS:-1}" \
+      --sweep-id "${BENCH_SWEEP_ID:-$BENCH_RUN_ID}" \
+      --sweep-label "${BENCH_SWEEP_LABEL:-$benchmark_type}" \
+      --benchmark-type "$benchmark_type" \
+      --output "$run_json" \
+      || echo "[proving_bench] scraper failed (non-fatal)"
+    network_bench_upload "$run_json" || echo "[network_bench] upload failed (non-fatal)"
+  else
+    echo "[proving_bench] no timing metadata at ${metadata}; skipping custom-pipeline scrape"
+  fi
+
+  return "$test_rc"
 }
 
 function block_capacity_bench {
@@ -222,7 +255,39 @@ function block_capacity_bench {
   gcp_auth
   export_admin_api_key
   export K8S_ENRICHER=${K8S_ENRICHER:-1}
-  block_capacity_bench_cmds | parallelize 1
+  export BENCH_RUN_ID="${BENCH_RUN_ID:-$(date -u +%Y%m%d)-block-capacity-${COMMIT_HASH:0:10}}"
+
+  # Capture the test exit code but don't abort: even a partial run produced blocks
+  # worth scraping. We scrape below, then re-surface the failure at the end.
+  local test_rc=0
+  block_capacity_bench_cmds | parallelize 1 || test_rc=$?
+  if [[ "$test_rc" -ne 0 ]]; then
+    echo "[block_capacity_bench] test exited ${test_rc}; scraping captured data anyway"
+  fi
+
+  # Publish to the custom pipeline (GCS -> network-dashboard) alongside the
+  # legacy github-action-benchmark output the workflow still uploads.
+  local metadata="/tmp/block_capacity_timing_data.json"
+  local run_json="bench-out/bench-block-capacity-${BENCH_RUN_ID}.json"
+  if [[ -f "$metadata" ]]; then
+    local started=$(jq -r .startedAt < "$metadata")
+    local ended=$(jq -r .endedAt < "$metadata")
+    echo "Scraping block-capacity run ${BENCH_RUN_ID} (started=${started} ended=${ended})"
+    NAMESPACE="$NAMESPACE" GCP_PROJECT_ID="${GCP_PROJECT_ID:-}" ./scripts/bench_10tps/bench_scrape.ts \
+      --run-id "$BENCH_RUN_ID" \
+      --started "$started" \
+      --ended "$ended" \
+      --sweep-id "${BENCH_SWEEP_ID:-$BENCH_RUN_ID}" \
+      --sweep-label "${BENCH_SWEEP_LABEL:-block-capacity}" \
+      --benchmark-type "${BENCH_BENCHMARK_TYPE:-block-capacity}" \
+      --output "$run_json" \
+      || echo "[block_capacity_bench] scraper failed (non-fatal)"
+    network_bench_upload "$run_json" || echo "[network_bench] upload failed (non-fatal)"
+  else
+    echo "[block_capacity_bench] no timing metadata at ${metadata}; skipping custom-pipeline scrape"
+  fi
+
+  return "$test_rc"
 }
 
 # One point of the inclusion sweep: a fixed 1 TPS of high-value txs (the
diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json
@@ -682,6 +682,11 @@
           "type": "number",
           "minimum": 0
         },
+        "manaPerSec": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Mana consumed per second while building this block (totalManaUsed / buildDuration), from the l2-block-built log. The headline throughput signal for the block-capacity benchmark. Optional; absent when the block-built log was not captured."
+        },
         "totalPublicGas": {
           "type": "object",
           "additionalProperties": false,
diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts
@@ -840,6 +840,32 @@ const PROVING_INFRA_DEFS: Record<string, TimeSeriesDef> = {
     unit: "count",
     query: queueRateByJobType("aztec_proving_queue_resolved_jobs_count"),
   },
+  // Epoch proving: how long the prover node takes to prove a job (epoch) and how
+  // much it has proven. The headline real-proving signal — compare against the
+  // epoch wall-clock (AZTEC_PROOF_SUBMISSION_EPOCHS * epoch duration).
+  epochProvingDurationP50: {
+    metric: "aztec_prover_node_job_duration",
+    unit: "s",
+    query: proverNodeHist(0.5, "aztec_prover_node_job_duration_seconds_bucket"),
+  },
+  epochProvingDurationP99: {
+    metric: "aztec_prover_node_job_duration",
+    unit: "s",
+    query: proverNodeHist(
+      0.99,
+      "aztec_prover_node_job_duration_seconds_bucket",
+    ),
+  },
+  provenBlocks: {
+    metric: "aztec_prover_node_job_blocks",
+    unit: "count",
+    query: `sum(aztec_prover_node_job_blocks${NS})`,
+  },
+  provenTransactions: {
+    metric: "aztec_prover_node_job_transactions",
+    unit: "count",
+    query: `sum(aztec_prover_node_job_transactions${NS})`,
+  },
 };
 
 // Scrape a map of slug -> PromQL def via query_range. One failing query emits an
@@ -1218,6 +1244,7 @@ type BlockRecord = {
   silentlySkippedCount: number;
   silentlySkippedDurationMs: number;
   buildDurationSeconds: number;
+  manaPerSec?: number;
   totalPublicGas?: { daGas: number; l2Gas: number };
   totalSizeInBytes?: number;
   source: "log";
@@ -1313,6 +1340,9 @@ async function scrapeBlocks(
           ? finiteOrZero(numberPayloadField(processorPayload ?? {}, "duration"))
           : finiteOrZero(numberPayloadField(built.jsonPayload, "duration")) /
             1000,
+      manaPerSec: numberOrUndefined(
+        numberPayloadField(built?.jsonPayload ?? {}, "manaPerSec"),
+      ),
       totalPublicGas: processorPayload?.totalPublicGas as
         | { daGas: number; l2Gas: number }
         | undefined,
diff --git a/spartan/terraform/gke-cluster/iam.tf b/spartan/terraform/gke-cluster/iam.tf
@@ -50,6 +50,18 @@ resource "google_project_iam_member" "helm_sa_roles" {
   member   = "serviceAccount:${google_service_account.helm_sa.email}"
 }
 
+# helm-sa is the CI deploy identity (GitHub Actions secret GCP_SA_KEY). The bench
+# scraper runs `gcloud logging read` as this account to collect block/event/
+# sequencer-state records (l2-block-handled / l2-block-built / public-processor
+# logs); without logging read the reads are permission-denied and the scraper
+# silently emits empty block data (null totalTxsMined, empty build/validator
+# fields). Prometheus metrics use a kube port-forward and are unaffected.
+resource "google_project_iam_member" "helm_sa_logging_viewer" {
+  project = var.project
+  role    = "roles/logging.viewer"
+  member  = "serviceAccount:${google_service_account.helm_sa.email}"
+}
+
 # Create a service account for CI
 resource "google_service_account" "ci" {
   account_id   = var.ci_service_account_id
diff --git a/yarn-project/end-to-end/src/spartan/block_capacity.test.ts b/yarn-project/end-to-end/src/spartan/block_capacity.test.ts
@@ -67,8 +67,12 @@ describe('block capacity benchmark', () => {
   let aztecNode: AztecNode;
   let originalSequencerConfig: Awaited<ReturnType<typeof getSequencersConfig>> | undefined;
   const benchmarkData: Array<{ name: string; unit: string; value: number }> = [];
+  // Window handed to bench_scrape.ts so the custom pipeline can scrape this run's
+  // blocks/metrics (see spartan/bootstrap.sh block_capacity_bench).
+  let benchStartedAt: string | undefined;
 
   beforeAll(async () => {
+    benchStartedAt = new Date().toISOString();
     logger.info('Setting up block capacity benchmark', {
       numWallets: NUM_WALLETS,
       txRealProofs,
@@ -140,6 +144,19 @@ describe('block capacity benchmark', () => {
       logger.info('Wrote benchmark output', { path: process.env.BENCH_OUTPUT, entries: finalData.length });
     }
 
+    // Hand the run window to the custom-pipeline scraper (bench_scrape.ts), which
+    // reads this file to bound its Prometheus/log queries for the block-capacity run.
+    const timingMetadataPath = '/tmp/block_capacity_timing_data.json';
+    await writeFile(
+      timingMetadataPath,
+      JSON.stringify({
+        startedAt: benchStartedAt ?? new Date().toISOString(),
+        endedAt: new Date().toISOString(),
+        runId: process.env.BENCH_RUN_ID,
+      }),
+    );
+    logger.info('Wrote block-capacity timing metadata', { path: timingMetadataPath });
+
     // Restore original sequencer config
     if (originalSequencerConfig?.[0]) {
       logger.info('Restoring original sequencer config');
diff --git a/yarn-project/end-to-end/src/spartan/n_tps_prove.test.ts b/yarn-project/end-to-end/src/spartan/n_tps_prove.test.ts
@@ -133,6 +133,9 @@ describe(`prove ${TARGET_TPS}TPS test`, () => {
   let rollupCheatCodes: RollupCheatCodes;
   let ethEndpoint: ServiceEndpoint | undefined;
   let metricsStartSnapshot: MetricsSnapshot | undefined;
+  // Window handed to bench_scrape.ts so the custom pipeline can scrape this run's
+  // proving-infra series (see spartan/bootstrap.sh proving_bench).
+  let benchStartedAt: string | undefined;
 
   afterAll(async () => {
     if (process.env.BENCH_OUTPUT && metrics && metricsStartSnapshot) {
@@ -212,6 +215,19 @@ describe(`prove ${TARGET_TPS}TPS test`, () => {
       await writeFile(process.env.BENCH_OUTPUT, JSON.stringify(metrics.toGithubActionBenchmarkJSON()));
     }
 
+    // Hand the run window to the custom-pipeline scraper (bench_scrape.ts), which
+    // reads this file to bound its Prometheus queries for the proving run.
+    const timingMetadataPath = '/tmp/n_tps_prove_timing_data.json';
+    await writeFile(
+      timingMetadataPath,
+      JSON.stringify({
+        startedAt: benchStartedAt ?? new Date().toISOString(),
+        endedAt: new Date().toISOString(),
+        runId: process.env.BENCH_RUN_ID,
+      }),
+    );
+    logger.info('Wrote proving-bench timing metadata', { path: timingMetadataPath });
+
     if (testWallets) {
       for (const tw of testWallets) {
         await tw.cleanup();
@@ -247,6 +263,7 @@ describe(`prove ${TARGET_TPS}TPS test`, () => {
       server: new URL(`http://127.0.0.1:${promPortForward.port}`),
     });
     metricsStartSnapshot = await captureMetricsSnapshot(prometheusClient, logger);
+    benchStartedAt = new Date().toISOString();
     promPortForward.process.kill();
     logger.info('Metrics snapshot captured');
 
