From 3f53f04796eda02fa77df0503c28a4157b78d284 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Tue, 23 Jun 2026 21:12:49 +0000 Subject: [PATCH 01/39] feat(bench): schema v4 + scrape proving-infra and per-role saturation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 (A-1221) + Phase 2 (A-1222) of the nightly benchmarking revamp. Schema v4 (additive over v3, all v3 fields retained): - New optional sections provingInfra and saturation (metricSeriesMap: open slug -> timeSeries map), plus run.sweepId/sweepLabel so a night's 1/5/10 TPS points group as one sweep. - Version gate bumped to "4" in bench_output.schema.json and the network_bench_upload check in bootstrap.sh; index.json entry now carries sweepId/sweepLabel. (Dashboard SUPPORTED_RUN_VERSION lives in AztecProtocol/explorations and must be bumped there separately.) - 10tps-readiness-spec.md: the canonical spec — tx-lifecycle stage waterfall mapped to metrics, headline KPIs + thresholds, sweep/run-group notion. Scraper (bench_scrape.ts): - provingInfra: prover-node hint-gen (public_processor.* + prover_node block/checkpoint processing, scoped to the prover-node pod) and proving-queue series broken down by aztec_proving_job_type (size/active/job_duration/ timed-out/resolved). There is no prover_node.execution.duration metric; the re-execution is the public_processor.* path, mapped accordingly. - saturation: per-role ELU/CPU/memory as max (hottest pod) AND avg, never a single hand-picked pod. ELU/mem from nodejs.*, CPU from host-metrics. - Sections scrape independently so one failure cannot drop the others. CPU and ELU may be telemetry-gated in the bench env (empty series, non-fatal); flagged in the spec for live verification per A-1222 acceptance. --- spartan/bootstrap.sh | 6 +- .../bench_10tps/10tps-readiness-spec.md | 69 ++++++ .../bench_10tps/bench_output.schema.json | 222 +++++------------- spartan/scripts/bench_10tps/bench_scrape.ts | 182 +++++++++++++- 4 files changed, 311 insertions(+), 168 deletions(-) create mode 100644 spartan/scripts/bench_10tps/10tps-readiness-spec.md diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 4a4a19db9d69..fc4e3b322a51 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -286,8 +286,8 @@ function network_bench_upload { # Reject anything that's not the schema we've designed the index against. local schema=$(jq -r .schemaVersion "$run_json") - if [[ "$schema" != "3" ]]; then - echo "[network_bench] run JSON has schemaVersion '$schema', expected '3'; skipping upload" + if [[ "$schema" != "4" ]]; then + echo "[network_bench] run JSON has schemaVersion '$schema', expected '4'; skipping upload" return 0 fi @@ -304,6 +304,8 @@ function network_bench_upload { startedAt: .run.startedAt, endedAt: .run.endedAt, targetTps: .run.targetTps, + sweepId: .run.sweepId, + sweepLabel: .run.sweepLabel, workload: .run.workload, testDurationSeconds: .run.testDurationSeconds, namespace: .run.namespace, diff --git a/spartan/scripts/bench_10tps/10tps-readiness-spec.md b/spartan/scripts/bench_10tps/10tps-readiness-spec.md new file mode 100644 index 000000000000..60e620f09d58 --- /dev/null +++ b/spartan/scripts/bench_10tps/10tps-readiness-spec.md @@ -0,0 +1,69 @@ +# 10 TPS readiness benchmark — spec (schema v4) + +Canonical contract for the custom benchmark pipeline: +`bench_scrape.ts` → `gs://aztec-testnet/network_bench/.json` (+ `index.json`) → `network-dashboard` (in `AztecProtocol/explorations`). + +This doc is the Phase 1 deliverable (Linear A-1221). It defines the tx-lifecycle stage list, the headline KPIs and their thresholds, and the sweep/run-group notion. The machine-readable contract is `bench_output.schema.json` (v4); the scraper that produces it is `bench_scrape.ts`. + +## 1. tx-lifecycle stage waterfall + +A tx's journey from client submit to epoch proof, each stage mapped to the Prometheus metric (from `yarn-project/telemetry-client/src/metrics.ts`) and where it lands in the run JSON. "ms" durations are histograms (use `_bucket` for quantiles, `_sum`/`_count` for means); never `sum(rate(...))` a metric every node observes — collapse per role with `avg`/`max` (see `network-dashboard/docs/dashboard-design.md`). + +| # | Stage | Primary metric(s) | Run-JSON location | +|---|---|---|---| +| 1 | Submit / ingest | `aztec.node.receive_tx_count` (RPC only — load hits one node) | `timeSeries.ingressTps` | +| 2 | P2P propagation | `aztec.p2p.gossip.message_latency`, `agg_message_latency_p50/p90`, `tx_received_count` | `timeSeries` (gossip) | +| 3 | Mempool wait | `aztec.mempool.tx_count` (pending depth), `aztec.mempool.tx_mined_delay` | `timeSeries.mempoolSize*`, `mempoolMinedMax` | +| 4 | Block build | `aztec.sequencer.block.build_duration`, `build_mana_per_second` | `timeSeries`, `sequencerStateSlots` | +| 5 | Public processing | `aztec.public_processor.tx_duration`, `phase_duration`, `gas_rate` | `timeSeries.publicProcessorGasRate`; **prover-node copy** in `provingInfra.hintGen*` | +| 6 | Attestation / consensus | `aztec.mempool.attestations_mined_delay`; attestation-collect duration vs slot allowance | `timeSeries.attestationsCollect*` | +| 7 | Checkpoint assemble → broadcast | `aztec.archiver.checkpoint_height`, checkpoint block/tx counts | `timeSeries.checkpoint*` | +| 8 | L1 inclusion | `aztec.archiver.checkpoint_l1_inclusion_delay`, `l1_block_height`, `block_height` | `timeSeries`, `blocks` | +| 9 | Proving (epoch) | `aztec.prover_node.checkpoint_proving.duration`, `aztec.archiver.rollup_proof_delay`, `aztec.proving_queue.*` by `job_type`, prover-node block/checkpoint processing | `provingInfra.*` | + +**Authoritative user-perceived latency** is client-observed, not Prometheus: `n_tps_test.tx_inclusion_time` (`timeSeries`, `source: client_observed`) — the wall-clock submit→mined delta for high-value txs, computed in the scraper from `n_tps.test.ts` records. Stages 1–8 explain *where* that latency is spent; stage 9 is the separate proving path. + +## 2. Headline KPIs + pass/fail thresholds + +Two independent verdicts — a run can pass inclusion and fail proving (or vice versa). Thresholds are starting points to refine against baselines; encode them in the dashboard, not the scraper (the scraper stays a faithful recorder). + +| KPI | Definition | Pass threshold | +|---|---|---| +| **Inclusion-TPS achieved / target** | `summary.inclusionTpsMean / run.targetTps` | ≥ 0.95 | +| **User-perceived inclusion latency p50** | p50 of `tx_inclusion_time` | ≤ 1 × `AZTEC_SLOT_DURATION` | +| **User-perceived inclusion latency p99** | p99 of `tx_inclusion_time` | ≤ 3 × `AZTEC_SLOT_DURATION` | +| **Proving headroom** | does each epoch's proof land within `AZTEC_PROOF_SUBMISSION_EPOCHS` of epoch close (no proof-window-expiry reorg)? | every epoch proven in window; `reorgCount` from window-expiry = 0 | +| **Reorgs** | `summary.reorgCount` | 0 | + +"Proving headroom" is the proving knee = distance from 10 TPS: the highest `targetTps` at which every epoch still proves within its window. Below the knee, headroom is positive; above it, epochs miss the window and the pending chain is pruned (the run #95 failure mode). + +## 3. Sweep / run-group notion + +A night's 1/5/10 TPS points are distinct runs (distinct namespaces — queries are namespace-scoped, one run per namespace) that the dashboard must view together. Grouping fields (schema v4, on `run` + mirrored into `index.json`): + +- `run.sweepId` — shared id across the points of one sweep (e.g. `incl-20260623`). Set via `--sweep-id` / `BENCH_SWEEP_ID`. +- `run.sweepLabel` — human label (e.g. `inclusion-sweep`, `proving-sweep`). `--sweep-label` / `BENCH_SWEEP_LABEL`. +- `run.targetTps` — the point within the sweep (already present in v3). + +`index.json` entries carry `sweepId`/`sweepLabel`/`targetTps` so the dashboard can group + order points without fetching every run JSON. + +## 4. schema v4 additions (additive over v3) + +All v3 fields retained; a v3-shaped run re-stamped `"4"` still validates (the new sections are optional). New: + +- `provingInfra` (`metricSeriesMap`): prover-node hint-gen (`public_processor.*` + `prover_node.*_processing.duration` scoped to the prover-node pod) and proving-queue series broken down by `aztec_proving_job_type` (size / active / job_duration p50·p99 / timed-out · resolved rates). **Note:** there is no `aztec.prover_node.execution.duration` metric — hint-gen is the `public_processor.*` re-execution on the prover-node pod, mapped accordingly. +- `saturation` (`metricSeriesMap`): per-role ELU / CPU / memory, each as **max (hottest pod)** and **avg**, for validator / rpc / fullNode / proverNode / broker / agent. Never a single hand-picked pod. ELU = `nodejs_eventloop_utilization`, memory = `nodejs_memory_v8_heap_usage` (both `nodejs.*`, not `aztec_`); CPU = `process_cpu_utilization` (from `@opentelemetry/host-metrics`). +- `run.sweepId` / `run.sweepLabel` (§3). + +### Version gate — three places, must stay in sync + +Bumping the schema version requires updating all three or v4 runs are silently rejected / mis-rendered: + +1. `bench_output.schema.json` — `schemaVersion.const` (✅ `"4"`). +2. `spartan/bootstrap.sh` — `network_bench_upload` schemaVersion check (✅ `"4"`). +3. **`network-dashboard/data.js` `SUPPORTED_RUN_VERSION`** — in `AztecProtocol/explorations`, **not this repo**. Must be bumped to `"4"` there before v4 runs render. Tracked as dashboard work (Phase 5). + +## 5. Verify-on-live caveats (A-1222 acceptance) + +- **CPU** (`process_cpu_utilization`) and **ELU** (`nodejs_eventloop_utilization`) come from telemetry that may be gated in the bench env. The scraper emits empty series (non-fatal) if a metric is absent; confirm both flow on a live bench run and fix the exporter/metric name if not. +- Proving-infra durations are recorded in **ms** by convention; confirm units against the live histograms before trusting absolute values. diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 3685c72960d8..ca1fd691187e 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -16,8 +16,8 @@ "properties": { "schemaVersion": { "type": "string", - "const": "3", - "description": "Bump when breaking the schema. Old JSONs keep their previous version so the dashboard can render them side-by-side. v3: timeSeries entries carry `series: [{labels, points}]` instead of bare `points` to support per-pod / per-label data." + "const": "4", + "description": "Bump when breaking the schema. Old JSONs keep their previous version so the dashboard can render them side-by-side. v3: timeSeries entries carry `series: [{labels, points}]` instead of bare `points`. v4 (additive): adds optional `provingInfra` (hint-gen + proving-queue-by-job_type series) and `saturation` (per-role ELU/CPU/memory, max + avg) sections, plus `run.sweepId`/`run.sweepLabel` so a night's 1/5/10 TPS points group as one sweep. All v3 fields retained; a v3 run is unchanged apart from this version string." }, "run": { "$ref": "#/$defs/runMeta" @@ -30,6 +30,14 @@ "$ref": "#/$defs/timeSeriesSection", "description": "PromQL query_range results. Continuous-sampled metrics keyed by unixEpoch; the dashboard normalises to time-within-run via unixEpoch - run.startedAt at render time so multiple runs can overlay on the same x-axis." }, + "provingInfra": { + "$ref": "#/$defs/metricSeriesMap", + "description": "v4. Proving-path series for the proving-infra view: prover-node hint-gen / tx re-execution (public_processor.* + prover_node.*_processing.duration scoped to the prover-node pod) and proving-queue behaviour broken down by job_type (aztec_proving_job_type label). Optional — empty/absent on inclusion-only runs." + }, + "saturation": { + "$ref": "#/$defs/metricSeriesMap", + "description": "v4. Per-role resource saturation: ELU, CPU, and memory for each role (validator / rpc / fullNode / proverNode / broker / agent), each emitted as both the hottest pod (max across pods) and the role average (avg across pods). Never a single hand-picked pod. Optional — absent on older runs." + }, "blocks": { "type": "array", "description": "Per-block records parsed from structured logs (each block emits one `Processed N successful txs and M failed txs ...` info line). Authoritative for per-block facts — Prometheus histograms cannot recover per-block samples.", @@ -63,12 +71,7 @@ "runMeta": { "type": "object", "additionalProperties": false, - "required": [ - "runId", - "startedAt", - "endedAt", - "namespace" - ], + "required": ["runId", "startedAt", "endedAt", "namespace"], "properties": { "runId": { "type": "string", @@ -96,9 +99,7 @@ }, "namespace": { "type": "string", - "examples": [ - "bench-10tps" - ] + "examples": ["bench-10tps"] }, "gcpProject": { "type": "string", @@ -120,15 +121,21 @@ "type": "number", "minimum": 0 }, + "sweepId": { + "type": "string", + "description": "v4. Groups the runs of one sweep (e.g. a night's 1/5/10 TPS points) so the dashboard can plot them together. Shared across the points of a sweep; absent for standalone runs." + }, + "sweepLabel": { + "type": "string", + "description": "v4. Human-readable label for the sweep this run belongs to (e.g. 'inclusion-sweep' or 'proving-sweep'). Optional." + }, "testDurationSeconds": { "type": "integer", "minimum": 0 }, "workload": { "type": "string", - "examples": [ - "sha256_hash_1024" - ] + "examples": ["sha256_hash_1024"] }, "aztecConfig": { "type": "object", @@ -167,11 +174,7 @@ "items": { "type": "object", "additionalProperties": false, - "required": [ - "role", - "podName", - "nodeName" - ], + "required": ["role", "podName", "nodeName"], "properties": { "role": { "type": "string" @@ -205,9 +208,7 @@ }, "profile": { "type": "string", - "examples": [ - "network-requirements" - ] + "examples": ["network-requirements"] } } }, @@ -239,40 +240,25 @@ "description": "Maximum time the scraper was allowed to wait for validator pending TxPool depth to reach zero." }, "pendingAtScrape": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "minimum": 0, "description": "Validator pending TxPool depth observed when scraping started, or null when the pending drain gate was disabled." }, "pendingByRoleAtScrape": { - "type": [ - "object", - "null" - ], + "type": ["object", "null"], "description": "Pending TxPool depth by pod role at scrape start. RPC/full-node pending can remain non-zero after validators drain, which indicates load that did not propagate to proposers before expiry.", "additionalProperties": false, "properties": { "rpc": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "minimum": 0 }, "validator": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "minimum": 0 }, "fullNode": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "minimum": 0 } } @@ -288,118 +274,66 @@ "summary": { "type": "object", "additionalProperties": false, - "required": [ - "headlineKpi", - "inclusionTpsMean", - "targetTps" - ], + "required": ["headlineKpi", "inclusionTpsMean", "targetTps"], "properties": { "headlineKpi": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "description": "inclusionTpsMean / targetTps. The single number on the dashboard top strip." }, "targetTps": { "type": "number" }, "inclusionTpsMean": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "description": "Inclusion throughput over the observed inclusion window. Uses exact block-log throughput when block records are available, otherwise falls back to the Prometheus inclusionTps mean." }, "inclusionTpsPeak": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "description": "Peak sampled Prometheus rolling inclusion rate over the observed scrape window." }, "inclusionLatencyP50Ms": { - "type": [ - "number", - "null" - ] + "type": ["number", "null"] }, "inclusionLatencyP95Ms": { - "type": [ - "number", - "null" - ] + "type": ["number", "null"] }, "inclusionLatencyP99Ms": { - "type": [ - "number", - "null" - ] + "type": ["number", "null"] }, "blockBuildDurationP50Ms": { - "type": [ - "number", - "null" - ] + "type": ["number", "null"] }, "blockBuildDurationP95Ms": { - "type": [ - "number", - "null" - ] + "type": ["number", "null"] }, "publicProcessorTxDurationP50Ms": { - "type": [ - "number", - "null" - ] + "type": ["number", "null"] }, "publicProcessorTxDurationP95Ms": { - "type": [ - "number", - "null" - ] + "type": ["number", "null"] }, "totalTxsMined": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "description": "Exact sum from per-block logs. Null when block logs were unavailable and inclusionTpsMean came from Prometheus." }, "totalTxsFailed": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "description": "Exact sum from per-block logs. Null when block logs were unavailable." }, "totalSilentSkipCount": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "description": "Sum of per-block silentlySkippedCount. > 0 means the post-process blob-field revert path fired during the run." }, "totalSilentSkipDurationMs": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "description": "Sum of per-block silentlySkippedDurationMs. Wall-clock 'wasted' on silently-skipped txs across the run." }, "reorgCount": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "description": "Count of `Chain pruned` events during the run." }, "deepestReorgBlocks": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "description": "Max (fromBlock - toBlock) across reorg events. 0 if no reorgs." } } @@ -510,14 +444,17 @@ } } }, + "metricSeriesMap": { + "type": "object", + "description": "v4. Open map of slug -> timeSeries, same per-series shape as timeSeriesSection but without a fixed slug list. Used for provingInfra and saturation, whose slugs are generated per role / job_type.", + "additionalProperties": { + "$ref": "#/$defs/timeSeries" + } + }, "timeSeries": { "type": "object", "additionalProperties": false, - "required": [ - "metric", - "source", - "series" - ], + "required": ["metric", "source", "series"], "properties": { "metric": { "type": "string", @@ -525,19 +462,11 @@ }, "unit": { "type": "string", - "examples": [ - "ms", - "tps", - "mana/s", - "count" - ] + "examples": ["ms", "tps", "mana/s", "count"] }, "source": { "type": "string", - "enum": [ - "promql", - "client_observed" - ], + "enum": ["promql", "client_observed"], "description": "Provenance: 'promql' = scraped via PromQL from cluster Prometheus; 'client_observed' = computed in this scraper from per-tx records emitted by n_tps.test.ts (e.g. headline tx_mined_delay)." }, "query": { @@ -560,10 +489,7 @@ "seriesEntry": { "type": "object", "additionalProperties": false, - "required": [ - "labels", - "points" - ], + "required": ["labels", "points"], "properties": { "labels": { "type": "object", @@ -584,20 +510,14 @@ "tsPoint": { "type": "object", "additionalProperties": false, - "required": [ - "unixEpoch", - "value" - ], + "required": ["unixEpoch", "value"], "properties": { "unixEpoch": { "type": "integer", "description": "Seconds since unix epoch for this sample. Dashboards normalise to time-within-run via unixEpoch - run.startedAt at render time." }, "value": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "description": "Metric value. null if Prom returned NaN / no data for this step." } } @@ -605,11 +525,7 @@ "blockRecord": { "type": "object", "additionalProperties": false, - "required": [ - "blockNumber", - "blockNumberInTest", - "minedAt" - ], + "required": ["blockNumber", "blockNumberInTest", "minedAt"], "properties": { "blockNumber": { "type": "integer", @@ -671,10 +587,7 @@ "event": { "type": "object", "additionalProperties": false, - "required": [ - "at", - "type" - ], + "required": ["at", "type"], "properties": { "at": { "type": "string", @@ -682,10 +595,7 @@ }, "type": { "type": "string", - "enum": [ - "chainPruned", - "slotSummary" - ] + "enum": ["chainPruned", "slotSummary"] }, "source": { "type": "string", @@ -761,13 +671,7 @@ "sequencerStateSlot": { "type": "object", "additionalProperties": false, - "required": [ - "slotNumber", - "startedAt", - "endedAt", - "totalMs", - "states" - ], + "required": ["slotNumber", "startedAt", "endedAt", "totalMs", "states"], "properties": { "slotNumber": { "type": "integer", @@ -803,4 +707,4 @@ } } } -} \ No newline at end of file +} diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index 77d1805d2d53..5197aa32e4d6 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -1,9 +1,15 @@ #!/usr/bin/env -S node --experimental-strip-types --no-warnings // // Scrape a completed bench-10tps run into a schema-conformant JSON payload. -// Contract: bench_output.schema.json (v3). Invoked by the bench_10tps function +// Contract: bench_output.schema.json (v4). Invoked by the bench_10tps function // in spartan/bootstrap.sh after n_tps.test.ts finishes. // +// v4 adds two PromQL sections alongside the inclusion timeSeries: +// - provingInfra: prover-node hint-gen (tx re-execution) + proving-queue +// behaviour broken down by job_type. +// - saturation: per-role ELU/CPU/memory, each as max (hottest pod) + avg. +// Both scrape independently so one failing does not abort the others. +// // Two independent scrape paths so one failing does not abort the other: // 1. Prometheus (port-forward to the cluster-shared metrics-prometheus-server) // 2. gcloud logging read (per-block and discrete-event records) @@ -56,6 +62,8 @@ type Args = { inclusionRecords: string | undefined; waitForPendingZero: boolean; maxPendingWaitSeconds: number; + sweepId: string | undefined; + sweepLabel: string | undefined; }; function parseArgs(): Args { @@ -94,6 +102,8 @@ function parseArgs(): Args { String(DEFAULT_MAX_PENDING_WAIT_SECONDS), ), ), + sweepId: get("--sweep-id", env.BENCH_SWEEP_ID ?? "") || undefined, + sweepLabel: get("--sweep-label", env.BENCH_SWEEP_LABEL ?? "") || undefined, }; } @@ -560,12 +570,145 @@ const TIME_SERIES_DEFS: Record = { }, }; -async function scrapeTimeSeries( +// --- v4: per-role resource saturation (ELU / CPU / memory) --- +// Roles are matched by pod-name prefix within the namespace. The proposer +// rotates, so never hand-pick a pod: emit max() (hottest pod) AND avg() per role. +const SATURATION_ROLES: Record = { + validator: `${NAMESPACE}-validator.*`, + rpc: `${NAMESPACE}-rpc.*`, + fullNode: `${NAMESPACE}-full-node.*`, + proverNode: `${NAMESPACE}-prover-node.*`, + broker: `${NAMESPACE}-prover-broker.*`, + agent: `${NAMESPACE}-prover-agent.*`, +}; + +// OTel metric -> Prometheus name. ELU + heap come from +// telemetry-client/src/nodejs_metrics_monitor.ts (nodejs.* prefix, NOT aztec_). +// CPU comes from @opentelemetry/host-metrics (process.cpu.utilization), not the +// nodejs monitor. NOTE: ELU and especially CPU may be telemetry-gated in the +// bench env — if so these series come back empty (A-1222 acceptance: verify on +// the live env and adjust the metric name / enable the exporter as needed). +const SATURATION_METRICS: { key: string; metric: string; unit: string }[] = [ + { key: "elu", metric: "nodejs_eventloop_utilization", unit: "ratio" }, + { key: "cpu", metric: "process_cpu_utilization", unit: "ratio" }, + { key: "mem", metric: "nodejs_memory_v8_heap_usage", unit: "bytes" }, +]; + +function buildSaturationDefs(): Record { + const defs: Record = {}; + for (const [role, podPattern] of Object.entries(SATURATION_ROLES)) { + const sel = `{k8s_namespace_name="${NAMESPACE}",k8s_pod_name=~"${podPattern}"}`; + const cap = role.charAt(0).toUpperCase() + role.slice(1); + for (const { key, metric, unit } of SATURATION_METRICS) { + // max() across pods = hottest pod; avg() = role average. Single series each. + defs[`${key}${cap}Max`] = { metric, unit, query: `max(${metric}${sel})` }; + defs[`${key}${cap}Avg`] = { metric, unit, query: `avg(${metric}${sel})` }; + } + } + return defs; +} +const SATURATION_DEFS = buildSaturationDefs(); + +// --- v4: proving-infra (hint-gen on the prover-node + proving-queue by job_type) --- +// "Hint generation" is the prover node re-executing the epoch's txs. There is no +// `aztec.prover_node.execution.duration` metric; the re-execution is instrumented +// as public_processor.* + prover_node.*_processing.duration on the prover-node +// pod. Proving-queue behaviour is broken down by the aztec_proving_job_type label. +const PROVER_NODE_SEL = `{k8s_namespace_name="${NAMESPACE}",k8s_pod_name=~"${NAMESPACE}-prover-node.*"}`; +const JOB_TYPE = "aztec_proving_job_type"; +const proverNodeHist = (q: number, bucket: string) => + `histogram_quantile(${q}, sum by (le)(rate(${bucket}${PROVER_NODE_SEL}[1m])))`; +const queueByJobType = (metric: string) => + `sum by (${JOB_TYPE})(${metric}${NS})`; +const queueRateByJobType = (metric: string) => + `sum by (${JOB_TYPE})(rate(${metric}${NS}[1m]))`; +const queueHistByJobType = (q: number, bucket: string) => + `histogram_quantile(${q}, sum by (le, ${JOB_TYPE})(rate(${bucket}${NS}[1m])))`; + +const PROVING_INFRA_DEFS: Record = { + // Hint-gen: prover-node tx re-execution (the proving bottleneck at high TPS). + hintGenPublicTxDurationP50: { + metric: "aztec_public_processor_tx_duration", + unit: "ms", + query: proverNodeHist(0.5, "aztec_public_processor_tx_duration_bucket"), + }, + hintGenPublicTxDurationP99: { + metric: "aztec_public_processor_tx_duration", + unit: "ms", + query: proverNodeHist(0.99, "aztec_public_processor_tx_duration_bucket"), + }, + hintGenPublicPhaseDurationP50: { + metric: "aztec_public_processor_phase_duration", + unit: "ms", + query: proverNodeHist(0.5, "aztec_public_processor_phase_duration_bucket"), + }, + hintGenBlockProcessingDurationP50: { + metric: "aztec_prover_node_block_processing_duration", + unit: "ms", + query: proverNodeHist( + 0.5, + "aztec_prover_node_block_processing_duration_bucket", + ), + }, + hintGenBlockProcessingDurationP99: { + metric: "aztec_prover_node_block_processing_duration", + unit: "ms", + query: proverNodeHist( + 0.99, + "aztec_prover_node_block_processing_duration_bucket", + ), + }, + hintGenCheckpointProcessingDurationP50: { + metric: "aztec_prover_node_checkpoint_processing_duration", + unit: "ms", + query: proverNodeHist( + 0.5, + "aztec_prover_node_checkpoint_processing_duration_bucket", + ), + }, + // Proving queue, broken down by job_type (one series per job type). + provingQueueSizeByJobType: { + metric: "aztec_proving_queue_size", + unit: "count", + query: queueByJobType("aztec_proving_queue_size"), + }, + provingQueueActiveJobsByJobType: { + metric: "aztec_proving_queue_active_jobs_count", + unit: "count", + query: queueByJobType("aztec_proving_queue_active_jobs_count"), + }, + provingQueueJobDurationP50ByJobType: { + metric: "aztec_proving_queue_job_duration", + unit: "ms", + query: queueHistByJobType(0.5, "aztec_proving_queue_job_duration_bucket"), + }, + provingQueueJobDurationP99ByJobType: { + metric: "aztec_proving_queue_job_duration", + unit: "ms", + query: queueHistByJobType(0.99, "aztec_proving_queue_job_duration_bucket"), + }, + // Rates of terminal job outcomes — the run #95 stall showed up as timeouts. + provingQueueTimedOutJobsByJobType: { + metric: "aztec_proving_queue_timed_out_jobs_count", + unit: "count", + query: queueRateByJobType("aztec_proving_queue_timed_out_jobs_count"), + }, + provingQueueResolvedJobsByJobType: { + metric: "aztec_proving_queue_resolved_jobs_count", + unit: "count", + query: queueRateByJobType("aztec_proving_queue_resolved_jobs_count"), + }, +}; + +// Scrape a map of slug -> PromQL def via query_range. One failing query emits an +// empty series for that slug rather than aborting the whole section. +async function scrapeDefs( + defs: Record, startedAtEpoch: number, endedAtEpoch: number, ): Promise> { const out: Record = {}; - for (const [slug, def] of Object.entries(TIME_SERIES_DEFS)) { + for (const [slug, def] of Object.entries(defs)) { try { const series = await queryRange(def.query, startedAtEpoch, endedAtEpoch); out[slug] = { @@ -577,7 +720,7 @@ async function scrapeTimeSeries( series, }; } catch (err) { - log(`timeSeries.${slug} scrape failed, emitting empty series`, { + log(`series.${slug} scrape failed, emitting empty series`, { err: err instanceof Error ? err.message : String(err), }); out[slug] = { @@ -593,6 +736,9 @@ async function scrapeTimeSeries( return out; } +const scrapeTimeSeries = (startedAtEpoch: number, endedAtEpoch: number) => + scrapeDefs(TIME_SERIES_DEFS, startedAtEpoch, endedAtEpoch); + // --- gcloud log scrape --- type GcloudEntry = { @@ -1668,6 +1814,8 @@ function assertShape(payload: Record): void { "run", "summary", "timeSeries", + "provingInfra", + "saturation", "blocks", "events", ] as const; @@ -1676,9 +1824,9 @@ function assertShape(payload: Record): void { throw new Error(`output missing required top-level key: ${key}`); } } - if (payload.schemaVersion !== "3") { + if (payload.schemaVersion !== "4") { throw new Error( - `schemaVersion must be "3", got ${String(payload.schemaVersion)}`, + `schemaVersion must be "4", got ${String(payload.schemaVersion)}`, ); } const run = payload.run as Record; @@ -1913,6 +2061,22 @@ async function main(): Promise { log("Scraping Prometheus time-series"); const timeSeries = await scrapeTimeSeries(startedAtEpoch, promEndEpoch); + // v4: proving-infra (hint-gen + queue by job_type) and per-role saturation. + // Independent of the inclusion timeSeries scrape so a failure here cannot + // drop inclusion data, and vice versa. + log("Scraping proving-infra series (hint-gen + queue by job_type)"); + const provingInfra = await scrapeDefs( + PROVING_INFRA_DEFS, + startedAtEpoch, + promEndEpoch, + ); + log("Scraping per-role saturation series (ELU/CPU/memory, max + avg)"); + const saturation = await scrapeDefs( + SATURATION_DEFS, + startedAtEpoch, + promEndEpoch, + ); + log("Loading client-observed inclusion records"); const inclusionRecords = await loadInclusionRecords(args.inclusionRecords); // Compute the headline inclusion-latency time series from per-tx records @@ -1999,7 +2163,7 @@ async function main(): Promise { }); const payload = { - schemaVersion: "3", + schemaVersion: "4", run: { runId: args.runId, startedAt: args.startedAt, @@ -2014,6 +2178,8 @@ async function main(): Promise { gkeCluster: GKE_CLUSTER, ...(image !== undefined && { image }), targetTps: args.targetTps, + ...(args.sweepId !== undefined && { sweepId: args.sweepId }), + ...(args.sweepLabel !== undefined && { sweepLabel: args.sweepLabel }), testDurationSeconds: windowSec, workload: args.workload, ...(Object.keys(aztecConfig).length > 0 && { aztecConfig }), @@ -2031,6 +2197,8 @@ async function main(): Promise { }, summary, timeSeries, + provingInfra, + saturation, blocks, events, sequencerStateSlots, From f8644d849f270a6f91e8b1badb90fb40711f2fd7 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Tue, 23 Jun 2026 21:58:28 +0000 Subject: [PATCH 02/39] feat(bench): Set A inclusion-sweep env + spot full-node profile + bench point Phase 3 (A-1223), infra + runner. The CI workflow that fans out the 1/5/10 sweep is intentionally not included yet (concurrency model is an open decision). - bench-inclusion-sweep.env: bench-10tps's custom scrape pipeline + the tps-scenario topology trimmed to 150 full nodes (12x4 validators, 10 RPC), local eth-devnet, fake (fixed-delay) proving so proving never bottlenecks inclusion, P2P instrumentation on. Target TPS / sweep id are passed per point. - full-node-resources-spot.yaml: full-node profile that *requires* spot (nodeSelector pool:spot + required gke-spot affinity + toleration), copying the prover-resources-dev.yaml pattern. Validators/RPC/prover stay on-demand. - spartan/bootstrap.sh bench_inclusion_point: one sweep point at TARGET_TPS in its own namespace, all high-value load, scraped with a shared BENCH_SWEEP_ID (schema v4) via the existing bench_scrape path; reuses network_bench_upload. Usable for local runs now: TARGET_TPS=5 BENCH_SWEEP_ID=incl- \ spartan/bootstrap.sh bench_inclusion_point bench-inclusion-sweep --- spartan/.gitignore | 1 + spartan/bootstrap.sh | 55 +++++++++- .../environments/bench-inclusion-sweep.env | 100 ++++++++++++++++++ .../values/full-node-resources-spot.yaml | 57 ++++++++++ 4 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 spartan/environments/bench-inclusion-sweep.env create mode 100644 spartan/terraform/deploy-aztec-infra/values/full-node-resources-spot.yaml diff --git a/spartan/.gitignore b/spartan/.gitignore index da7cd59acdb2..0e2a2e78ade5 100644 --- a/spartan/.gitignore +++ b/spartan/.gitignore @@ -35,5 +35,6 @@ environments/* !environments/alpha-net.env !environments/mbps-pipeline.env !environments/bench-10tps.env +!environments/bench-inclusion-sweep.env *.tfvars !terraform/deploy-external-secrets/*.tfvars diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index fc4e3b322a51..1567830119d4 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -273,6 +273,59 @@ function bench_10tps { fi } +# One point of the Set A inclusion sweep (A-1223). Same scrape+upload path as +# bench_10tps, but parameterized by TARGET_TPS and tagged with a shared +# BENCH_SWEEP_ID so the 1/5/10 points group as one sweep (schema v4). Load is +# all high-value at TARGET_TPS so the headline client-observed inclusion latency +# reflects the full target rate. Each point runs in its own namespace. +function bench_inclusion_point_cmds { + local tps=${TARGET_TPS:-10} + local test_duration=${TEST_DURATION_SECONDS:-600} # 10 mins + local timeout=${BENCH_TIMEOUT_SECONDS:-7200} # account for committee formation + local scenario="incl_${tps/./_}tps" + echo "$(hash):TIMEOUT=${timeout} BENCH_RUN_ID=${BENCH_RUN_ID:-} BENCH_OUTPUT=bench-out/n_tps.${scenario}.bench.json BENCH_SCENARIO=${scenario} LOW_VALUE_TPS=0 HIGH_VALUE_TPS=${tps} TEST_DURATION_SECONDS=${test_duration} $root/yarn-project/end-to-end/scripts/run_test.sh simple n_tps.test.ts" +} + +function bench_inclusion_point { + rm -rf bench-out + mkdir -p bench-out + + local env_file="$1" + source_network_env $env_file + + local tps=${TARGET_TPS:-10} + echo_header "spartan inclusion-sweep point (${tps} TPS)" + gcp_auth + export_admin_api_key + export K8S_ENRICHER=${K8S_ENRICHER:-1} + export BENCH_RUN_ID="${BENCH_RUN_ID:-$(date -u +%Y%m%d)-incl-${tps}tps-${COMMIT_HASH:0:10}}" + bench_inclusion_point_cmds | parallelize 1 + + local metadata="/tmp/n_tps_timing_data.json" + local run_json="bench-out/bench-inclusion-${tps}tps-${BENCH_RUN_ID}.json" + if [[ -f "$metadata" ]]; then + local started=$(jq -r .startedAt < "$metadata") + local ended=$(jq -r .endedAt < "$metadata") + echo "Scraping inclusion-sweep point ${tps} TPS (run ${BENCH_RUN_ID}, started=${started} ended=${ended})" + NAMESPACE="$NAMESPACE" GCP_PROJECT_ID="${GCP_PROJECT_ID:-}" ./scripts/bench_10tps/bench_scrape.ts \ + --run-id "$BENCH_RUN_ID" \ + --started "$started" \ + --ended "$ended" \ + --target-tps "$tps" \ + --sweep-id "${BENCH_SWEEP_ID:-}" \ + --sweep-label "${BENCH_SWEEP_LABEL:-inclusion-sweep}" \ + --workload sha256_hash_1024 \ + --output "$run_json" \ + --inclusion-records "$metadata" \ + --wait-for-pending-zero \ + --max-pending-wait-seconds "${BENCH_SCRAPE_MAX_PENDING_WAIT_SECONDS:-3600}" \ + || echo "[bench_inclusion_point] scraper failed (non-fatal)" + network_bench_upload "$run_json" || echo "[network_bench] upload failed (non-fatal)" + else + echo "[bench_inclusion_point] no timing metadata at ${metadata}; skipping scraper" + fi +} + function network_bench_upload { local run_json=$1 if [[ "${CI:-0}" != "1" ]]; then @@ -417,7 +470,7 @@ case "$cmd" in run_network_tests "$1" "$2" ;; - network_tests|network_tests_1|network_tests_2|network_bench|proving_bench|block_capacity_bench|bench_10tps) + network_tests|network_tests_1|network_tests_2|network_bench|proving_bench|block_capacity_bench|bench_10tps|bench_inclusion_point) env_file="$1" $cmd "$env_file" ;; diff --git a/spartan/environments/bench-inclusion-sweep.env b/spartan/environments/bench-inclusion-sweep.env new file mode 100644 index 000000000000..d5e1e84f31e7 --- /dev/null +++ b/spartan/environments/bench-inclusion-sweep.env @@ -0,0 +1,100 @@ +# Set A — inclusion sweep (Linear A-1223). +# One point of the 1/5/10 TPS inclusion sweep: bench-10tps's custom scrape +# pipeline + a production-scale topology (from tps-scenario), on a local +# eth-devnet with fake proving so proving never bottlenecks inclusion. +# +# The target TPS and sweep grouping are NOT set here — they are passed per point +# by the workflow via TARGET_TPS / BENCH_SWEEP_ID / BENCH_SWEEP_LABEL, and the +# namespace is overridden per point (one run per namespace, per the scraper). +NAMESPACE=${NAMESPACE:-bench-inclusion-sweep} +CLUSTER=aztec-gke-private +RESOURCE_PROFILE=prod +GCP_REGION=us-west1-a +DESTROY_NAMESPACE=true +DESTROY_AZTEC_INFRA=true + +CREATE_ETH_DEVNET=true +DESTROY_ETH_DEVNET=true +ETHEREUM_CHAIN_ID=1337 +LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" +FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +CREATE_ROLLUP_CONTRACTS=true +VERIFY_CONTRACTS=false + +# Production-like epoch timing — do NOT shorten (keeps load realistic). +AZTEC_EPOCH_DURATION=32 +AZTEC_SLOT_DURATION=72 +AZTEC_PROOF_SUBMISSION_EPOCHS=2 +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 +AZTEC_INBOX_LAG=2 + +# 2B mana target - good for about ~800 txs at 2.5M mana each +AZTEC_MANA_TARGET=2000000000 + +SPONSORED_FPC=true + +OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET + +# Large topology from tps-scenario.env: 12x4 = 48 validators, 10 RPC, 150 full +# nodes. Full nodes go on spot (full-node-resources-spot.yaml); validators / RPC +# / prover stay on-demand so inclusion is not corrupted by mid-block eviction. +VALIDATOR_REPLICAS=12 +VALIDATORS_PER_NODE=4 +VALIDATOR_PUBLISHERS_PER_REPLICA=4 +VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 +VALIDATOR_RESOURCE_PROFILE="prod" +VALIDATOR_HA_REPLICAS=0 + +SEQ_BLOCK_DURATION_MS=6000 +SEQ_L1_PUBLISHING_TIME_ALLOWANCE_IN_SLOT=36 +SEQ_MAX_TX_PER_BLOCK=200 +SEQ_MAX_TX_PER_CHECKPOINT=800 +P2P_MAX_PENDING_TX_COUNT=20000 +SEQ_MIN_TX_PER_BLOCK=1 +SEQ_BUILD_CHECKPOINT_IF_EMPTY=true + +RPC_REPLICAS=10 +RPC_RESOURCE_PROFILE="prod" +RPC_INGRESS_ENABLED=false + +FULL_NODE_REPLICAS=150 +FULL_NODE_RESOURCE_PROFILE="spot" + +# Fake proving (fixed, cheap delay) — this run does not measure proving; it only +# must not bottleneck inclusion (proving sweep is Set B / A-1224). +REAL_VERIFIER=false +PROVER_RESOURCE_PROFILE="dev-hi-tps" +PUBLISHERS_PER_PROVER=1 +PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 +PROVER_AGENT_POLL_INTERVAL_MS=10000 +PROVER_TEST_DELAY_TYPE=fixed +PROVER_TEST_VERIFICATION_DELAY_MS=250 +PROVER_AGENT_KEDA_ENABLED=true +PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET +PROVER_AGENT_KEDA_MIN_REPLICAS=0 +PROVER_AGENT_KEDA_MAX_REPLICAS=10 +PROVER_AGENT_KEDA_SCALING_BANDS='[ + { + queueSize = 0 + replicas = 10 + } +]' + +AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1 +AZTEC_SLASHING_QUORUM=20 +AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0 +AZTEC_SLASHING_OFFSET_IN_ROUNDS=1 +AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000 + +DEBUG_P2P_INSTRUMENT_MESSAGES=true +# Reduce metrics volume from the 150 full nodes (mirrors tps-scenario). +FULL_NODE_INCLUDE_METRICS="aztec.p2p.gossip.agg_" +LOG_LEVEL='info;debug:simulator:public-processor,sequencer:state,sequencer:checkpoint-events' + +VALIDATOR_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 +VALIDATOR_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 +PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 +PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 + +RUN_TESTS=false diff --git a/spartan/terraform/deploy-aztec-infra/values/full-node-resources-spot.yaml b/spartan/terraform/deploy-aztec-infra/values/full-node-resources-spot.yaml new file mode 100644 index 000000000000..876cbd04a341 --- /dev/null +++ b/spartan/terraform/deploy-aztec-infra/values/full-node-resources-spot.yaml @@ -0,0 +1,57 @@ +# Full-node profile that REQUIRES spot nodes (Linear A-1223, Set A inclusion +# sweep). Same shape + resources as full-node-resources-prod.yaml, but the +# spot affinity is required (not merely preferred) so full nodes always land on +# spot, copying the pattern from prover-resources-dev.yaml. Only full nodes use +# this profile; validators / RPC / prover stay on-demand so mid-block spot +# eviction cannot corrupt the inclusion measurement. +nodeSelector: + local-ssd: "false" + node-type: "network" + pool: "spot" + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-spot + operator: Exists + +tolerations: + - key: "cloud.google.com/gke-spot" + operator: "Equal" + value: "true" + effect: "NoSchedule" + +replicaCount: 1 + +node: + resources: + requests: + cpu: "0.5" + memory: "2Gi" + limits: + cpu: "1.5" + memory: "6Gi" +persistence: + enabled: true + +statefulSet: + enabled: true + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 16Gi + +service: + p2p: + enabled: true + nodePortEnabled: false + admin: + enabled: true + headless: + enabled: false From 2cb19dfcd81a141551b09be72a57d5e2b707e1ef Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 24 Jun 2026 09:17:16 +0000 Subject: [PATCH 03/39] ci(bench): nightly inclusion-sweep workflow (sequential 1/5/10 TPS) Phase 3 (A-1223) CI wiring for the Set A inclusion sweep. - nightly-bench-inclusion-sweep.yml: runs the three points SEQUENTIALLY (one ~150-full-node network at a time to bound cluster load), each in its own namespace (bench-incl-sweep-{1,5,10}tps), all tagged with a shared sweepId computed once in select-image. Each point chains deploy -> wait -> bench -> teardown; the next point's deploy waits on the previous point's teardown, so peak load stays at one network. A failed point drops only itself. - ci.sh network-inclusion-sweep: threads TARGET_TPS / BENCH_SWEEP_ID / BENCH_SWEEP_LABEL into the EC2 bootstrap command (mirrors network-bench-10tps). - bootstrap.sh ci-network-inclusion-sweep: runs one point via spartan/bootstrap.sh bench_inclusion_point; v4 run JSON (tagged sweepId) is uploaded to GCS inside that function. SKIP_NETWORK_DEPLOY=1 (deploy/teardown done by the workflow), mirroring the ci-network-bench-10tps path. --- .../nightly-bench-inclusion-sweep.yml | 356 ++++++++++++++++++ bootstrap.sh | 30 ++ ci.sh | 15 + 3 files changed, 401 insertions(+) create mode 100644 .github/workflows/nightly-bench-inclusion-sweep.yml diff --git a/.github/workflows/nightly-bench-inclusion-sweep.yml b/.github/workflows/nightly-bench-inclusion-sweep.yml new file mode 100644 index 000000000000..579d9fafbd01 --- /dev/null +++ b/.github/workflows/nightly-bench-inclusion-sweep.yml @@ -0,0 +1,356 @@ +name: Nightly Bench Inclusion Sweep + +# Set A inclusion sweep (Linear A-1223): runs the 1/5/10 TPS points +# SEQUENTIALLY (one ~150-full-node network at a time to bound cluster load), +# each in its own namespace, all tagged with a shared sweepId so the dashboard +# groups them. Each point: deploy -> wait -> bench -> teardown; the next point's +# deploy waits on the previous point's teardown. + +on: + schedule: + - cron: "30 3 * * *" + workflow_dispatch: + inputs: + docker_image: + description: "Full Docker image (e.g., aztecprotocol/aztec:my-tag). Takes precedence over nightly_tag." + required: false + type: string + nightly_tag: + description: "Nightly tag to use (e.g., 2.3.4-nightly.20251209). Ignored if docker_image is set. Leave empty to auto-detect." + required: false + type: string + source_ref: + description: "Git ref to checkout (e.g., v5.0.0-nightly.20260529). Required when docker_image is not a standard nightly tag." + required: false + type: string + +concurrency: + group: nightly-bench-inclusion-sweep-${{ github.ref }} + cancel-in-progress: true + +jobs: + select-image: + runs-on: ubuntu-latest + outputs: + docker_image: ${{ steps.docker-image.outputs.docker_image }} + image_label: ${{ steps.docker-image.outputs.image_label }} + source_ref: ${{ steps.docker-image.outputs.source_ref }} + sweep_id: ${{ steps.sweep.outputs.sweep_id }} + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: next + + - name: Determine docker image + id: docker-image + run: | + if [[ -n "${{ inputs.docker_image }}" ]]; then + docker_image="${{ inputs.docker_image }}" + image_label="${{ inputs.docker_image }}" + image_tag="${docker_image##*:}" + elif [[ -n "${{ inputs.nightly_tag }}" ]]; then + nightly_tag="${{ inputs.nightly_tag }}" + docker_image="aztecprotocol/aztec:${nightly_tag}" + image_label="${nightly_tag}" + image_tag="${nightly_tag}" + else + current_version=$(jq -r '."."' .release-please-manifest.json) + nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)" + docker_image="aztecprotocol/aztec:${nightly_tag}" + image_label="${nightly_tag}" + image_tag="${nightly_tag}" + fi + + if [[ -n "${{ inputs.source_ref }}" ]]; then + source_ref="${{ inputs.source_ref }}" + elif [[ "${image_tag:-}" =~ ^[0-9]+\.[0-9]+\.[0-9]+-nightly\.[0-9]{8}$ ]]; then + source_ref="v${image_tag}" + else + echo "Could not infer git ref from docker image. Pass workflow input source_ref." + exit 1 + fi + + echo "docker_image=$docker_image" >> "$GITHUB_OUTPUT" + echo "image_label=$image_label" >> "$GITHUB_OUTPUT" + echo "source_ref=$source_ref" >> "$GITHUB_OUTPUT" + echo "Using docker image: $docker_image" + echo "Using source ref: $source_ref" + + - name: Verify source git ref + run: | + set -euo pipefail + source_ref="${{ steps.docker-image.outputs.source_ref }}" + git fetch --depth 1 origin "refs/tags/${source_ref}:refs/tags/${source_ref}" + git rev-parse "${source_ref}^{}" + + - name: Check if Docker image exists + run: | + DOCKER_IMAGE="${{ steps.docker-image.outputs.docker_image }}" + if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then + echo "Docker image exists: $DOCKER_IMAGE" + else + echo "Docker image does not exist: $DOCKER_IMAGE" + exit 1 + fi + + - name: Compute shared sweep id + id: sweep + run: | + # Shared across all three points so the dashboard groups the sweep. + echo "sweep_id=incl-$(date -u +%Y%m%d)-${{ github.run_id }}" >> "$GITHUB_OUTPUT" + + # ---- Point: 1 TPS ---- + deploy-1tps: + needs: select-image + uses: ./.github/workflows/deploy-network.yml + with: + network: bench-inclusion-sweep + namespace: bench-incl-sweep-1tps + aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + ref: ${{ needs.select-image.outputs.source_ref }} + notify_on_failure: false + secrets: inherit + + wait-1tps: + needs: [select-image, deploy-1tps] + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + with: + install_components: gke-gcloud-auth-plugin + - name: Wait for first L2 block + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NAMESPACE: bench-incl-sweep-1tps + run: cd spartan && ./bootstrap.sh wait_for_l2_block bench-inclusion-sweep + + bench-1tps: + needs: [select-image, wait-1tps] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - name: Run 1 TPS inclusion point + timeout-minutes: 120 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + RUN_ID: ${{ github.run_id }} + AWS_SHUTDOWN_TIME: 180 + NO_SPOT: 1 + SKIP_NETWORK_DEPLOY: "1" + TARGET_TPS: "1" + BENCH_SWEEP_ID: ${{ needs.select-image.outputs.sweep_id }} + BENCH_SWEEP_LABEL: inclusion-sweep + run: ./.github/ci3.sh network-inclusion-sweep bench-inclusion-sweep bench-incl-sweep-1tps "${{ needs.select-image.outputs.docker_image }}" + + cleanup-1tps: + if: always() + needs: [select-image, deploy-1tps, wait-1tps, bench-1tps] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - name: Cleanup network resources + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NO_SPOT: 1 + run: ./.github/ci3.sh network-teardown bench-inclusion-sweep bench-incl-sweep-1tps + + # ---- Point: 5 TPS (starts after 1 TPS teardown) ---- + deploy-5tps: + needs: [select-image, cleanup-1tps] + uses: ./.github/workflows/deploy-network.yml + with: + network: bench-inclusion-sweep + namespace: bench-incl-sweep-5tps + aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + ref: ${{ needs.select-image.outputs.source_ref }} + notify_on_failure: false + secrets: inherit + + wait-5tps: + needs: [select-image, deploy-5tps] + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + with: + install_components: gke-gcloud-auth-plugin + - name: Wait for first L2 block + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NAMESPACE: bench-incl-sweep-5tps + run: cd spartan && ./bootstrap.sh wait_for_l2_block bench-inclusion-sweep + + bench-5tps: + needs: [select-image, wait-5tps] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - name: Run 5 TPS inclusion point + timeout-minutes: 120 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + RUN_ID: ${{ github.run_id }} + AWS_SHUTDOWN_TIME: 180 + NO_SPOT: 1 + SKIP_NETWORK_DEPLOY: "1" + TARGET_TPS: "5" + BENCH_SWEEP_ID: ${{ needs.select-image.outputs.sweep_id }} + BENCH_SWEEP_LABEL: inclusion-sweep + run: ./.github/ci3.sh network-inclusion-sweep bench-inclusion-sweep bench-incl-sweep-5tps "${{ needs.select-image.outputs.docker_image }}" + + cleanup-5tps: + if: always() + needs: [select-image, deploy-5tps, wait-5tps, bench-5tps] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - name: Cleanup network resources + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NO_SPOT: 1 + run: ./.github/ci3.sh network-teardown bench-inclusion-sweep bench-incl-sweep-5tps + + # ---- Point: 10 TPS (starts after 5 TPS teardown) ---- + deploy-10tps: + needs: [select-image, cleanup-5tps] + uses: ./.github/workflows/deploy-network.yml + with: + network: bench-inclusion-sweep + namespace: bench-incl-sweep-10tps + aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + ref: ${{ needs.select-image.outputs.source_ref }} + notify_on_failure: false + secrets: inherit + + wait-10tps: + needs: [select-image, deploy-10tps] + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + with: + install_components: gke-gcloud-auth-plugin + - name: Wait for first L2 block + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NAMESPACE: bench-incl-sweep-10tps + run: cd spartan && ./bootstrap.sh wait_for_l2_block bench-inclusion-sweep + + bench-10tps: + needs: [select-image, wait-10tps] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - name: Run 10 TPS inclusion point + timeout-minutes: 120 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + RUN_ID: ${{ github.run_id }} + AWS_SHUTDOWN_TIME: 180 + NO_SPOT: 1 + SKIP_NETWORK_DEPLOY: "1" + TARGET_TPS: "10" + BENCH_SWEEP_ID: ${{ needs.select-image.outputs.sweep_id }} + BENCH_SWEEP_LABEL: inclusion-sweep + run: ./.github/ci3.sh network-inclusion-sweep bench-inclusion-sweep bench-incl-sweep-10tps "${{ needs.select-image.outputs.docker_image }}" + + cleanup-10tps: + if: always() + needs: [select-image, deploy-10tps, wait-10tps, bench-10tps] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - name: Cleanup network resources + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NO_SPOT: 1 + run: ./.github/ci3.sh network-teardown bench-inclusion-sweep bench-incl-sweep-10tps + + notify-failure: + if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }} + needs: + - select-image + - bench-1tps + - bench-5tps + - bench-10tps + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - name: Notify Slack on failure + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + run: | + IMAGE="${{ needs.select-image.outputs.image_label || 'unknown' }}" + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + ./ci3/slack_notify_with_claudebox_kickoff "#alerts-next-scenario" \ + "Nightly inclusion sweep FAILED (image ${IMAGE}): <${RUN_URL}|View Run> (🤖)" \ + "Nightly inclusion sweep failed (image ${IMAGE}). CI run: ${RUN_URL}. A failed point drops only that point; check which TPS point(s) failed." \ + --link "$RUN_URL" diff --git a/bootstrap.sh b/bootstrap.sh index 919c1a331448..8461f10be43b 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -936,6 +936,36 @@ case "$cmd" in bench_merge cache_upload spartan-bench-10tps-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json ;; + "ci-network-inclusion-sweep") + # Args: [docker_image] + # Runs one Set A inclusion-sweep point (TARGET_TPS) on the given network. + # The v4 run JSON (tagged BENCH_SWEEP_ID) is uploaded to GCS inside + # bench_inclusion_point; deploy/teardown of each point's namespace is done by + # the workflow, so this is normally called with SKIP_NETWORK_DEPLOY=1. + export CI=1 + env_file="${1:?env_file is required}" + namespace="${2:?namespace is required}" + docker_image="${3:-}" + build + export NAMESPACE="$namespace" + if [ "${SKIP_NETWORK_DEPLOY:-0}" != "1" ]; then + # If no docker image provided, build and push to aztecdev + if [ -z "$docker_image" ]; then + release-image/bootstrap.sh push_pr + docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)" + fi + export AZTEC_DOCKER_IMAGE="$docker_image" + spartan/bootstrap.sh network_deploy "${env_file}" + else + echo "SKIP_NETWORK_DEPLOY=1, running inclusion-sweep point (${TARGET_TPS:-10} TPS) against existing network '$namespace'." + fi + # Run one inclusion-sweep point (TARGET_TPS / BENCH_SWEEP_ID from env). + spartan/bootstrap.sh bench_inclusion_point "${env_file}" + rm -rf bench-out + mkdir -p bench-out + bench_merge + cache_upload spartan-bench-inclusion-${TARGET_TPS:-10}tps-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json + ;; "ci-network-teardown") # Args: # Tears down a deployed network. diff --git a/ci.sh b/ci.sh index 418474fc82f7..1740eee8bd68 100755 --- a/ci.sh +++ b/ci.sh @@ -297,6 +297,21 @@ case "$cmd" in [ "${SKIP_NETWORK_DEPLOY:-0}" = "1" ] && skip_network_deploy=1 bootstrap_ec2 "SKIP_NETWORK_DEPLOY=$skip_network_deploy ./bootstrap.sh ci-network-bench-10tps $*" ;; + network-inclusion-sweep) + # Args: [docker_image] + # Runs one inclusion-sweep point (Set A) at TARGET_TPS against an existing + # network, tagged with BENCH_SWEEP_ID. The workflow deploys/tears down each + # point's namespace separately, so this is normally called with + # SKIP_NETWORK_DEPLOY=1. TARGET_TPS / BENCH_SWEEP_ID / BENCH_SWEEP_LABEL come + # from the caller's env and are threaded into the remote command. + export CI_DASHBOARD="network" + export JOB_ID="x-${2:?namespace is required}-network-inclusion-sweep" CPUS=16 + export AWS_SHUTDOWN_TIME=${AWS_SHUTDOWN_TIME:-180} + export INSTANCE_POSTFIX="n-incl-sweep" + skip_network_deploy=0 + [ "${SKIP_NETWORK_DEPLOY:-0}" = "1" ] && skip_network_deploy=1 + bootstrap_ec2 "TARGET_TPS=${TARGET_TPS:-10} BENCH_SWEEP_ID=${BENCH_SWEEP_ID:-} BENCH_SWEEP_LABEL=${BENCH_SWEEP_LABEL:-inclusion-sweep} SKIP_NETWORK_DEPLOY=$skip_network_deploy ./bootstrap.sh ci-network-inclusion-sweep $*" + ;; network-teardown) # Args: export CI_DASHBOARD="network" From 2e633ca9ca061482168b1306e9d712f45057cb08 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Tue, 23 Jun 2026 20:40:27 +0000 Subject: [PATCH 04/39] test(spartan): scale prover agents to max sooner in prove-n-tps-fake Run #95's proving sweep stalled with the agent fleet parked around ~50 replicas: under fake-realistic delays (PUBLIC_VM ~180s/job) the ~50 agents could not drain a 32-checkpoint epoch's jobs within the proof-submission window, so the proven tip never advanced past epoch 1 and the run ended in a proof-window-expiry reorg. The KEDA bands only requested the full 200 agents once the unassigned queue exceeded 200; at moderate backlog the fleet was capped at 10. Ramp to 75 agents at queue>25 and the full 200 at queue>100 so a fresh epoch's job burst pulls in the fleet it needs. Tuning hypothesis to validate on the next run; node-pool/spot autoscaling may still gate how quickly 200 agents actually schedule. --- spartan/environments/prove-n-tps-fake.env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spartan/environments/prove-n-tps-fake.env b/spartan/environments/prove-n-tps-fake.env index 1808fb482c31..a66544e5c0e6 100644 --- a/spartan/environments/prove-n-tps-fake.env +++ b/spartan/environments/prove-n-tps-fake.env @@ -43,14 +43,14 @@ PROVER_AGENT_KEDA_MAX_REPLICAS=200 PROVER_AGENT_KEDA_SCALING_BANDS='[ { queueSize = 0 - replicas = 4 + replicas = 8 }, { - queueSize = 50 - replicas = 10 + queueSize = 25 + replicas = 75 }, { - queueSize = 200 + queueSize = 100 replicas = 200 } ]' From 9adcdca8379194b34bd487bba3df527a91a078e8 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Wed, 17 Jun 2026 14:25:20 +0000 Subject: [PATCH 05/39] ci: reusable network-teardown action that runs on GitHub runners (cherry picked from commit ef43cb7fa209ce4ee886e1432ed942cbaee9835e) --- .github/actions/network-teardown/action.yml | 52 ++++++++++++++++++++ .github/workflows/nightly-bench-10tps.yml | 15 +++--- .github/workflows/nightly-spartan-bench.yml | 45 +++++++---------- .github/workflows/test-network-scenarios.yml | 17 +++---- 4 files changed, 83 insertions(+), 46 deletions(-) create mode 100644 .github/actions/network-teardown/action.yml diff --git a/.github/actions/network-teardown/action.yml b/.github/actions/network-teardown/action.yml new file mode 100644 index 000000000000..857a6593daf3 --- /dev/null +++ b/.github/actions/network-teardown/action.yml @@ -0,0 +1,52 @@ +name: "Spartan network teardown" +description: > + Tear down a deployed Spartan network (delete any Chaos Mesh experiments and + the Kubernetes namespace) directly on the GitHub runner. This replaces the + EC2-backed `ci3.sh network-teardown` path for cleanup: teardown is pure + gcloud/kubectl work, so there is no need to provision a build instance. +inputs: + env_file: + description: "Spartan environment file name (e.g. tps-scenario), used to source CLUSTER and GCP_REGION." + required: true + namespace: + description: "Kubernetes namespace to tear down." + required: true + gcp_sa_key: + description: "GCP service account key JSON (secrets.GCP_SA_KEY)." + required: true + gcp_project_id: + description: "GCP project id (secrets.GCP_PROJECT_ID)." + required: true +runs: + using: "composite" + steps: + - name: Setup gcloud and install GKE auth plugin + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v2.1.4 + with: + install_components: "gke-gcloud-auth-plugin" + + - name: Tear down network + shell: bash + env: + GCP_SA_KEY: ${{ inputs.gcp_sa_key }} + GCP_PROJECT_ID: ${{ inputs.gcp_project_id }} + NAMESPACE: ${{ inputs.namespace }} + ENV_FILE: ${{ inputs.env_file }} + GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp-key.json + CI: "1" + run: | + set -euo pipefail + # Store the GCP service account key for gcloud auth and the teardown. + set +x + umask 077 + printf '%s' "$GCP_SA_KEY" > "$GOOGLE_APPLICATION_CREDENTIALS" + jq -e . "$GOOGLE_APPLICATION_CREDENTIALS" >/dev/null + + cd "$(git rev-parse --show-toplevel)/spartan" + # Reuse the same env + auth + teardown scripts the EC2 path runs, but + # execute them directly on the runner instead of via bootstrap_ec2. + source ./scripts/source_env_basic.sh + source ./scripts/gcp_auth.sh + source_env_basic "$ENV_FILE" + gcp_auth + ./scripts/network_teardown.sh diff --git a/.github/workflows/nightly-bench-10tps.yml b/.github/workflows/nightly-bench-10tps.yml index 8ac7fee22519..9f85e2a7eaed 100644 --- a/.github/workflows/nightly-bench-10tps.yml +++ b/.github/workflows/nightly-bench-10tps.yml @@ -173,15 +173,12 @@ jobs: ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: ./.github/ci3.sh network-teardown bench-10tps bench-10tps + uses: ./.github/actions/network-teardown + with: + env_file: bench-10tps + namespace: bench-10tps + gcp_sa_key: ${{ secrets.GCP_SA_KEY }} + gcp_project_id: ${{ secrets.GCP_PROJECT_ID }} notify-failure: if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }} diff --git a/.github/workflows/nightly-spartan-bench.yml b/.github/workflows/nightly-spartan-bench.yml index 7a7aa5b499ea..640b3adcead0 100644 --- a/.github/workflows/nightly-spartan-bench.yml +++ b/.github/workflows/nightly-spartan-bench.yml @@ -178,15 +178,12 @@ jobs: ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: ./.github/ci3.sh network-teardown tps-scenario nightly-bench + uses: ./.github/actions/network-teardown + with: + env_file: tps-scenario + namespace: nightly-bench + gcp_sa_key: ${{ secrets.GCP_SA_KEY }} + gcp_project_id: ${{ secrets.GCP_PROJECT_ID }} notify-bench-failure: if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }} @@ -329,15 +326,12 @@ jobs: ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: ./.github/ci3.sh network-teardown prove-n-tps-fake prove-n-tps-fake + uses: ./.github/actions/network-teardown + with: + env_file: prove-n-tps-fake + namespace: prove-n-tps-fake + gcp_sa_key: ${{ secrets.GCP_SA_KEY }} + gcp_project_id: ${{ secrets.GCP_PROJECT_ID }} notify-proving-failure: if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }} @@ -480,15 +474,12 @@ jobs: ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: ./.github/ci3.sh network-teardown block-capacity nightly-block-capacity + uses: ./.github/actions/network-teardown + with: + env_file: block-capacity + namespace: nightly-block-capacity + gcp_sa_key: ${{ secrets.GCP_SA_KEY }} + gcp_project_id: ${{ secrets.GCP_PROJECT_ID }} notify-block-capacity-failure: if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }} diff --git a/.github/workflows/test-network-scenarios.yml b/.github/workflows/test-network-scenarios.yml index 2393ba576920..a9a5f4416ae3 100644 --- a/.github/workflows/test-network-scenarios.yml +++ b/.github/workflows/test-network-scenarios.yml @@ -60,16 +60,13 @@ jobs: - name: Cleanup network resources if: always() - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: | - ./.github/ci3.sh network-teardown "${{ inputs.env_file }}" "${NAMESPACE}-${{ matrix.test_set }}" || true + continue-on-error: true + uses: ./.github/actions/network-teardown + with: + env_file: ${{ inputs.env_file }} + namespace: ${{ env.NAMESPACE }}-${{ matrix.test_set }} + gcp_sa_key: ${{ secrets.GCP_SA_KEY }} + gcp_project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Notify Slack and dispatch ClaudeBox on failure if: failure() From d232977b72fc6c3bb26c12cc57fc34a67b38ca53 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 24 Jun 2026 13:27:37 +0000 Subject: [PATCH 06/39] fix(bench): correct Prometheus metric names for saturation + hint-gen series Verified against a live smoke run: the v4 scraper's memory and duration-histogram series came back empty because the OTel->Prometheus exporter appends unit suffixes the queries omitted: - nodejs heap gauge is nodejs_memory_v8_heap_usage_bytes (was ..._usage) - duration histograms carry a _milliseconds infix, e.g. aztec_proving_queue_job_duration_milliseconds_bucket, aztec_public_processor_tx_duration_milliseconds_bucket, aztec_prover_node_{block,checkpoint}_processing_duration_milliseconds_bucket With these, memory (all roles), hint-gen, and proving-queue job_duration series now populate. ELU + CPU were already correct. Full-node saturation remains empty by design (FULL_NODE_INCLUDE_METRICS restricts full nodes to gossip). --- spartan/scripts/bench_10tps/bench_scrape.ts | 34 +++++++++++++++------ 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index 5197aa32e4d6..99d66da297be 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -591,7 +591,8 @@ const SATURATION_ROLES: Record = { const SATURATION_METRICS: { key: string; metric: string; unit: string }[] = [ { key: "elu", metric: "nodejs_eventloop_utilization", unit: "ratio" }, { key: "cpu", metric: "process_cpu_utilization", unit: "ratio" }, - { key: "mem", metric: "nodejs_memory_v8_heap_usage", unit: "bytes" }, + // OTel exports the v8 heap gauge with a `_bytes` unit suffix. + { key: "mem", metric: "nodejs_memory_v8_heap_usage_bytes", unit: "bytes" }, ]; function buildSaturationDefs(): Record { @@ -630,24 +631,33 @@ const PROVING_INFRA_DEFS: Record = { hintGenPublicTxDurationP50: { metric: "aztec_public_processor_tx_duration", unit: "ms", - query: proverNodeHist(0.5, "aztec_public_processor_tx_duration_bucket"), + query: proverNodeHist( + 0.5, + "aztec_public_processor_tx_duration_milliseconds_bucket", + ), }, hintGenPublicTxDurationP99: { metric: "aztec_public_processor_tx_duration", unit: "ms", - query: proverNodeHist(0.99, "aztec_public_processor_tx_duration_bucket"), + query: proverNodeHist( + 0.99, + "aztec_public_processor_tx_duration_milliseconds_bucket", + ), }, hintGenPublicPhaseDurationP50: { metric: "aztec_public_processor_phase_duration", unit: "ms", - query: proverNodeHist(0.5, "aztec_public_processor_phase_duration_bucket"), + query: proverNodeHist( + 0.5, + "aztec_public_processor_phase_duration_milliseconds_bucket", + ), }, hintGenBlockProcessingDurationP50: { metric: "aztec_prover_node_block_processing_duration", unit: "ms", query: proverNodeHist( 0.5, - "aztec_prover_node_block_processing_duration_bucket", + "aztec_prover_node_block_processing_duration_milliseconds_bucket", ), }, hintGenBlockProcessingDurationP99: { @@ -655,7 +665,7 @@ const PROVING_INFRA_DEFS: Record = { unit: "ms", query: proverNodeHist( 0.99, - "aztec_prover_node_block_processing_duration_bucket", + "aztec_prover_node_block_processing_duration_milliseconds_bucket", ), }, hintGenCheckpointProcessingDurationP50: { @@ -663,7 +673,7 @@ const PROVING_INFRA_DEFS: Record = { unit: "ms", query: proverNodeHist( 0.5, - "aztec_prover_node_checkpoint_processing_duration_bucket", + "aztec_prover_node_checkpoint_processing_duration_milliseconds_bucket", ), }, // Proving queue, broken down by job_type (one series per job type). @@ -680,12 +690,18 @@ const PROVING_INFRA_DEFS: Record = { provingQueueJobDurationP50ByJobType: { metric: "aztec_proving_queue_job_duration", unit: "ms", - query: queueHistByJobType(0.5, "aztec_proving_queue_job_duration_bucket"), + query: queueHistByJobType( + 0.5, + "aztec_proving_queue_job_duration_milliseconds_bucket", + ), }, provingQueueJobDurationP99ByJobType: { metric: "aztec_proving_queue_job_duration", unit: "ms", - query: queueHistByJobType(0.99, "aztec_proving_queue_job_duration_bucket"), + query: queueHistByJobType( + 0.99, + "aztec_proving_queue_job_duration_milliseconds_bucket", + ), }, // Rates of terminal job outcomes — the run #95 stall showed up as timeouts. provingQueueTimedOutJobsByJobType: { From e59107ec6d2040b9cfb7107ca193ae87d7edc66f Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 24 Jun 2026 13:32:59 +0000 Subject: [PATCH 07/39] fix(bench): always scrape an inclusion-sweep point, even if the load test fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bench_inclusion_point ran the scrape only if the load test succeeded — a non-zero test exit aborted the function first, discarding the inclusion records and per-role/proving metrics it had already captured. A degraded point (e.g. a higher-TPS point that misses its target, or any failed assertion) is exactly where that data is most useful. Capture the test exit code, always scrape + upload, then re-surface the failure via the return code so job status still reflects it. --- spartan/bootstrap.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 1567830119d4..2cc19da83bc1 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -299,7 +299,15 @@ function bench_inclusion_point { export_admin_api_key export K8S_ENRICHER=${K8S_ENRICHER:-1} export BENCH_RUN_ID="${BENCH_RUN_ID:-$(date -u +%Y%m%d)-incl-${tps}tps-${COMMIT_HASH:0:10}}" - bench_inclusion_point_cmds | parallelize 1 + # Capture the load-test exit code but do NOT abort: a degraded point (e.g. a + # higher-TPS point that misses its target, or any failed assertion) still + # produced inclusion records worth scraping. We always scrape below, then + # re-surface the failure at the end so the job status still reflects it. + local test_rc=0 + bench_inclusion_point_cmds | parallelize 1 || test_rc=$? + if [[ "$test_rc" -ne 0 ]]; then + echo "[bench_inclusion_point] load test exited ${test_rc}; scraping captured data anyway" + fi local metadata="/tmp/n_tps_timing_data.json" local run_json="bench-out/bench-inclusion-${tps}tps-${BENCH_RUN_ID}.json" @@ -324,6 +332,9 @@ function bench_inclusion_point { else echo "[bench_inclusion_point] no timing metadata at ${metadata}; skipping scraper" fi + + # Re-surface the load-test failure (if any) now that data has been scraped. + return "$test_rc" } function network_bench_upload { From e83809b650e5aae2a7ab748f5ad6223272beba9b Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 24 Jun 2026 15:22:11 +0000 Subject: [PATCH 08/39] test(bench): record inclusion outcome as a metric; harden n_tps teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two robustness changes to n_tps.test.ts, surfaced by a smoke run where the inclusion benchmark mined 300/300 txs but still exited code 1: - Replace the strict "successCount === totalHighValueSent" assertion with metrics.recordInclusionOutcome(mined, failed). A degraded point (a TPS target the network can't fully include) should report its mined-vs-failed counts + success_ratio, not fail the run — the ratio is itself the headline result we track. A total submission failure (nothing sent) is still a hard error. - Wrap the afterAll wallet-cleanup loop and uninstallChaosMesh in try/catch so a teardown error (dead PXE, missing chaos-mesh CRDs on a run that didn't install it) is logged, not thrown — it was turning successful runs red. tx_metrics.ts: TxInclusionMetrics gains recordInclusionOutcome, emitting inclusion/mined_count, inclusion/failed_count, inclusion/success_ratio. --- .../end-to-end/src/spartan/n_tps.test.ts | 35 ++++++++++++++----- .../end-to-end/src/spartan/tx_metrics.ts | 16 +++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/yarn-project/end-to-end/src/spartan/n_tps.test.ts b/yarn-project/end-to-end/src/spartan/n_tps.test.ts index 13d2cd7dabe0..1ff44e705bef 100644 --- a/yarn-project/end-to-end/src/spartan/n_tps.test.ts +++ b/yarn-project/end-to-end/src/spartan/n_tps.test.ts @@ -214,14 +214,25 @@ describe('sustained N TPS test', () => { walletCount: testWallets?.length ?? 0, endpointCount: endpoints?.length ?? 0, }); - for (const { cleanup } of testWallets!) { - await cleanup(); + // Teardown must not fail the suite: the benchmark result is already captured + // by this point, so a cleanup error (dead PXE, missing chaos-mesh CRDs, etc.) + // should be logged, not thrown — otherwise it turns a successful run red. + try { + for (const { cleanup } of testWallets ?? []) { + await cleanup(); + } + } catch (err) { + logger.warn(`Failed to clean up wallets: ${err}`, { err }); } - endpoints.forEach(e => e.process?.kill()); + endpoints?.forEach(e => e.process?.kill()); promProcess?.process?.kill(); - await uninstallChaosMesh(CHAOS_MESH_NAME, config.NAMESPACE, logger); + try { + await uninstallChaosMesh(CHAOS_MESH_NAME, config.NAMESPACE, logger); + } catch (err) { + logger.warn(`Failed to uninstall Chaos Mesh: ${err}`, { err }); + } }); beforeAll(async () => { @@ -726,13 +737,21 @@ describe('sustained N TPS test', () => { logger.info(`Transaction inclusion summary: ${successCount} succeeded, ${failureCount} failed`); logger.info('Inclusion time stats', inclusionStats); + // A total submission failure (nothing sent when load was requested) is still + // a hard error — it means the run is broken, not just degraded. if (totalHighValueSent === 0 && highValueTps > 0) { throw new Error('No high-value txs were sent; check earlier submission errors'); } - if (successCount !== totalHighValueSent) { - const message = `Only ${successCount}/${totalHighValueSent} high-value txs were included; ${failureCount} failed`; - throw new Error(message); - } + // Otherwise record mined-vs-failed as a metric rather than asserting strict + // 1:1 inclusion. A degraded point (e.g. a TPS target the network can't fully + // include) should report its numbers, not fail the run — the inclusion + // success ratio is itself the headline result we want to track over time. + metrics.recordInclusionOutcome(successCount, failureCount); + logger.info('Recorded inclusion outcome', { + mined: successCount, + failed: failureCount, + sent: totalHighValueSent, + }); }); }); diff --git a/yarn-project/end-to-end/src/spartan/tx_metrics.ts b/yarn-project/end-to-end/src/spartan/tx_metrics.ts index bac38184aac7..df76dee027b2 100644 --- a/yarn-project/end-to-end/src/spartan/tx_metrics.ts +++ b/yarn-project/end-to-end/src/spartan/tx_metrics.ts @@ -161,6 +161,7 @@ export class TxInclusionMetrics { private mempoolMinedDelay: | { txP50: number; txP95: number; attestationP50: number; attestationP95: number } | undefined; + private inclusionOutcome: { mined: number; failed: number } | undefined; constructor( private aztecNode: AztecNode, @@ -342,6 +343,11 @@ export class TxInclusionMetrics { this.mempoolMinedDelay = { txP50, txP95, attestationP50, attestationP95 }; } + /** Mined vs failed counts for the high-value lane — recorded instead of asserting strict 1:1 inclusion. */ + public recordInclusionOutcome(mined: number, failed: number): void { + this.inclusionOutcome = { mined, failed }; + } + toGithubActionBenchmarkJSON(): Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> { const data: Array<{ name: string; unit: string; value: number; range?: number; extra?: string }> = []; for (const group of this.groups) { @@ -423,6 +429,16 @@ export class TxInclusionMetrics { ); } + if (this.inclusionOutcome) { + const { mined, failed } = this.inclusionOutcome; + const total = mined + failed; + data.push( + { name: 'inclusion/mined_count', unit: 'count', value: mined }, + { name: 'inclusion/failed_count', unit: 'count', value: failed }, + { name: 'inclusion/success_ratio', unit: 'ratio', value: total > 0 ? mined / total : 0 }, + ); + } + const scenario = process.env.BENCH_SCENARIO?.trim(); if (!scenario) { return data; From 38f696a3fd664d5faaf7d8b7648801020bba8415 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 24 Jun 2026 17:20:11 +0000 Subject: [PATCH 09/39] test(bench): inclusion sweep uses bench-10tps load model (1 TPS high-value + low-value fill) Each inclusion-sweep point now drives a fixed 1 TPS of high-value txs (the measured inclusion lane) plus (TARGET_TPS - 1) TPS of low-value background traffic, instead of all-high-value at the target. This matches bench_10tps and measures what we actually care about: whether a properly-paying user's tx is included promptly while the network runs at the target total TPS. The 10 TPS point is now identical to bench-10tps (1 high + 9 low). --- spartan/bootstrap.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 2cc19da83bc1..4e4a52376025 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -274,16 +274,23 @@ function bench_10tps { } # One point of the Set A inclusion sweep (A-1223). Same scrape+upload path as -# bench_10tps, but parameterized by TARGET_TPS and tagged with a shared -# BENCH_SWEEP_ID so the 1/5/10 points group as one sweep (schema v4). Load is -# all high-value at TARGET_TPS so the headline client-observed inclusion latency -# reflects the full target rate. Each point runs in its own namespace. +# bench_10tps, and the same load model: a fixed 1 TPS of high-value txs (the +# measured inclusion lane) plus (TARGET_TPS - 1) TPS of low-value background +# traffic to bring total load to the target. So inclusion latency is always +# measured for a properly-paying user's tx while the network runs at TARGET_TPS. +# Tagged with a shared BENCH_SWEEP_ID so the 1/5/10 points group as one sweep +# (schema v4). Each point runs in its own namespace. function bench_inclusion_point_cmds { local tps=${TARGET_TPS:-10} + local high_value_tps=1 + # Low-value lane makes up the difference to the target; clamp at 0 for a 1 TPS + # target. awk handles any fractional TARGET_TPS. + local low_value_tps + low_value_tps=$(awk "BEGIN{l=${tps}-${high_value_tps}; if(l<0)l=0; print l}") local test_duration=${TEST_DURATION_SECONDS:-600} # 10 mins local timeout=${BENCH_TIMEOUT_SECONDS:-7200} # account for committee formation local scenario="incl_${tps/./_}tps" - echo "$(hash):TIMEOUT=${timeout} BENCH_RUN_ID=${BENCH_RUN_ID:-} BENCH_OUTPUT=bench-out/n_tps.${scenario}.bench.json BENCH_SCENARIO=${scenario} LOW_VALUE_TPS=0 HIGH_VALUE_TPS=${tps} TEST_DURATION_SECONDS=${test_duration} $root/yarn-project/end-to-end/scripts/run_test.sh simple n_tps.test.ts" + echo "$(hash):TIMEOUT=${timeout} BENCH_RUN_ID=${BENCH_RUN_ID:-} BENCH_OUTPUT=bench-out/n_tps.${scenario}.bench.json BENCH_SCENARIO=${scenario} LOW_VALUE_TPS=${low_value_tps} HIGH_VALUE_TPS=${high_value_tps} TEST_DURATION_SECONDS=${test_duration} $root/yarn-project/end-to-end/scripts/run_test.sh simple n_tps.test.ts" } function bench_inclusion_point { From 749326a7d8f00e8a362b7c99ec525b2142a54a25 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 24 Jun 2026 18:25:38 +0000 Subject: [PATCH 10/39] test(bench): count a tx as included on first sighting, ignore later reorgs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The high-value success/fail count came from waitForTx(waitForStatus: PROPOSED), which throws/times out if a reorg drops a mined tx back to pending or the pool evicts it — so a tx that WAS included got counted as failed under reorg churn (inflating failed_count on reorg-heavy high-TPS runs). Inclusion is a first-sighting event: the block-watcher already stamps minedAtMs idempotently (never cleared on reorg). Add TxInclusionMetrics.wasMined(txHash) and, in the waitForTx catch path, count the tx as included if it was ever observed in a block — we don't care what happens to it afterwards. --- yarn-project/end-to-end/src/spartan/n_tps.test.ts | 9 +++++++++ yarn-project/end-to-end/src/spartan/tx_metrics.ts | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/yarn-project/end-to-end/src/spartan/n_tps.test.ts b/yarn-project/end-to-end/src/spartan/n_tps.test.ts index 1ff44e705bef..33bee9d5a677 100644 --- a/yarn-project/end-to-end/src/spartan/n_tps.test.ts +++ b/yarn-project/end-to-end/src/spartan/n_tps.test.ts @@ -680,6 +680,15 @@ describe('sustained N TPS test', () => { await metrics.recordMinedTx(receipt); results.push({ success: true, txHash }); } catch (error) { + // Once a tx has been observed in a block we count it as included, even if + // waitForTx then threw because a reorg dropped it back to pending or the + // pool evicted it. Inclusion is a first-sighting event here; we don't care + // what happens to the tx afterwards. + if (metrics.wasMined(txHash)) { + logger.info(`${txName} was mined (ignoring post-inclusion reorg/drop)`, { txName, txHash }); + results.push({ success: true, txHash }); + return; + } const receipt = await aztecNode.getTxReceipt(TxHash.fromString(txHash)).catch(() => undefined); logger.error(`${txName} was not included`, { txName, diff --git a/yarn-project/end-to-end/src/spartan/tx_metrics.ts b/yarn-project/end-to-end/src/spartan/tx_metrics.ts index df76dee027b2..d60cc90f128e 100644 --- a/yarn-project/end-to-end/src/spartan/tx_metrics.ts +++ b/yarn-project/end-to-end/src/spartan/tx_metrics.ts @@ -250,6 +250,16 @@ export class TxInclusionMetrics { } } + /** + * Whether this tx was ever observed in a block (by the block-watcher or a mined receipt). + * Idempotent first-sighting semantics: a later reorg / pool eviction never clears it, so callers + * can treat "ever mined" as included regardless of what happens to the tx afterwards. + */ + public wasMined(txHash: string): boolean { + const d = this.data.get(txHash); + return !!d && d.minedAtMs !== -1; + } + /** Per-tx inclusion records for a group. Used to serialise out for downstream tooling. */ getInclusionRecords(group?: string): TxInclusionData[] { const out: TxInclusionData[] = []; From d5e4ee59879dc4302835c3537174f298e1054888 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Thu, 25 Jun 2026 09:24:34 +0000 Subject: [PATCH 11/39] feat(bench): scrape RPC receive_tx duration (node.receive_tx.duration) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ingressTxDurationP50/P99 from aztec.node.receive_tx.duration (histogram, RPC-side tx-ingest cost on the submission path). This is the metric to watch when RPC ingress is the bottleneck — it climbs as the RPC saturates, distinct from the ingress-TPS rate. Flows into the existing timeSeries section (open map, no schema change). --- spartan/scripts/bench_10tps/bench_scrape.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index 99d66da297be..e186f2c8bee4 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -383,6 +383,25 @@ const TIME_SERIES_DEFS: Record = { unit: "tps", query: `sum(rate(aztec_node_receive_tx_count${NS}[1m]))`, }, + // Duration of the RPC node's receiveTx handler (aztec.node.receive_tx.duration, + // a histogram). This is the tx-ingest cost on the submission path — the metric + // to watch when RPC ingress is the bottleneck (climbs as the RPC saturates). + ingressTxDurationP50: { + metric: "aztec_node_receive_tx_duration_milliseconds", + unit: "ms", + query: histQuantile( + 0.5, + "aztec_node_receive_tx_duration_milliseconds_bucket", + ), + }, + ingressTxDurationP99: { + metric: "aztec_node_receive_tx_duration_milliseconds", + unit: "ms", + query: histQuantile( + 0.99, + "aztec_node_receive_tx_duration_milliseconds_bucket", + ), + }, // Pending mempool size sliced by pod role. Three single-series slugs make cross-run // overlay clean: pod names are unstable (replica counts and restart suffixes // change between runs) but role is stable. Each query filters to TxPool to From 4b3005a9c292f6960be413d796361c43d478c564 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Thu, 25 Jun 2026 14:00:11 +0000 Subject: [PATCH 12/39] feat(p2p): record accurate pending-to-mined tx delay Reuse MEMPOOL_TX_MINED_DELAY but derive the value from the tx's persisted receivedAt at the mined transition in handleMinedBlock, instead of the add/remove timestamp map. The map-based recording fired on any removal (eviction/deletion too), conflating mined delay with evicted delay; it was also in-memory and lost across restarts. handleMinedBlock now passes a per-tx minedDelayMs (now - receivedAt) through onTxsMined, left undefined for txs hydrated from the DB on restart (receivedAt === 0) so unknown receive times don't pollute the histogram. The shared PoolInstrumentation map mechanism is retained for the attestation pool. --- .../p2p/src/mem_pools/instrumentation.ts | 24 +++--- .../mem_pools/tx_pool_v2/tx_pool_v2.test.ts | 80 ++++++++++++++++++- .../src/mem_pools/tx_pool_v2/tx_pool_v2.ts | 14 +++- .../mem_pools/tx_pool_v2/tx_pool_v2_impl.ts | 20 ++++- 4 files changed, 120 insertions(+), 18 deletions(-) diff --git a/yarn-project/p2p/src/mem_pools/instrumentation.ts b/yarn-project/p2p/src/mem_pools/instrumentation.ts index 57acd5d57f76..df5cebd5ec27 100644 --- a/yarn-project/p2p/src/mem_pools/instrumentation.ts +++ b/yarn-project/p2p/src/mem_pools/instrumentation.ts @@ -1,5 +1,4 @@ import type { Gossipable } from '@aztec/stdlib/p2p'; -import type { Tx } from '@aztec/stdlib/tx'; import { Attributes, type BatchObservableResult, @@ -117,16 +116,6 @@ export class PoolInstrumentation { this.addObjectCounter.add(count); } - public transactionsAdded(transactions: Tx[]) { - transactions.forEach(tx => this.trackMempoolItemAdded(tx.txHash.toBigInt())); - } - - public transactionsRemoved(hashes: Iterable | Iterable) { - for (const hash of hashes) { - this.trackMempoolItemRemoved(BigInt(hash)); - } - } - public trackMempoolItemAdded(key: bigint | string): void { this.mempoolItemAddedTimestamp.set(key, Date.now()); } @@ -142,6 +131,19 @@ export class PoolInstrumentation { } } + /** + * Records a pre-computed pending-to-mined delay directly, bypassing the + * `mempoolItemAddedTimestamp` map. Used by the tx pool, which derives the + * delay from the persisted `receivedAt` on tx metadata at the mined + * transition — more accurate than the add/remove map (which also fires on + * eviction) and resilient to the caller not having tracked the add. + */ + public recordMinedDelay(delayMs: number): void { + if (delayMs > 0) { + this.minedDelay.record(delayMs); + } + } + private observeStats = async (observer: BatchObservableResult) => { const { itemCount } = await this.poolStats(); if (typeof itemCount === 'number') { diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts index 53f0ec8d9be5..c28ee3da302f 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts @@ -8,7 +8,8 @@ import { import { BlockNumber, CheckpointNumber, IndexWithinCheckpoint, SlotNumber } from '@aztec/foundation/branded-types'; import { timesAsync } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; -import { DateProvider } from '@aztec/foundation/timer'; +import { createLogger } from '@aztec/foundation/log'; +import { DateProvider, TestDateProvider } from '@aztec/foundation/timer'; import type { AztecAsyncMap } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { RevertCode } from '@aztec/stdlib/avm'; @@ -24,6 +25,7 @@ import { PublicDataTreeLeafPreimage, } from '@aztec/stdlib/trees'; import { BlockHeader, GlobalVariables, Tx, TxEffect, TxHash, type TxValidator } from '@aztec/stdlib/tx'; +import { getTelemetryClient } from '@aztec/telemetry-client'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -32,6 +34,7 @@ import { GasLimitsValidator, MaxFeePerGasValidator } from '../../msg_validators/ import { AllowedSetupCallsMetaValidator } from '../../msg_validators/tx_validator/phases_validator.js'; import type { TxMetaData } from './tx_metadata.js'; import { AztecKVTxPoolV2 } from './tx_pool_v2.js'; +import { type MinedTxInfo, TxPoolV2Impl } from './tx_pool_v2_impl.js'; // Tx type alias for cleaner type annotations type MockTx = Awaited>; @@ -1545,6 +1548,81 @@ describe('TxPoolV2', () => { expectNoCallbacks(); }); + describe('pending-to-mined delay', () => { + // The wrapper feeds onTxsMined's delays into MEMPOOL_TX_MINED_DELAY. We construct the impl + // directly with a spy callback + controllable clock so we can assert the emitted delay, which + // is the behaviour the histogram records. + const makeImpl = async (onTxsMined: (m: MinedTxInfo[]) => void, dateProvider: DateProvider) => { + const implStore = await openTmpStore('impl-p2p'); + const implArchive = await openTmpStore('impl-archive'); + const impl = new TxPoolV2Impl( + implStore, + implArchive, + { + l2BlockSource: mockL2BlockSource, + worldStateSynchronizer: mockWorldState, + createTxValidator: () => Promise.resolve(alwaysValidValidator), + checkAllowedSetupCalls: () => Promise.resolve(true), + blockMinFeesProvider: { getCurrentMinFees: () => Promise.resolve(GasFees.empty()) }, + }, + { onTxsAdded: () => {}, onTxsRemoved: () => {}, onTxsMined }, + getTelemetryClient(), + {}, + dateProvider, + createLogger('test:tx-pool-mined-delay'), + ); + const cleanup = async () => { + await implStore.delete(); + await implArchive.delete(); + }; + return { impl, cleanup }; + }; + + it('reports now - receivedAt for a tx that was pending in the pool', async () => { + const minedCalls: MinedTxInfo[][] = []; + const dateProvider = new TestDateProvider(); + const { impl, cleanup } = await makeImpl(m => minedCalls.push(m), dateProvider); + try { + const tx = await mockTx(1); + await impl.addPendingTxs([tx], {}); // receivedAt stamped at "now" + dateProvider.advanceTime(5); // 5s in the pool before it's mined + + await impl.handleMinedBlock(makeBlock([tx], slot1Header)); + + expect(minedCalls).toHaveLength(1); + const minedTxs = minedCalls[0]; + expect(minedTxs).toHaveLength(1); + expect(minedTxs[0].txHash).toBe(hashOf(tx)); + // 5s advance dominates; real elapsed during the test is a few ms. + expect(minedTxs[0].minedDelayMs).toBeGreaterThanOrEqual(5000); + expect(minedTxs[0].minedDelayMs).toBeLessThan(6000); + } finally { + await cleanup(); + } + }); + + it('leaves the delay undefined when receivedAt is unknown (hydrated/restart tx)', async () => { + const minedCalls: MinedTxInfo[][] = []; + const dateProvider = new TestDateProvider(); + const { impl, cleanup } = await makeImpl(m => minedCalls.push(m), dateProvider); + try { + const tx = await mockTx(1); + await impl.addPendingTxs([tx], {}); + // Simulate a tx whose receive time was lost (e.g. rebuilt from the DB on restart). + impl.getPoolReadAccess().getMetadata(hashOf(tx))!.receivedAt = 0; + + await impl.handleMinedBlock(makeBlock([tx], slot1Header)); + + expect(minedCalls).toHaveLength(1); + const minedTxs = minedCalls[0]; + expect(minedTxs[0].txHash).toBe(hashOf(tx)); + expect(minedTxs[0].minedDelayMs).toBeUndefined(); + } finally { + await cleanup(); + } + }); + }); + it('deletes pending transactions with conflicting nullifiers', async () => { // Create two transactions with the same nullifier const txLow = await mockPublicTx(1, 5); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts index 9fc60e7c2c3b..f02173f85eed 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts @@ -20,6 +20,7 @@ import type { TxPoolV2Events, } from './interfaces.js'; import type { TxState } from './tx_metadata.js'; +import type { MinedTxInfo } from './tx_pool_v2_impl.js'; import { TxPoolV2Impl } from './tx_pool_v2_impl.js'; /** @@ -56,17 +57,22 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte // Create callbacks that the impl uses to notify us about events and metrics const callbacks = { onTxsAdded: (txs: Tx[], opts: { source?: string }) => { - this.#metrics?.transactionsAdded(txs); this.emit('txs-added', { txs, ...opts }); }, onTxsRemoved: (txHashes: string[] | bigint[]) => { - this.#metrics?.transactionsRemoved(txHashes); // Convert to TxHash objects for the event const hashes = txHashes.map(h => (typeof h === 'string' ? TxHash.fromString(h) : TxHash.fromBigInt(h))); this.emit('txs-removed', { txHashes: hashes }); }, - onTxsMined: (txHashes: string[]) => { - this.#metrics?.transactionsRemoved(txHashes); + onTxsMined: (minedTxs: MinedTxInfo[]) => { + // Pending-to-mined delay is derived from the tx's persisted receivedAt at the mined + // transition (see TxPoolV2Impl.handleMinedBlock), not the add/remove timestamp map — + // so eviction no longer pollutes MEMPOOL_TX_MINED_DELAY. + for (const { minedDelayMs } of minedTxs) { + if (minedDelayMs !== undefined) { + this.#metrics?.recordMinedDelay(minedDelayMs); + } + } }, }; diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts index 8cfb4a7c24ab..6c31e09eb333 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts @@ -51,10 +51,18 @@ const FINALIZE_BLOCK_CHUNK_SIZE = 100; /** * Callbacks for the implementation to notify the outer class about events and metrics. */ +/** A tx that has just transitioned to mined, with the time it spent in the pool. */ +export interface MinedTxInfo { + txHash: string; + /** Wall-clock ms from receipt into the pool to being marked mined. Undefined when the + * receive time is unknown (e.g. a tx hydrated from the DB on restart, receivedAt === 0). */ + minedDelayMs?: number; +} + export interface TxPoolV2Callbacks { onTxsAdded: (txs: Tx[], opts: { source?: string }) => void; onTxsRemoved: (txHashes: string[] | bigint[]) => void; - onTxsMined: (txHashes: string[]) => void; + onTxsMined: (minedTxs: MinedTxInfo[]) => void; } /** @@ -560,7 +568,15 @@ export class TxPoolV2Impl { }); if (found.length > 0) { - this.#callbacks.onTxsMined(found.map(m => m.txHash)); + // receivedAt is 0 for txs hydrated from the DB on restart (true receive time lost) — leave + // their delay undefined so the metric isn't polluted by epoch-sized values. + const now = this.#dateProvider.now(); + this.#callbacks.onTxsMined( + found.map(m => ({ + txHash: m.txHash, + minedDelayMs: m.receivedAt > 0 ? now - m.receivedAt : undefined, + })), + ); } this.#log.info(`Marked ${found.length} txs as mined in block ${blockId.number}`); From 922ac7bcd5cde8b7ef6affca507e87b6479f22f3 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Thu, 25 Jun 2026 16:42:03 +0000 Subject: [PATCH 13/39] feat(telemetry): widen buckets for mempool tx mined-delay histogram The pending-to-mined delay (aztec.mempool.tx_mined_delay) routinely exceeds the 60s ceiling of the shared ms-unit histogram view under load, saturating every percentile at 60000. Add a name-scoped view with buckets from 1s to 10min, placed before the generic ms view so it wins the bucket boundaries (the SDK keeps the first-registered compatible storage; other ms histograms are unaffected). Also fix the now-stale metric description. --- yarn-project/telemetry-client/src/metrics.ts | 2 +- yarn-project/telemetry-client/src/otel.ts | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 66c7ef86bdbb..520775a3badc 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -162,7 +162,7 @@ export const MEMPOOL_TX_ADDED_COUNT: MetricDefinition = { }; export const MEMPOOL_TX_MINED_DELAY: MetricDefinition = { name: 'aztec.mempool.tx_mined_delay', - description: 'Delay between transaction added and evicted from the mempool', + description: 'Delay (ms) from a transaction being received into the pool to being mined', unit: 'ms', valueType: ValueType.INT, }; diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts index a67cb76ef637..f06a9e333eda 100644 --- a/yarn-project/telemetry-client/src/otel.ts +++ b/yarn-project/telemetry-client/src/otel.ts @@ -242,6 +242,23 @@ export class OpenTelemetryClient implements TelemetryClient { true, ), }), + // Pending-to-mined delay routinely exceeds the 1-minute ceiling of the generic `ms` + // view below under load, so it would saturate at 60s. Give this one metric wider + // buckets (1s to 10min). This must precede the generic `ms` view: when multiple views + // match an instrument, the SDK keeps the first-registered compatible storage, so the + // first view in this list wins the bucket boundaries. + new View({ + instrumentType: InstrumentType.HISTOGRAM, + instrumentName: 'aztec.mempool.tx_mined_delay', + instrumentUnit: 'ms', + aggregation: new ExplicitBucketHistogramAggregation( + [ + 1_000, 2_500, 5_000, 7_500, 10_000, 15_000, 30_000, 45_000, 60_000, 90_000, 120_000, 180_000, 300_000, + 600_000, + ], + true, + ), + }), new View({ instrumentType: InstrumentType.HISTOGRAM, instrumentUnit: 'ms', From bff9c2d5100b813530f6d139114e25174f36cf07 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Thu, 25 Jun 2026 17:04:35 +0000 Subject: [PATCH 14/39] feat(bench): scrape per-tx block build, pool mined-delay, and per-role event-loop delay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three additive metrics in the v4 scrape output: - Per-tx block build decomposition in summary: blockBuildPerTxMsAggregate, blockBuildMarginalMsPerTx (OLS slope), blockBuildFixedOverheadMsPerBlock (OLS intercept), blockBuildAvgTxsPerBlock — computed from per-block records. - Pool-side pending-to-mined delay (aztec_mempool_tx_mined_delay) as both timeSeries (mempoolTxMinedDelayP50/P95/P99) and summary scalars, distinct from the client-observed inclusionLatency*. - Event-loop delay (nodejs_eventloop_delay_mean) added to per-role saturation, surfacing the prover node's multi-second main-thread blockage at 10 TPS. --- .../bench_10tps/bench_output.schema.json | 28 ++++- spartan/scripts/bench_10tps/bench_scrape.ts | 111 +++++++++++++++++- 2 files changed, 137 insertions(+), 2 deletions(-) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index ca1fd691187e..652750906a14 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -36,7 +36,7 @@ }, "saturation": { "$ref": "#/$defs/metricSeriesMap", - "description": "v4. Per-role resource saturation: ELU, CPU, and memory for each role (validator / rpc / fullNode / proverNode / broker / agent), each emitted as both the hottest pod (max across pods) and the role average (avg across pods). Never a single hand-picked pod. Optional — absent on older runs." + "description": "v4. Per-role resource saturation: ELU, CPU, memory, and event-loop delay (mean, raw nanoseconds) for each role (validator / rpc / fullNode / proverNode / broker / agent), each emitted as both the hottest pod (max across pods) and the role average (avg across pods). Never a single hand-picked pod. Optional — absent on older runs." }, "blocks": { "type": "array", @@ -306,12 +306,38 @@ "blockBuildDurationP95Ms": { "type": ["number", "null"] }, + "blockBuildAvgTxsPerBlock": { + "type": ["number", "null"], + "description": "Mean successfulCount over non-empty blocks in the inclusion window. Context for the per-tx build figures below." + }, + "blockBuildPerTxMsAggregate": { + "type": ["number", "null"], + "description": "Block build cost per tx: sum(buildDurationMs) / sum(txs) over non-empty blocks. Single headline 'build-ms per tx'." + }, + "blockBuildMarginalMsPerTx": { + "type": ["number", "null"], + "description": "Marginal build cost of one tx: OLS slope of buildDurationMs vs txCount across non-empty blocks (buildMs = fixed + marginal*txCount). Null if < 2 blocks or uniform tx counts." + }, + "blockBuildFixedOverheadMsPerBlock": { + "type": ["number", "null"], + "description": "Fixed per-block build overhead: OLS intercept of buildDurationMs vs txCount. The load-sensitive component independent of tx count." + }, "publicProcessorTxDurationP50Ms": { "type": ["number", "null"] }, "publicProcessorTxDurationP95Ms": { "type": ["number", "null"] }, + "mempoolTxMinedDelayP50Ms": { + "type": ["number", "null"], + "description": "Pool-side pending-to-mined delay (aztec_mempool_tx_mined_delay, now - receivedAt at the mined transition), across all pool txs. Distinct from the client-observed inclusionLatency* (high-value lane only)." + }, + "mempoolTxMinedDelayP95Ms": { + "type": ["number", "null"] + }, + "mempoolTxMinedDelayP99Ms": { + "type": ["number", "null"] + }, "totalTxsMined": { "type": ["integer", "null"], "description": "Exact sum from per-block logs. Null when block logs were unavailable and inclusionTpsMean came from Prometheus." diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index e186f2c8bee4..fafd504fd011 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -402,6 +402,37 @@ const TIME_SERIES_DEFS: Record = { "aztec_node_receive_tx_duration_milliseconds_bucket", ), }, + // Pool-side pending->mined delay measured inside the tx pool + // (aztec_mempool_tx_mined_delay, recorded as now - receivedAt at the mined + // transition). This is the node's own view of how long txs sat in the mempool + // before being mined, across ALL pool txs. Distinct from txMinedDelay* below, + // which is the client-observed inclusion latency for the high-value lane only. + // No aztec_pool_name filter needed: the attestation pool uses a different + // metric name (aztec_mempool_attestations_mined_delay). + mempoolTxMinedDelayP50: { + metric: "aztec_mempool_tx_mined_delay_milliseconds", + unit: "ms", + query: histQuantile( + 0.5, + "aztec_mempool_tx_mined_delay_milliseconds_bucket", + ), + }, + mempoolTxMinedDelayP95: { + metric: "aztec_mempool_tx_mined_delay_milliseconds", + unit: "ms", + query: histQuantile( + 0.95, + "aztec_mempool_tx_mined_delay_milliseconds_bucket", + ), + }, + mempoolTxMinedDelayP99: { + metric: "aztec_mempool_tx_mined_delay_milliseconds", + unit: "ms", + query: histQuantile( + 0.99, + "aztec_mempool_tx_mined_delay_milliseconds_bucket", + ), + }, // Pending mempool size sliced by pod role. Three single-series slugs make cross-run // overlay clean: pod names are unstable (replica counts and restart suffixes // change between runs) but role is stable. Each query filters to TxPool to @@ -612,6 +643,14 @@ const SATURATION_METRICS: { key: string; metric: string; unit: string }[] = [ { key: "cpu", metric: "process_cpu_utilization", unit: "ratio" }, // OTel exports the v8 heap gauge with a `_bytes` unit suffix. { key: "mem", metric: "nodejs_memory_v8_heap_usage_bytes", unit: "bytes" }, + // Event-loop delay (mean of the per-scrape distribution), raw nanoseconds. The + // max-across-pods series surfaces the role's most event-loop-blocked pod — e.g. + // the prover node, whose synchronous hint-gen blocks its main thread for seconds. + { + key: "eventLoopDelay", + metric: "nodejs_eventloop_delay_mean_nanoseconds", + unit: "ns", + }, ]; function buildSaturationDefs(): Record { @@ -1717,6 +1756,51 @@ type SummaryArgs = { inclusionRecords: InclusionRecord[]; }; +type BlockBuildPerTx = { + blockBuildAvgTxsPerBlock: number | null; + blockBuildPerTxMsAggregate: number | null; + blockBuildMarginalMsPerTx: number | null; + blockBuildFixedOverheadMsPerBlock: number | null; +}; + +// Decomposes block build time into a fixed per-block overhead and a marginal +// per-tx cost via an ordinary least-squares fit of build-ms against tx count +// over non-empty blocks (buildMs = fixed + marginal * txCount). The marginal +// slope is the (load-invariant) cost of adding one tx to a block; the intercept +// is the per-block overhead that grows with block size / system load. The +// aggregate is the simpler sum(buildMs)/sum(tx) — a single "build-ms per tx" +// headline. The OLS slope needs >= 2 blocks with differing tx counts, so +// marginal/fixed are null for a run with too few or uniform-width blocks. +function blockBuildPerTx(blocks: BlockRecord[]): BlockBuildPerTx { + const pts = blocks + .filter( + (b) => b.successfulCount > 0 && Number.isFinite(b.buildDurationSeconds), + ) + .map((b) => ({ x: b.successfulCount, y: b.buildDurationSeconds * 1000 })); + if (pts.length === 0) { + return { + blockBuildAvgTxsPerBlock: null, + blockBuildPerTxMsAggregate: null, + blockBuildMarginalMsPerTx: null, + blockBuildFixedOverheadMsPerBlock: null, + }; + } + const n = pts.length; + const sx = pts.reduce((s, p) => s + p.x, 0); + const sy = pts.reduce((s, p) => s + p.y, 0); + const sxx = pts.reduce((s, p) => s + p.x * p.x, 0); + const sxy = pts.reduce((s, p) => s + p.x * p.y, 0); + const denom = n * sxx - sx * sx; + const marginal = n >= 2 && denom !== 0 ? (n * sxy - sx * sy) / denom : null; + const fixed = marginal !== null ? (sy - marginal * sx) / n : null; + return { + blockBuildAvgTxsPerBlock: sx / n, + blockBuildPerTxMsAggregate: sx > 0 ? sy / sx : null, + blockBuildMarginalMsPerTx: marginal, + blockBuildFixedOverheadMsPerBlock: fixed, + }; +} + async function buildSummary(a: SummaryArgs): Promise> { // inclusionTps is single-series; series[0] holds all points. const inclusionPoints = ( @@ -1780,7 +1864,15 @@ async function buildSummary(a: SummaryArgs): Promise> { log("No inclusion records loaded; summary.inclusionLatencyP* will be null"); } - const [buildP50, buildP95, ppTxP50, ppTxP95] = await Promise.all([ + const [ + buildP50, + buildP95, + ppTxP50, + ppTxP95, + mempoolMinedP50, + mempoolMinedP95, + mempoolMinedP99, + ] = await Promise.all([ safeInstant( oneShotQuantile( 0.5, @@ -1805,6 +1897,19 @@ async function buildSummary(a: SummaryArgs): Promise> { "aztec_public_processor_tx_duration_milliseconds_bucket", ), ), + // Pool-side pending->mined delay (now - receivedAt at the mined transition), + // across all pool txs. Companion to the time series of the same name; lets + // the dashboard trend it and compare against the client-observed + // inclusionLatency* (high-value lane only). + safeInstant( + oneShotQuantile(0.5, "aztec_mempool_tx_mined_delay_milliseconds_bucket"), + ), + safeInstant( + oneShotQuantile(0.95, "aztec_mempool_tx_mined_delay_milliseconds_bucket"), + ), + safeInstant( + oneShotQuantile(0.99, "aztec_mempool_tx_mined_delay_milliseconds_bucket"), + ), ]); const reorgs = a.events.filter((e) => e.type === "chainPruned"); @@ -1824,8 +1929,12 @@ async function buildSummary(a: SummaryArgs): Promise> { inclusionLatencyP99Ms: inclLatP99, blockBuildDurationP50Ms: buildP50, blockBuildDurationP95Ms: buildP95, + ...blockBuildPerTx(inclusionBlocks), publicProcessorTxDurationP50Ms: ppTxP50, publicProcessorTxDurationP95Ms: ppTxP95, + mempoolTxMinedDelayP50Ms: mempoolMinedP50, + mempoolTxMinedDelayP95Ms: mempoolMinedP95, + mempoolTxMinedDelayP99Ms: mempoolMinedP99, totalTxsMined, totalTxsFailed: hasInclusionBlockRecords ? inclusionBlocks.reduce((s, b) => s + b.failedCount, 0) From de4bd020b17e2418f37322bd1d14549f75d9dab6 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Thu, 25 Jun 2026 20:21:08 +0000 Subject: [PATCH 15/39] feat(bench): dedicated guaranteed-QoS resource profiles for bench nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validator and RPC pods previously requested only 0.5 CPU with a node-type=network selector, so the scheduler bin-packed them across the shared cluster's mixed 2-/4-core network pools. Validators landing on 2-core nodes ran their 4 attesters on too little CPU, fell behind on re-execution, and failed to attest ('Timed out waiting for block with archive matching checkpoint proposal') — dragging the committee below quorum and causing checkpoint prunes (reorgs) at 10 TPS. Add bench resource profiles (request=limit, Guaranteed QoS) so every node gets dedicated, un-contended cores and lands on an adequately-sized node: - validator: 3.5 CPU / 12Gi (one pod, 4 attesters, per dedicated 4-core node) - rpc: 3.5 CPU / 8Gi (dedicated 4-core; drops prod's preferred cores=2 affinity) - full-node: 3.5 CPU / 8Gi (kept on spot; packs onto 8-/32-core spot nodes) Point both bench-inclusion-sweep envs at VALIDATOR/RPC/FULL_NODE_RESOURCE_PROFILE=bench. Prover/broker keep dev-hi-tps (already explicitly sized, never bin-packed). --- .../environments/bench-inclusion-sweep.env | 14 +++-- .../values/full-node-resources-bench.yaml | 59 +++++++++++++++++++ .../values/rpc-resources-bench.yaml | 43 ++++++++++++++ .../values/validator-resources-bench.yaml | 33 +++++++++++ 4 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 spartan/terraform/deploy-aztec-infra/values/full-node-resources-bench.yaml create mode 100644 spartan/terraform/deploy-aztec-infra/values/rpc-resources-bench.yaml create mode 100644 spartan/terraform/deploy-aztec-infra/values/validator-resources-bench.yaml diff --git a/spartan/environments/bench-inclusion-sweep.env b/spartan/environments/bench-inclusion-sweep.env index d5e1e84f31e7..ffbb9d63205c 100644 --- a/spartan/environments/bench-inclusion-sweep.env +++ b/spartan/environments/bench-inclusion-sweep.env @@ -43,7 +43,7 @@ VALIDATOR_REPLICAS=12 VALIDATORS_PER_NODE=4 VALIDATOR_PUBLISHERS_PER_REPLICA=4 VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 -VALIDATOR_RESOURCE_PROFILE="prod" +VALIDATOR_RESOURCE_PROFILE="bench" VALIDATOR_HA_REPLICAS=0 SEQ_BLOCK_DURATION_MS=6000 @@ -55,11 +55,15 @@ SEQ_MIN_TX_PER_BLOCK=1 SEQ_BUILD_CHECKPOINT_IF_EMPTY=true RPC_REPLICAS=10 -RPC_RESOURCE_PROFILE="prod" -RPC_INGRESS_ENABLED=false +RPC_RESOURCE_PROFILE="bench" +# Load-balanced external IP so the test's per-wallet clients spread across the 10 +# RPC pods. With ingress disabled the test hits a single-pod port-forward, so the +# extra replicas would sit idle behind one pod (the ingress bottleneck seen at +# 5/10 TPS on the smoke net). +RPC_INGRESS_ENABLED=true -FULL_NODE_REPLICAS=150 -FULL_NODE_RESOURCE_PROFILE="spot" +FULL_NODE_REPLICAS=10 +FULL_NODE_RESOURCE_PROFILE="bench" # Fake proving (fixed, cheap delay) — this run does not measure proving; it only # must not bottleneck inclusion (proving sweep is Set B / A-1224). diff --git a/spartan/terraform/deploy-aztec-infra/values/full-node-resources-bench.yaml b/spartan/terraform/deploy-aztec-infra/values/full-node-resources-bench.yaml new file mode 100644 index 000000000000..1a1967133ec3 --- /dev/null +++ b/spartan/terraform/deploy-aztec-infra/values/full-node-resources-bench.yaml @@ -0,0 +1,59 @@ +# Bench profile: deterministic full-node sizing for benchmarking. +# Full nodes still validate every gossiped tx and apply every block, so at +# 10 TPS they need the same 4-core tier as validators/RPC (request=limit, +# Guaranteed QoS, 3.5 CPU). There is no 4-core spot pool, so they land on the +# 8-core spot pool. Spot affinity is REQUIRED (as in the prod-spot profile) so +# full nodes always land on spot; validators / RPC / prover stay on-demand so a +# mid-block spot eviction cannot corrupt the inclusion measurement (full nodes +# are not on the measurement path). +nodeSelector: + local-ssd: "false" + node-type: "network" + pool: "spot" + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-spot + operator: Exists + +tolerations: + - key: "cloud.google.com/gke-spot" + operator: "Equal" + value: "true" + effect: "NoSchedule" + +replicaCount: 1 + +node: + resources: + requests: + cpu: "3.5" + memory: "8Gi" + limits: + cpu: "3.5" + memory: "8Gi" +persistence: + enabled: true + +statefulSet: + enabled: true + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 16Gi + +service: + p2p: + enabled: true + nodePortEnabled: false + admin: + enabled: true + headless: + enabled: false diff --git a/spartan/terraform/deploy-aztec-infra/values/rpc-resources-bench.yaml b/spartan/terraform/deploy-aztec-infra/values/rpc-resources-bench.yaml new file mode 100644 index 000000000000..2024f4c22b37 --- /dev/null +++ b/spartan/terraform/deploy-aztec-infra/values/rpc-resources-bench.yaml @@ -0,0 +1,43 @@ +# Bench profile: deterministic, guaranteed RPC sizing for benchmarking. +# request=limit (Guaranteed QoS) reserves the cores; a >1.93-core request keeps +# RPC off the 2-core pool: one pod per dedicated 4-core node, on-demand. +# +# NOTE: the prod profile carries a `preferredDuringScheduling` affinity for +# cores=2 (so RPC normally favours the small pool). That preference is +# intentionally omitted here — the bench needs RPC on adequately-sized nodes. +nodeSelector: + local-ssd: "false" + node-type: "network" + +replicaCount: 1 + +node: + resources: + requests: + cpu: "3.5" + memory: "8Gi" + limits: + cpu: "3.5" + memory: "8Gi" +persistence: + enabled: true + +statefulSet: + enabled: true + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 16Gi + +service: + p2p: + enabled: true + nodePortEnabled: false + admin: + enabled: true + headless: + enabled: false diff --git a/spartan/terraform/deploy-aztec-infra/values/validator-resources-bench.yaml b/spartan/terraform/deploy-aztec-infra/values/validator-resources-bench.yaml new file mode 100644 index 000000000000..d78e266f2f22 --- /dev/null +++ b/spartan/terraform/deploy-aztec-infra/values/validator-resources-bench.yaml @@ -0,0 +1,33 @@ +# Bench profile: deterministic, guaranteed validator sizing for benchmarking. +# The prod profile requests only 0.5 CPU with a node-type=network selector, so +# the scheduler bin-packs validators across the shared cluster's mixed 2-/4-core +# network pools. Pods that land on 2-core nodes run their 4 attesters on too +# little CPU, fall behind on re-execution, and fail to attest ("Timed out +# waiting for block with archive matching checkpoint proposal") — dragging the +# committee below quorum and causing checkpoint prunes at 10 TPS. +# +# Here request=limit (Guaranteed QoS) reserves the cores so nothing is +# contended, and a >1.93-core request keeps validators off the 2-core pool: +# one pod (4 attesters) per dedicated 4-core node. On-demand (no spot) so +# eviction can't corrupt the measurement. +validator: + nodeSelector: + local-ssd: "false" + node-type: "network" + node: + resources: + requests: + cpu: "3.5" + memory: "12Gi" + limits: + cpu: "3.5" + memory: "12Gi" + statefulSet: + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 16Gi From 22d8b2d7ba180b318ca078687b33f1d4e9c4ab07 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 14:14:19 +0000 Subject: [PATCH 16/39] feat(bench): add 4-core spot node pool and pin bench full nodes to it Adds a dedicated 4-core spot pool (spot_nodes_4core, t2d-standard-4) to the GKE cluster. Benchmark full nodes request 3.5 CPU but there was no 4-core spot pool (spot pools were 2/8/32-core), so they packed several onto 8-core spot boxes. Pin the bench full-node profile to the new pool via a cores=4 nodeSelector so each full node gets a dedicated 4-core spot node, without consuming the on-demand pool that validators / RPC / prover run on. Requires a terraform apply of the gke-cluster/cluster module before the next bench deploy, otherwise the cores=4 + pool=spot full-node pods stay Pending. --- .../values/full-node-resources-bench.yaml | 12 +++-- spartan/terraform/gke-cluster/cluster/main.tf | 48 ++++++++++++++++++- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/spartan/terraform/deploy-aztec-infra/values/full-node-resources-bench.yaml b/spartan/terraform/deploy-aztec-infra/values/full-node-resources-bench.yaml index 1a1967133ec3..194256b5e0f6 100644 --- a/spartan/terraform/deploy-aztec-infra/values/full-node-resources-bench.yaml +++ b/spartan/terraform/deploy-aztec-infra/values/full-node-resources-bench.yaml @@ -1,15 +1,17 @@ # Bench profile: deterministic full-node sizing for benchmarking. # Full nodes still validate every gossiped tx and apply every block, so at # 10 TPS they need the same 4-core tier as validators/RPC (request=limit, -# Guaranteed QoS, 3.5 CPU). There is no 4-core spot pool, so they land on the -# 8-core spot pool. Spot affinity is REQUIRED (as in the prod-spot profile) so -# full nodes always land on spot; validators / RPC / prover stay on-demand so a -# mid-block spot eviction cannot corrupt the inclusion measurement (full nodes -# are not on the measurement path). +# Guaranteed QoS, 3.5 CPU). The `cores: "4"` selector pins them to the dedicated +# 4-core spot pool (cluster/main.tf spot_nodes_4core) — one full node per spot +# box — instead of packing several onto 8-core spot. Spot affinity is REQUIRED +# (as in the prod-spot profile) so full nodes always land on spot; validators / +# RPC / prover stay on-demand so a mid-block spot eviction cannot corrupt the +# inclusion measurement (full nodes are not on the measurement path). nodeSelector: local-ssd: "false" node-type: "network" pool: "spot" + cores: "4" affinity: nodeAffinity: diff --git a/spartan/terraform/gke-cluster/cluster/main.tf b/spartan/terraform/gke-cluster/cluster/main.tf index 5d18eda4ecb4..c94c434e3e27 100644 --- a/spartan/terraform/gke-cluster/cluster/main.tf +++ b/spartan/terraform/gke-cluster/cluster/main.tf @@ -15,7 +15,7 @@ resource "google_container_cluster" "primary" { channel = "UNSPECIFIED" } - # NOTE: Enabling workload identity at the cluster level means new node pools may default to GKE_METADATA mode. + # NOTE: Enabling workload identity at the cluster level means new node pools may default to GKE_METADATA mode. dynamic "workload_identity_config" { for_each = var.enable_workload_identity ? [1] : [] content { @@ -344,6 +344,52 @@ resource "google_container_node_pool" "spot_nodes_2core" { } } +# Create 4 core spot instance node pool with autoscaling +resource "google_container_node_pool" "spot_nodes_4core" { + name = "${var.cluster_name}-4core-spot" + location = var.zone + cluster = var.cluster_name + version = var.node_version + # Enable autoscaling + autoscaling { + min_node_count = 0 + max_node_count = 1500 + } + + # Node configuration + node_config { + machine_type = "t2d-standard-4" + spot = true + + service_account = var.service_account + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + env = "production" + pool = "spot" + local-ssd = "false" + node-type = "network" + cores = "4" + } + tags = ["aztec-gke-node", "spot"] + + # Spot instance termination handler + taint { + key = "cloud.google.com/gke-spot" + value = "true" + effect = "NO_SCHEDULE" + } + } + + # Management configuration + management { + auto_repair = true + auto_upgrade = false + } +} + # Create 8 core high memory (64 GB) node pool with autoscaling, used for metrics resource "google_container_node_pool" "infra_nodes_8core_highmem" { name = "${var.cluster_name}-infra-8core-hi-mem" From b931bff7e75492bec6735eaf247220860cfe466c Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 15:44:46 +0000 Subject: [PATCH 17/39] ci(bench): retire nightly-bench-10tps; point inclusion-sweep env at 72s slots The inclusion sweep (1/5/10 TPS) is a superset of the single-point 10 TPS nightly (it reuses the bench-10tps load model), so retire nightly-bench-10tps.yml and let nightly-bench-inclusion-sweep.yml be the nightly TPS/inclusion benchmark. Rebase bench-inclusion-sweep.env on the smoke env (same 12x4 validator / 10 RPC / 10 full-node topology on dedicated guaranteed-QoS bench profiles, RPC ingress off, fake proving) but at a representative 72s slot duration and 800 tx per checkpoint. 8-slot epochs keep nightly committee warm-up short. Test duration is already 10 min (bench_inclusion_point default TEST_DURATION_SECONDS=600; the workflow does not override it). --- .github/workflows/nightly-bench-10tps.yml | 208 ---------------------- 1 file changed, 208 deletions(-) delete mode 100644 .github/workflows/nightly-bench-10tps.yml diff --git a/.github/workflows/nightly-bench-10tps.yml b/.github/workflows/nightly-bench-10tps.yml deleted file mode 100644 index 9f85e2a7eaed..000000000000 --- a/.github/workflows/nightly-bench-10tps.yml +++ /dev/null @@ -1,208 +0,0 @@ -name: Nightly Bench 10 TPS - -on: - schedule: - - cron: "30 6 * * *" - workflow_dispatch: - inputs: - docker_image: - description: "Full Docker image (e.g., aztecprotocol/aztec:my-tag). Takes precedence over nightly_tag." - required: false - type: string - nightly_tag: - description: "Nightly tag to use (e.g., 2.3.4-nightly.20251209). Ignored if docker_image is set. Leave empty to auto-detect." - required: false - type: string - source_ref: - description: "Git ref to checkout (e.g., v5.0.0-nightly.20260529). Required when docker_image is not a standard nightly tag." - required: false - type: string - -concurrency: - group: nightly-bench-10tps-${{ github.ref }} - cancel-in-progress: true - -jobs: - select-image: - runs-on: ubuntu-latest - outputs: - docker_image: ${{ steps.docker-image.outputs.docker_image }} - image_label: ${{ steps.docker-image.outputs.image_label }} - source_ref: ${{ steps.docker-image.outputs.source_ref }} - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: next - - - name: Determine docker image - id: docker-image - run: | - if [[ -n "${{ inputs.docker_image }}" ]]; then - docker_image="${{ inputs.docker_image }}" - image_label="${{ inputs.docker_image }}" - image_tag="${docker_image##*:}" - elif [[ -n "${{ inputs.nightly_tag }}" ]]; then - nightly_tag="${{ inputs.nightly_tag }}" - docker_image="aztecprotocol/aztec:${nightly_tag}" - image_label="${nightly_tag}" - image_tag="${nightly_tag}" - else - current_version=$(jq -r '."."' .release-please-manifest.json) - nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)" - docker_image="aztecprotocol/aztec:${nightly_tag}" - image_label="${nightly_tag}" - image_tag="${nightly_tag}" - fi - - if [[ -n "${{ inputs.source_ref }}" ]]; then - source_ref="${{ inputs.source_ref }}" - elif [[ "${image_tag:-}" =~ ^[0-9]+\.[0-9]+\.[0-9]+-nightly\.[0-9]{8}$ ]]; then - source_ref="v${image_tag}" - else - echo "Could not infer git ref from docker image. Pass workflow input source_ref." - exit 1 - fi - - echo "docker_image=$docker_image" >> "$GITHUB_OUTPUT" - echo "image_label=$image_label" >> "$GITHUB_OUTPUT" - echo "source_ref=$source_ref" >> "$GITHUB_OUTPUT" - echo "Using docker image: $docker_image" - echo "Using source ref: $source_ref" - - - name: Verify source git ref - id: verify-source - run: | - set -euo pipefail - source_ref="${{ steps.docker-image.outputs.source_ref }}" - git fetch --depth 1 origin "refs/tags/${source_ref}:refs/tags/${source_ref}" - source_sha=$(git rev-parse "${source_ref}^{}") - echo "Nightly source commit: $source_sha" - - - name: Check if Docker image exists - run: | - DOCKER_IMAGE="${{ steps.docker-image.outputs.docker_image }}" - echo "Checking if Docker image exists: $DOCKER_IMAGE" - if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then - echo "Docker image exists: $DOCKER_IMAGE" - else - echo "Docker image does not exist: $DOCKER_IMAGE" - exit 1 - fi - - deploy-bench-10tps-network: - needs: select-image - uses: ./.github/workflows/deploy-network.yml - with: - network: bench-10tps - namespace: bench-10tps - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false - secrets: inherit - - wait-for-first-l2-block: - needs: - - select-image - - deploy-bench-10tps-network - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - with: - install_components: gke-gcloud-auth-plugin - - - name: Wait for first L2 block - env: - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - run: | - cd spartan - ./bootstrap.sh wait_for_l2_block bench-10tps - - benchmark: - needs: - - select-image - - wait-for-first-l2-block - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - - name: Run 10 TPS benchmark - timeout-minutes: 240 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - RUN_ID: ${{ github.run_id }} - AWS_SHUTDOWN_TIME: 240 - NO_SPOT: 1 - SKIP_NETWORK_DEPLOY: "1" - run: | - ./.github/ci3.sh network-bench-10tps bench-10tps bench-10tps "${{ needs.select-image.outputs.docker_image }}" - - cleanup: - if: always() - needs: - - select-image - - deploy-bench-10tps-network - - wait-for-first-l2-block - - benchmark - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - - name: Cleanup network resources - uses: ./.github/actions/network-teardown - with: - env_file: bench-10tps - namespace: bench-10tps - gcp_sa_key: ${{ secrets.GCP_SA_KEY }} - gcp_project_id: ${{ secrets.GCP_PROJECT_ID }} - - notify-failure: - if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }} - needs: - - select-image - - deploy-bench-10tps-network - - wait-for-first-l2-block - - benchmark - - cleanup - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - - name: Notify Slack on failure - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - run: | - IMAGE="${{ needs.select-image.outputs.image_label || 'unknown' }}" - RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - ./ci3/slack_notify_with_claudebox_kickoff "#alerts-next-scenario" \ - "Nightly 10 TPS benchmark FAILED (image ${IMAGE}): <${RUN_URL}|View Run> (🤖)" \ - "Nightly 10 TPS benchmark failed (image ${IMAGE}). CI run: ${RUN_URL}. Investigate the benchmark failure and identify the root cause." \ - --link "$RUN_URL" From d9d47b8c13e383324520e062d5b769e086fa5b82 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 15:45:34 +0000 Subject: [PATCH 18/39] feat(bench): rebase inclusion-sweep env on smoke topology at 72s slots Replace bench-inclusion-sweep.env with the smoke env's topology (12x4 validators, 10 RPC, 10 full nodes on dedicated guaranteed-QoS bench profiles, RPC ingress off, fake proving, 8-slot epochs) but at a representative 72s slot duration and 800 tx per checkpoint. Drops the prior production timing / RPC LoadBalancer / per-block cap that the sweep no longer uses. --- .../environments/bench-inclusion-sweep.env | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/spartan/environments/bench-inclusion-sweep.env b/spartan/environments/bench-inclusion-sweep.env index ffbb9d63205c..cff4d89d7463 100644 --- a/spartan/environments/bench-inclusion-sweep.env +++ b/spartan/environments/bench-inclusion-sweep.env @@ -1,11 +1,4 @@ -# Set A — inclusion sweep (Linear A-1223). -# One point of the 1/5/10 TPS inclusion sweep: bench-10tps's custom scrape -# pipeline + a production-scale topology (from tps-scenario), on a local -# eth-devnet with fake proving so proving never bottlenecks inclusion. -# -# The target TPS and sweep grouping are NOT set here — they are passed per point -# by the workflow via TARGET_TPS / BENCH_SWEEP_ID / BENCH_SWEEP_LABEL, and the -# namespace is overridden per point (one run per namespace, per the scraper). +# Set A inclusion sweep — nightly benchmark network. NAMESPACE=${NAMESPACE:-bench-inclusion-sweep} CLUSTER=aztec-gke-private RESOURCE_PROFILE=prod @@ -21,24 +14,19 @@ FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf CREATE_ROLLUP_CONTRACTS=true VERIFY_CONTRACTS=false -# Production-like epoch timing — do NOT shorten (keeps load realistic). -AZTEC_EPOCH_DURATION=32 +AZTEC_EPOCH_DURATION=8 AZTEC_SLOT_DURATION=72 -AZTEC_PROOF_SUBMISSION_EPOCHS=2 +AZTEC_PROOF_SUBMISSION_EPOCHS=100 AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 AZTEC_INBOX_LAG=2 -# 2B mana target - good for about ~800 txs at 2.5M mana each AZTEC_MANA_TARGET=2000000000 SPONSORED_FPC=true OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET -# Large topology from tps-scenario.env: 12x4 = 48 validators, 10 RPC, 150 full -# nodes. Full nodes go on spot (full-node-resources-spot.yaml); validators / RPC -# / prover stay on-demand so inclusion is not corrupted by mid-block eviction. VALIDATOR_REPLICAS=12 VALIDATORS_PER_NODE=4 VALIDATOR_PUBLISHERS_PER_REPLICA=4 @@ -47,8 +35,6 @@ VALIDATOR_RESOURCE_PROFILE="bench" VALIDATOR_HA_REPLICAS=0 SEQ_BLOCK_DURATION_MS=6000 -SEQ_L1_PUBLISHING_TIME_ALLOWANCE_IN_SLOT=36 -SEQ_MAX_TX_PER_BLOCK=200 SEQ_MAX_TX_PER_CHECKPOINT=800 P2P_MAX_PENDING_TX_COUNT=20000 SEQ_MIN_TX_PER_BLOCK=1 @@ -56,24 +42,18 @@ SEQ_BUILD_CHECKPOINT_IF_EMPTY=true RPC_REPLICAS=10 RPC_RESOURCE_PROFILE="bench" -# Load-balanced external IP so the test's per-wallet clients spread across the 10 -# RPC pods. With ingress disabled the test hits a single-pod port-forward, so the -# extra replicas would sit idle behind one pod (the ingress bottleneck seen at -# 5/10 TPS on the smoke net). -RPC_INGRESS_ENABLED=true +RPC_INGRESS_ENABLED=false FULL_NODE_REPLICAS=10 FULL_NODE_RESOURCE_PROFILE="bench" -# Fake proving (fixed, cheap delay) — this run does not measure proving; it only -# must not bottleneck inclusion (proving sweep is Set B / A-1224). REAL_VERIFIER=false PROVER_RESOURCE_PROFILE="dev-hi-tps" PUBLISHERS_PER_PROVER=1 PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 PROVER_AGENT_POLL_INTERVAL_MS=10000 PROVER_TEST_DELAY_TYPE=fixed -PROVER_TEST_VERIFICATION_DELAY_MS=250 +PROVER_TEST_VERIFICATION_DELAY_MS=60 PROVER_AGENT_KEDA_ENABLED=true PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET PROVER_AGENT_KEDA_MIN_REPLICAS=0 @@ -86,13 +66,12 @@ PROVER_AGENT_KEDA_SCALING_BANDS='[ ]' AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1 -AZTEC_SLASHING_QUORUM=20 +AZTEC_SLASHING_QUORUM=5 AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0 AZTEC_SLASHING_OFFSET_IN_ROUNDS=1 AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000 DEBUG_P2P_INSTRUMENT_MESSAGES=true -# Reduce metrics volume from the 150 full nodes (mirrors tps-scenario). FULL_NODE_INCLUDE_METRICS="aztec.p2p.gossip.agg_" LOG_LEVEL='info;debug:simulator:public-processor,sequencer:state,sequencer:checkpoint-events' @@ -102,3 +81,5 @@ PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 RUN_TESTS=false + +P2P_TX_POOL_DELETE_TXS_AFTER_REORG=true From 647503f28af31ac8ce18a68ab3f4eaeb98c4414f Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 17:30:54 +0000 Subject: [PATCH 19/39] refactor(ci): extract reusable bench-inclusion-point workflow The inclusion-sweep workflow repeated the deploy -> wait -> bench -> teardown block once per TPS point (~250 lines of near-duplicate jobs). Extract it into a reusable workflow (bench-inclusion-point.yml, workflow_call) parametrized by tps/namespace/image/ref/sweep_id, and call it 3x from the sweep with needs chaining for the sequential one-network-at-a-time ordering. An `if: !cancelled` guard on the later points preserves point independence (a failed point drops only itself). Sweep workflow drops 357 -> 161 lines; per-point logic defined once. --- .github/workflows/bench-inclusion-point.yml | 112 ++++++++ .../nightly-bench-inclusion-sweep.yml | 255 +++--------------- 2 files changed, 144 insertions(+), 223 deletions(-) create mode 100644 .github/workflows/bench-inclusion-point.yml diff --git a/.github/workflows/bench-inclusion-point.yml b/.github/workflows/bench-inclusion-point.yml new file mode 100644 index 000000000000..67d2be684af5 --- /dev/null +++ b/.github/workflows/bench-inclusion-point.yml @@ -0,0 +1,112 @@ +name: Bench Inclusion Point + +# Reusable: runs ONE inclusion-sweep TPS point end-to-end — deploy a fresh +# network, wait for the first L2 block, run the bench point, then tear the +# network down (always). Called once per point (1/5/10 TPS) by +# nightly-bench-inclusion-sweep.yml, which chains the calls so only one network +# exists at a time. + +on: + workflow_call: + inputs: + tps: + description: "Target TPS for this point" + required: true + type: string + namespace: + description: "k8s namespace for this point's network" + required: true + type: string + network: + description: "Network env name (environments/.env)" + required: false + type: string + default: bench-inclusion-sweep + docker_image: + description: "Full aztec docker image to deploy" + required: true + type: string + source_ref: + description: "Git ref to checkout for the bench/teardown scripts" + required: true + type: string + sweep_id: + description: "Shared sweep id so the dashboard groups the points" + required: true + type: string + +jobs: + deploy: + uses: ./.github/workflows/deploy-network.yml + with: + network: ${{ inputs.network }} + namespace: ${{ inputs.namespace }} + aztec_docker_image: ${{ inputs.docker_image }} + ref: ${{ inputs.source_ref }} + notify_on_failure: false + secrets: inherit + + wait: + needs: deploy + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ inputs.source_ref }} + - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + with: + install_components: gke-gcloud-auth-plugin + - name: Wait for first L2 block + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NAMESPACE: ${{ inputs.namespace }} + run: cd spartan && ./bootstrap.sh wait_for_l2_block ${{ inputs.network }} + + bench: + needs: wait + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ inputs.source_ref }} + - name: Run ${{ inputs.tps }} TPS inclusion point + timeout-minutes: 120 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + RUN_ID: ${{ github.run_id }} + AWS_SHUTDOWN_TIME: 180 + NO_SPOT: 1 + SKIP_NETWORK_DEPLOY: "1" + TARGET_TPS: ${{ inputs.tps }} + BENCH_SWEEP_ID: ${{ inputs.sweep_id }} + BENCH_SWEEP_LABEL: inclusion-sweep + run: ./.github/ci3.sh network-inclusion-sweep ${{ inputs.network }} ${{ inputs.namespace }} "${{ inputs.docker_image }}" + + cleanup: + if: always() + needs: [deploy, wait, bench] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ inputs.source_ref }} + - name: Cleanup network resources + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NO_SPOT: 1 + run: ./.github/ci3.sh network-teardown ${{ inputs.network }} ${{ inputs.namespace }} diff --git a/.github/workflows/nightly-bench-inclusion-sweep.yml b/.github/workflows/nightly-bench-inclusion-sweep.yml index 579d9fafbd01..875ceeacc4f1 100644 --- a/.github/workflows/nightly-bench-inclusion-sweep.yml +++ b/.github/workflows/nightly-bench-inclusion-sweep.yml @@ -3,8 +3,9 @@ name: Nightly Bench Inclusion Sweep # Set A inclusion sweep (Linear A-1223): runs the 1/5/10 TPS points # SEQUENTIALLY (one ~150-full-node network at a time to bound cluster load), # each in its own namespace, all tagged with a shared sweepId so the dashboard -# groups them. Each point: deploy -> wait -> bench -> teardown; the next point's -# deploy waits on the previous point's teardown. +# groups them. Each point (deploy -> wait -> bench -> teardown) is the reusable +# bench-inclusion-point.yml workflow; the next point's call `needs` the previous +# one, so deploy N+1 only starts after point N's teardown. on: schedule: @@ -100,244 +101,52 @@ jobs: # Shared across all three points so the dashboard groups the sweep. echo "sweep_id=incl-$(date -u +%Y%m%d)-${{ github.run_id }}" >> "$GITHUB_OUTPUT" - # ---- Point: 1 TPS ---- - deploy-1tps: + point-1tps: needs: select-image - uses: ./.github/workflows/deploy-network.yml + uses: ./.github/workflows/bench-inclusion-point.yml with: - network: bench-inclusion-sweep + tps: "1" namespace: bench-incl-sweep-1tps - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + docker_image: ${{ needs.select-image.outputs.docker_image }} + source_ref: ${{ needs.select-image.outputs.source_ref }} + sweep_id: ${{ needs.select-image.outputs.sweep_id }} secrets: inherit - wait-1tps: - needs: [select-image, deploy-1tps] - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - with: - install_components: gke-gcloud-auth-plugin - - name: Wait for first L2 block - env: - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NAMESPACE: bench-incl-sweep-1tps - run: cd spartan && ./bootstrap.sh wait_for_l2_block bench-inclusion-sweep - - bench-1tps: - needs: [select-image, wait-1tps] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - name: Run 1 TPS inclusion point - timeout-minutes: 120 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - RUN_ID: ${{ github.run_id }} - AWS_SHUTDOWN_TIME: 180 - NO_SPOT: 1 - SKIP_NETWORK_DEPLOY: "1" - TARGET_TPS: "1" - BENCH_SWEEP_ID: ${{ needs.select-image.outputs.sweep_id }} - BENCH_SWEEP_LABEL: inclusion-sweep - run: ./.github/ci3.sh network-inclusion-sweep bench-inclusion-sweep bench-incl-sweep-1tps "${{ needs.select-image.outputs.docker_image }}" - - cleanup-1tps: - if: always() - needs: [select-image, deploy-1tps, wait-1tps, bench-1tps] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - name: Cleanup network resources - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: ./.github/ci3.sh network-teardown bench-inclusion-sweep bench-incl-sweep-1tps - - # ---- Point: 5 TPS (starts after 1 TPS teardown) ---- - deploy-5tps: - needs: [select-image, cleanup-1tps] - uses: ./.github/workflows/deploy-network.yml + # 5 TPS starts only after the 1 TPS point finishes (incl. its teardown), but + # runs regardless of whether 1 TPS passed — a failed point drops only itself. + point-5tps: + needs: [select-image, point-1tps] + if: ${{ !cancelled() && needs.select-image.result == 'success' }} + uses: ./.github/workflows/bench-inclusion-point.yml with: - network: bench-inclusion-sweep + tps: "5" namespace: bench-incl-sweep-5tps - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + docker_image: ${{ needs.select-image.outputs.docker_image }} + source_ref: ${{ needs.select-image.outputs.source_ref }} + sweep_id: ${{ needs.select-image.outputs.sweep_id }} secrets: inherit - wait-5tps: - needs: [select-image, deploy-5tps] - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - with: - install_components: gke-gcloud-auth-plugin - - name: Wait for first L2 block - env: - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NAMESPACE: bench-incl-sweep-5tps - run: cd spartan && ./bootstrap.sh wait_for_l2_block bench-inclusion-sweep - - bench-5tps: - needs: [select-image, wait-5tps] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - name: Run 5 TPS inclusion point - timeout-minutes: 120 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - RUN_ID: ${{ github.run_id }} - AWS_SHUTDOWN_TIME: 180 - NO_SPOT: 1 - SKIP_NETWORK_DEPLOY: "1" - TARGET_TPS: "5" - BENCH_SWEEP_ID: ${{ needs.select-image.outputs.sweep_id }} - BENCH_SWEEP_LABEL: inclusion-sweep - run: ./.github/ci3.sh network-inclusion-sweep bench-inclusion-sweep bench-incl-sweep-5tps "${{ needs.select-image.outputs.docker_image }}" - - cleanup-5tps: - if: always() - needs: [select-image, deploy-5tps, wait-5tps, bench-5tps] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - name: Cleanup network resources - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: ./.github/ci3.sh network-teardown bench-inclusion-sweep bench-incl-sweep-5tps - - # ---- Point: 10 TPS (starts after 5 TPS teardown) ---- - deploy-10tps: - needs: [select-image, cleanup-5tps] - uses: ./.github/workflows/deploy-network.yml + # 10 TPS starts only after the 5 TPS point finishes (incl. its teardown), but + # runs regardless of whether 5 TPS passed — a failed point drops only itself. + point-10tps: + needs: [select-image, point-5tps] + if: ${{ !cancelled() && needs.select-image.result == 'success' }} + uses: ./.github/workflows/bench-inclusion-point.yml with: - network: bench-inclusion-sweep + tps: "10" namespace: bench-incl-sweep-10tps - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + docker_image: ${{ needs.select-image.outputs.docker_image }} + source_ref: ${{ needs.select-image.outputs.source_ref }} + sweep_id: ${{ needs.select-image.outputs.sweep_id }} secrets: inherit - wait-10tps: - needs: [select-image, deploy-10tps] - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - with: - install_components: gke-gcloud-auth-plugin - - name: Wait for first L2 block - env: - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NAMESPACE: bench-incl-sweep-10tps - run: cd spartan && ./bootstrap.sh wait_for_l2_block bench-inclusion-sweep - - bench-10tps: - needs: [select-image, wait-10tps] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - name: Run 10 TPS inclusion point - timeout-minutes: 120 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - RUN_ID: ${{ github.run_id }} - AWS_SHUTDOWN_TIME: 180 - NO_SPOT: 1 - SKIP_NETWORK_DEPLOY: "1" - TARGET_TPS: "10" - BENCH_SWEEP_ID: ${{ needs.select-image.outputs.sweep_id }} - BENCH_SWEEP_LABEL: inclusion-sweep - run: ./.github/ci3.sh network-inclusion-sweep bench-inclusion-sweep bench-incl-sweep-10tps "${{ needs.select-image.outputs.docker_image }}" - - cleanup-10tps: - if: always() - needs: [select-image, deploy-10tps, wait-10tps, bench-10tps] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ needs.select-image.outputs.source_ref }} - - name: Cleanup network resources - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: ./.github/ci3.sh network-teardown bench-inclusion-sweep bench-incl-sweep-10tps - notify-failure: if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }} needs: - select-image - - bench-1tps - - bench-5tps - - bench-10tps + - point-1tps + - point-5tps + - point-10tps runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 From 6a55f274d149ed06635e52dddbd2db95db965c0f Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 17:33:59 +0000 Subject: [PATCH 20/39] chore(bench): remove 10tps-readiness-spec doc --- .../bench_10tps/10tps-readiness-spec.md | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100644 spartan/scripts/bench_10tps/10tps-readiness-spec.md diff --git a/spartan/scripts/bench_10tps/10tps-readiness-spec.md b/spartan/scripts/bench_10tps/10tps-readiness-spec.md deleted file mode 100644 index 60e620f09d58..000000000000 --- a/spartan/scripts/bench_10tps/10tps-readiness-spec.md +++ /dev/null @@ -1,69 +0,0 @@ -# 10 TPS readiness benchmark — spec (schema v4) - -Canonical contract for the custom benchmark pipeline: -`bench_scrape.ts` → `gs://aztec-testnet/network_bench/.json` (+ `index.json`) → `network-dashboard` (in `AztecProtocol/explorations`). - -This doc is the Phase 1 deliverable (Linear A-1221). It defines the tx-lifecycle stage list, the headline KPIs and their thresholds, and the sweep/run-group notion. The machine-readable contract is `bench_output.schema.json` (v4); the scraper that produces it is `bench_scrape.ts`. - -## 1. tx-lifecycle stage waterfall - -A tx's journey from client submit to epoch proof, each stage mapped to the Prometheus metric (from `yarn-project/telemetry-client/src/metrics.ts`) and where it lands in the run JSON. "ms" durations are histograms (use `_bucket` for quantiles, `_sum`/`_count` for means); never `sum(rate(...))` a metric every node observes — collapse per role with `avg`/`max` (see `network-dashboard/docs/dashboard-design.md`). - -| # | Stage | Primary metric(s) | Run-JSON location | -|---|---|---|---| -| 1 | Submit / ingest | `aztec.node.receive_tx_count` (RPC only — load hits one node) | `timeSeries.ingressTps` | -| 2 | P2P propagation | `aztec.p2p.gossip.message_latency`, `agg_message_latency_p50/p90`, `tx_received_count` | `timeSeries` (gossip) | -| 3 | Mempool wait | `aztec.mempool.tx_count` (pending depth), `aztec.mempool.tx_mined_delay` | `timeSeries.mempoolSize*`, `mempoolMinedMax` | -| 4 | Block build | `aztec.sequencer.block.build_duration`, `build_mana_per_second` | `timeSeries`, `sequencerStateSlots` | -| 5 | Public processing | `aztec.public_processor.tx_duration`, `phase_duration`, `gas_rate` | `timeSeries.publicProcessorGasRate`; **prover-node copy** in `provingInfra.hintGen*` | -| 6 | Attestation / consensus | `aztec.mempool.attestations_mined_delay`; attestation-collect duration vs slot allowance | `timeSeries.attestationsCollect*` | -| 7 | Checkpoint assemble → broadcast | `aztec.archiver.checkpoint_height`, checkpoint block/tx counts | `timeSeries.checkpoint*` | -| 8 | L1 inclusion | `aztec.archiver.checkpoint_l1_inclusion_delay`, `l1_block_height`, `block_height` | `timeSeries`, `blocks` | -| 9 | Proving (epoch) | `aztec.prover_node.checkpoint_proving.duration`, `aztec.archiver.rollup_proof_delay`, `aztec.proving_queue.*` by `job_type`, prover-node block/checkpoint processing | `provingInfra.*` | - -**Authoritative user-perceived latency** is client-observed, not Prometheus: `n_tps_test.tx_inclusion_time` (`timeSeries`, `source: client_observed`) — the wall-clock submit→mined delta for high-value txs, computed in the scraper from `n_tps.test.ts` records. Stages 1–8 explain *where* that latency is spent; stage 9 is the separate proving path. - -## 2. Headline KPIs + pass/fail thresholds - -Two independent verdicts — a run can pass inclusion and fail proving (or vice versa). Thresholds are starting points to refine against baselines; encode them in the dashboard, not the scraper (the scraper stays a faithful recorder). - -| KPI | Definition | Pass threshold | -|---|---|---| -| **Inclusion-TPS achieved / target** | `summary.inclusionTpsMean / run.targetTps` | ≥ 0.95 | -| **User-perceived inclusion latency p50** | p50 of `tx_inclusion_time` | ≤ 1 × `AZTEC_SLOT_DURATION` | -| **User-perceived inclusion latency p99** | p99 of `tx_inclusion_time` | ≤ 3 × `AZTEC_SLOT_DURATION` | -| **Proving headroom** | does each epoch's proof land within `AZTEC_PROOF_SUBMISSION_EPOCHS` of epoch close (no proof-window-expiry reorg)? | every epoch proven in window; `reorgCount` from window-expiry = 0 | -| **Reorgs** | `summary.reorgCount` | 0 | - -"Proving headroom" is the proving knee = distance from 10 TPS: the highest `targetTps` at which every epoch still proves within its window. Below the knee, headroom is positive; above it, epochs miss the window and the pending chain is pruned (the run #95 failure mode). - -## 3. Sweep / run-group notion - -A night's 1/5/10 TPS points are distinct runs (distinct namespaces — queries are namespace-scoped, one run per namespace) that the dashboard must view together. Grouping fields (schema v4, on `run` + mirrored into `index.json`): - -- `run.sweepId` — shared id across the points of one sweep (e.g. `incl-20260623`). Set via `--sweep-id` / `BENCH_SWEEP_ID`. -- `run.sweepLabel` — human label (e.g. `inclusion-sweep`, `proving-sweep`). `--sweep-label` / `BENCH_SWEEP_LABEL`. -- `run.targetTps` — the point within the sweep (already present in v3). - -`index.json` entries carry `sweepId`/`sweepLabel`/`targetTps` so the dashboard can group + order points without fetching every run JSON. - -## 4. schema v4 additions (additive over v3) - -All v3 fields retained; a v3-shaped run re-stamped `"4"` still validates (the new sections are optional). New: - -- `provingInfra` (`metricSeriesMap`): prover-node hint-gen (`public_processor.*` + `prover_node.*_processing.duration` scoped to the prover-node pod) and proving-queue series broken down by `aztec_proving_job_type` (size / active / job_duration p50·p99 / timed-out · resolved rates). **Note:** there is no `aztec.prover_node.execution.duration` metric — hint-gen is the `public_processor.*` re-execution on the prover-node pod, mapped accordingly. -- `saturation` (`metricSeriesMap`): per-role ELU / CPU / memory, each as **max (hottest pod)** and **avg**, for validator / rpc / fullNode / proverNode / broker / agent. Never a single hand-picked pod. ELU = `nodejs_eventloop_utilization`, memory = `nodejs_memory_v8_heap_usage` (both `nodejs.*`, not `aztec_`); CPU = `process_cpu_utilization` (from `@opentelemetry/host-metrics`). -- `run.sweepId` / `run.sweepLabel` (§3). - -### Version gate — three places, must stay in sync - -Bumping the schema version requires updating all three or v4 runs are silently rejected / mis-rendered: - -1. `bench_output.schema.json` — `schemaVersion.const` (✅ `"4"`). -2. `spartan/bootstrap.sh` — `network_bench_upload` schemaVersion check (✅ `"4"`). -3. **`network-dashboard/data.js` `SUPPORTED_RUN_VERSION`** — in `AztecProtocol/explorations`, **not this repo**. Must be bumped to `"4"` there before v4 runs render. Tracked as dashboard work (Phase 5). - -## 5. Verify-on-live caveats (A-1222 acceptance) - -- **CPU** (`process_cpu_utilization`) and **ELU** (`nodejs_eventloop_utilization`) come from telemetry that may be gated in the bench env. The scraper emits empty series (non-fatal) if a metric is absent; confirm both flow on a live bench run and fix the exporter/metric name if not. -- Proving-infra durations are recorded in **ms** by convention; confirm units against the live histograms before trusting absolute values. From e72a95364012758a21751c21bbae92516816062b Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 17:51:07 +0000 Subject: [PATCH 21/39] refactor(ci): inclusion sweep deploys once, runs 1/5/10 on the same network Previously each TPS point deployed and tore down its own network (3x deploy + teardown). Switch to deploying a single network, running the 1/5/10 points sequentially against it (the scraper drains the mempool to zero between points), and tearing it down once. Cheaper, faster, and matches how it's run locally. deploy/wait/teardown move to the top-level workflow (run once); the reusable bench-inclusion-point.yml is now just the bench step (one point, against the existing network, SKIP_NETWORK_DEPLOY=1), called 3x with needs chaining so the points don't overlap. Later points are gated on `wait` success so a failed point still drops only itself. --- .github/workflows/bench-inclusion-point.yml | 66 ++------------ .../nightly-bench-inclusion-sweep.yml | 91 +++++++++++++++---- 2 files changed, 80 insertions(+), 77 deletions(-) diff --git a/.github/workflows/bench-inclusion-point.yml b/.github/workflows/bench-inclusion-point.yml index 67d2be684af5..3e26f8ea385b 100644 --- a/.github/workflows/bench-inclusion-point.yml +++ b/.github/workflows/bench-inclusion-point.yml @@ -1,10 +1,10 @@ name: Bench Inclusion Point -# Reusable: runs ONE inclusion-sweep TPS point end-to-end — deploy a fresh -# network, wait for the first L2 block, run the bench point, then tear the -# network down (always). Called once per point (1/5/10 TPS) by -# nightly-bench-inclusion-sweep.yml, which chains the calls so only one network -# exists at a time. +# Reusable: runs ONE inclusion-sweep TPS point against an ALREADY-DEPLOYED +# network (SKIP_NETWORK_DEPLOY=1). The caller (nightly-bench-inclusion-sweep.yml) +# deploys the network once and calls this for each point (1/5/10 TPS), chaining +# the calls so the points run sequentially on the same network. Deploy / wait / +# teardown are done once by the caller, not here. on: workflow_call: @@ -14,7 +14,7 @@ on: required: true type: string namespace: - description: "k8s namespace for this point's network" + description: "k8s namespace of the already-deployed network" required: true type: string network: @@ -23,11 +23,11 @@ on: type: string default: bench-inclusion-sweep docker_image: - description: "Full aztec docker image to deploy" + description: "Full aztec docker image (already deployed)" required: true type: string source_ref: - description: "Git ref to checkout for the bench/teardown scripts" + description: "Git ref to checkout for the bench scripts" required: true type: string sweep_id: @@ -36,38 +36,7 @@ on: type: string jobs: - deploy: - uses: ./.github/workflows/deploy-network.yml - with: - network: ${{ inputs.network }} - namespace: ${{ inputs.namespace }} - aztec_docker_image: ${{ inputs.docker_image }} - ref: ${{ inputs.source_ref }} - notify_on_failure: false - secrets: inherit - - wait: - needs: deploy - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ inputs.source_ref }} - - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - with: - install_components: gke-gcloud-auth-plugin - - name: Wait for first L2 block - env: - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NAMESPACE: ${{ inputs.namespace }} - run: cd spartan && ./bootstrap.sh wait_for_l2_block ${{ inputs.network }} - bench: - needs: wait runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 @@ -91,22 +60,3 @@ jobs: BENCH_SWEEP_ID: ${{ inputs.sweep_id }} BENCH_SWEEP_LABEL: inclusion-sweep run: ./.github/ci3.sh network-inclusion-sweep ${{ inputs.network }} ${{ inputs.namespace }} "${{ inputs.docker_image }}" - - cleanup: - if: always() - needs: [deploy, wait, bench] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: ${{ inputs.source_ref }} - - name: Cleanup network resources - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - NO_SPOT: 1 - run: ./.github/ci3.sh network-teardown ${{ inputs.network }} ${{ inputs.namespace }} diff --git a/.github/workflows/nightly-bench-inclusion-sweep.yml b/.github/workflows/nightly-bench-inclusion-sweep.yml index 875ceeacc4f1..23a97474ed18 100644 --- a/.github/workflows/nightly-bench-inclusion-sweep.yml +++ b/.github/workflows/nightly-bench-inclusion-sweep.yml @@ -1,11 +1,11 @@ name: Nightly Bench Inclusion Sweep -# Set A inclusion sweep (Linear A-1223): runs the 1/5/10 TPS points -# SEQUENTIALLY (one ~150-full-node network at a time to bound cluster load), -# each in its own namespace, all tagged with a shared sweepId so the dashboard -# groups them. Each point (deploy -> wait -> bench -> teardown) is the reusable -# bench-inclusion-point.yml workflow; the next point's call `needs` the previous -# one, so deploy N+1 only starts after point N's teardown. +# Set A inclusion sweep (Linear A-1223): deploys ONE network and runs the +# 1/5/10 TPS points SEQUENTIALLY against it, all tagged with a shared sweepId so +# the dashboard groups them. The scraper drains the mempool to zero between +# points. Each point is the reusable bench-inclusion-point.yml workflow +# (SKIP_NETWORK_DEPLOY=1); the next point's call `needs` the previous one so they +# don't overlap. Deploy / wait-for-first-block / teardown happen once. on: schedule: @@ -101,49 +101,102 @@ jobs: # Shared across all three points so the dashboard groups the sweep. echo "sweep_id=incl-$(date -u +%Y%m%d)-${{ github.run_id }}" >> "$GITHUB_OUTPUT" - point-1tps: + # ---- Deploy the single network the whole sweep runs against ---- + deploy: needs: select-image + uses: ./.github/workflows/deploy-network.yml + with: + network: bench-inclusion-sweep + namespace: bench-inclusion-sweep + aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + ref: ${{ needs.select-image.outputs.source_ref }} + notify_on_failure: false + secrets: inherit + + wait: + needs: [select-image, deploy] + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + with: + install_components: gke-gcloud-auth-plugin + - name: Wait for first L2 block + env: + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NAMESPACE: bench-inclusion-sweep + run: cd spartan && ./bootstrap.sh wait_for_l2_block bench-inclusion-sweep + + # ---- Three points, sequential, on the SAME network ---- + point-1tps: + needs: [select-image, wait] uses: ./.github/workflows/bench-inclusion-point.yml with: tps: "1" - namespace: bench-incl-sweep-1tps + namespace: bench-inclusion-sweep docker_image: ${{ needs.select-image.outputs.docker_image }} source_ref: ${{ needs.select-image.outputs.source_ref }} sweep_id: ${{ needs.select-image.outputs.sweep_id }} secrets: inherit - # 5 TPS starts only after the 1 TPS point finishes (incl. its teardown), but - # runs regardless of whether 1 TPS passed — a failed point drops only itself. + # 5 TPS runs after 1 TPS finishes — gated on the network being up (wait), not + # on the 1 TPS result, so a failed point drops only itself. point-5tps: - needs: [select-image, point-1tps] - if: ${{ !cancelled() && needs.select-image.result == 'success' }} + needs: [select-image, wait, point-1tps] + if: ${{ !cancelled() && needs.wait.result == 'success' }} uses: ./.github/workflows/bench-inclusion-point.yml with: tps: "5" - namespace: bench-incl-sweep-5tps + namespace: bench-inclusion-sweep docker_image: ${{ needs.select-image.outputs.docker_image }} source_ref: ${{ needs.select-image.outputs.source_ref }} sweep_id: ${{ needs.select-image.outputs.sweep_id }} secrets: inherit - # 10 TPS starts only after the 5 TPS point finishes (incl. its teardown), but - # runs regardless of whether 5 TPS passed — a failed point drops only itself. point-10tps: - needs: [select-image, point-5tps] - if: ${{ !cancelled() && needs.select-image.result == 'success' }} + needs: [select-image, wait, point-5tps] + if: ${{ !cancelled() && needs.wait.result == 'success' }} uses: ./.github/workflows/bench-inclusion-point.yml with: tps: "10" - namespace: bench-incl-sweep-10tps + namespace: bench-inclusion-sweep docker_image: ${{ needs.select-image.outputs.docker_image }} source_ref: ${{ needs.select-image.outputs.source_ref }} sweep_id: ${{ needs.select-image.outputs.sweep_id }} secrets: inherit + # ---- Tear the network down once, regardless of point outcomes ---- + cleanup: + if: always() + needs: [select-image, deploy, wait, point-1tps, point-5tps, point-10tps] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ needs.select-image.outputs.source_ref }} + - name: Cleanup network resources + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NO_SPOT: 1 + run: ./.github/ci3.sh network-teardown bench-inclusion-sweep bench-inclusion-sweep + notify-failure: if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' }} needs: - select-image + - deploy + - wait - point-1tps - point-5tps - point-10tps @@ -161,5 +214,5 @@ jobs: RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" ./ci3/slack_notify_with_claudebox_kickoff "#alerts-next-scenario" \ "Nightly inclusion sweep FAILED (image ${IMAGE}): <${RUN_URL}|View Run> (🤖)" \ - "Nightly inclusion sweep failed (image ${IMAGE}). CI run: ${RUN_URL}. A failed point drops only that point; check which TPS point(s) failed." \ + "Nightly inclusion sweep failed (image ${IMAGE}). CI run: ${RUN_URL}. Deploy/wait failures fail the whole sweep; a failed individual point drops only that point." \ --link "$RUN_URL" From 3ad26a8e2917bbd1517949e1ee9a9783c3f6a08e Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 19:21:16 +0000 Subject: [PATCH 22/39] feat(bench): record per-role node resources in inclusion-sweep JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capture each pod's aztec-container CPU/memory requests and limits during the infrastructure scrape, exposed as `resources` per node and a deduped `resourceProfiles` per role, so a run records the machine sizing the validators/rpc/full-node/prover pods actually got (request==limit for the guaranteed-QoS bench profiles). Also anchor the role-classification patterns to the namespace-stripped pod name so the L1 infra pods (eth-validator / eth-execution / eth-beacon) are no longer misread as aztec roles — an unanchored /-validator/ matched "-eth-validator-0" and polluted the validator role. --- .../bench_10tps/bench_output.schema.json | 29 ++++++++ spartan/scripts/bench_10tps/bench_scrape.ts | 71 +++++++++++++++++-- 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 652750906a14..86bee2025065 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -68,6 +68,25 @@ } }, "$defs": { + "containerResources": { + "type": "object", + "additionalProperties": false, + "description": "CPU/memory requests and limits of a pod's main aztec container, captured verbatim from the Kubernetes pod spec. Values are Kubernetes quantity strings (e.g. cpu '3500m', memory '12Gi').", + "properties": { + "requests": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "limits": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, "runMeta": { "type": "object", "additionalProperties": false, @@ -169,6 +188,13 @@ }, "description": "Distinct node pools used by this role, when exposed as node labels." }, + "resourceProfiles": { + "type": "array", + "description": "Distinct CPU/memory request+limit profiles of the role's pods (deduped by content). Normally one entry per role; a second entry indicates pods of this role were sized differently. For the bench profiles request==limit (guaranteed QoS).", + "items": { + "$ref": "#/$defs/containerResources" + } + }, "nodes": { "type": "array", "items": { @@ -190,6 +216,9 @@ }, "nodePool": { "type": "string" + }, + "resources": { + "$ref": "#/$defs/containerResources" } } } diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index fafd504fd011..d0c6d2f47b96 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -865,17 +865,27 @@ const timeFilter = (startedAt: string, endedAt: string) => // --- Run-context capture (image + aztec config env) --- +// CPU/memory requests and limits for a pod's main container, captured verbatim +// from the Kubernetes pod spec (e.g. {cpu: "3500m", memory: "12Gi"}). For the +// bench profiles request==limit (guaranteed QoS), so both are usually equal. +type ContainerResources = { + requests?: Record; + limits?: Record; +}; + type RoleNode = { role: string; podName: string; nodeName: string; instanceType?: string; nodePool?: string; + resources?: ContainerResources; }; type RoleInfrastructure = { instanceTypes: string[]; nodePools: string[]; + resourceProfiles: ContainerResources[]; nodes: RoleNode[]; }; @@ -883,11 +893,15 @@ type Infrastructure = { roles: Record; }; +// Patterns are matched against the pod name with the namespace prefix removed +// and anchored with ^, so the L1 infra pods (eth-validator / eth-execution / +// eth-beacon) are not misread as aztec roles — an unanchored /-validator/ would +// match "-eth-validator-0". const INFRASTRUCTURE_ROLE_PATTERNS = [ - { role: "validator", pattern: /-validator(?:-|$)/ }, - { role: "prover", pattern: /-prover-(?:agent|node|broker)(?:-|$)/ }, - { role: "rpc", pattern: /-rpc(?:-|$)/ }, - { role: "fullNode", pattern: /-full-node(?:-|$)/ }, + { role: "validator", pattern: /^validator(?:-|$)/ }, + { role: "prover", pattern: /^prover-(?:agent|node|broker)(?:-|$)/ }, + { role: "rpc", pattern: /^rpc(?:-|$)/ }, + { role: "fullNode", pattern: /^full-node(?:-|$)/ }, ]; // Curated subset of env vars worth recording per run so the dashboard can @@ -991,7 +1005,13 @@ async function captureInfrastructure(): Promise { const podsJson = JSON.parse(podsOut) as { items?: Array<{ metadata?: { name?: string }; - spec?: { nodeName?: string }; + spec?: { + nodeName?: string; + containers?: Array<{ + name?: string; + resources?: ContainerResources; + }>; + }; }>; }; const nodesJson = JSON.parse(nodesOut) as { @@ -1034,6 +1054,7 @@ async function captureInfrastructure(): Promise { nodePool: labels["cloud.google.com/gke-nodepool"] ?? labels["eks.amazonaws.com/nodegroup"], + resources: resourcesForPod(pod.spec?.containers), }; }) .filter((node): node is RoleNode => node !== undefined) @@ -1061,15 +1082,50 @@ async function captureInfrastructure(): Promise { } function roleForPodName(podName: string): string | undefined { - if (!podName.startsWith(`${NAMESPACE}-`)) { + const prefix = `${NAMESPACE}-`; + if (!podName.startsWith(prefix)) { return undefined; } + const suffix = podName.slice(prefix.length); return INFRASTRUCTURE_ROLE_PATTERNS.find(({ pattern }) => - pattern.test(podName), + pattern.test(suffix), )?.role; } +// Resource requests/limits of a pod's main "aztec" container (falls back to the +// first container if none is named "aztec"). Returns undefined when the spec +// declares no resources, so a pod without sizing is simply omitted. +function resourcesForPod( + containers: + | Array<{ name?: string; resources?: ContainerResources }> + | undefined, +): ContainerResources | undefined { + const container = + containers?.find((c) => c.name === "aztec") ?? containers?.[0]; + const resources = container?.resources; + if (!resources) { + return undefined; + } + const out: ContainerResources = {}; + if (resources.requests && Object.keys(resources.requests).length > 0) { + out.requests = resources.requests; + } + if (resources.limits && Object.keys(resources.limits).length > 0) { + out.limits = resources.limits; + } + return out.requests || out.limits ? out : undefined; +} + function infrastructureForNodes(nodes: RoleNode[]): RoleInfrastructure { + // Pods of a role normally share one resource profile, but dedupe by content + // so a mid-run resize or a stray pod surfaces as a second distinct profile + // rather than being silently collapsed. + const profilesByKey = new Map(); + for (const node of nodes) { + if (node.resources) { + profilesByKey.set(JSON.stringify(node.resources), node.resources); + } + } return { instanceTypes: Array.from( new Set( @@ -1079,6 +1135,7 @@ function infrastructureForNodes(nodes: RoleNode[]): RoleInfrastructure { nodePools: Array.from( new Set(nodes.flatMap((node) => (node.nodePool ? [node.nodePool] : []))), ).sort(), + resourceProfiles: [...profilesByKey.values()], nodes, }; } From 87922cadd5081a712ecd1c6cd2f7e2a28aca7ce3 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 19:29:47 +0000 Subject: [PATCH 23/39] feat(bench): scrape validator attestation-failure metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node emits aztec.validator.attestation_failed_node_issue_count (the attester-side timeout/missing-data counter that diagnosed the 10 TPS reorgs) alongside the bad-proposal and success counters, but the bench scraper never queried them — only the proposer-side collect-duration gauges reached the JSON. Add three timeSeries (node-issue and bad-proposal failures broken down by error_type, plus a success rate for a failure-ratio denominator) and three summary scalars totalling each outcome over the observed window, so the attestation node-issue count — the signal upstream of committee quorum loss — is recorded per run. --- .../bench_10tps/bench_output.schema.json | 21 ++++++++++ spartan/scripts/bench_10tps/bench_scrape.ts | 41 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 86bee2025065..32e2defe04fe 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -367,6 +367,18 @@ "mempoolTxMinedDelayP99Ms": { "type": ["number", "null"] }, + "attestationFailedNodeIssueCount": { + "type": ["number", "null"], + "description": "Total attester-side attestation failures over the observed window (aztec_validator_attestation_failed_node_issue_count: timeout / txs_not_available / parent_block_not_found / etc.). The reorg-diagnostic headline — high values mean attesters could not re-execute proposals in time, dragging the committee below quorum. null if the metric was unavailable." + }, + "attestationFailedBadProposalCount": { + "type": ["number", "null"], + "description": "Total attestations rejected over the window because the proposal was bad (invalid_proposal / state_mismatch / failed_txs / …), distinct from node-side issues." + }, + "attestationSuccessCount": { + "type": ["number", "null"], + "description": "Total successful attestations over the window. Denominator for an attestation failure ratio against the two failure counts above." + }, "totalTxsMined": { "type": ["integer", "null"], "description": "Exact sum from per-block logs. Null when block logs were unavailable and inclusionTpsMean came from Prometheus." @@ -479,6 +491,15 @@ "attestationsCollectAllowanceMean": { "$ref": "#/$defs/timeSeries" }, + "attestationFailedNodeIssueByErrorTypeRate": { + "$ref": "#/$defs/timeSeries" + }, + "attestationFailedBadProposalByErrorTypeRate": { + "$ref": "#/$defs/timeSeries" + }, + "attestationSuccessRate": { + "$ref": "#/$defs/timeSeries" + }, "txCollectorTxsFromMempoolRate": { "$ref": "#/$defs/timeSeries" }, diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index d0c6d2f47b96..b9f3d0d6fc4e 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -560,6 +560,29 @@ const TIME_SERIES_DEFS: Record = { // code records attestationTimeAllowed in seconds. query: `avg(aztec_sequencer_attestations_collect_allowance_milliseconds${NS}) * 1000`, }, + // Attester-side attestation failures, broken down by error_type. The + // error_type=timeout slice is the signal that diagnosed the 10 TPS reorgs: an + // attester that could not re-execute the proposed checkpoint in time, dragging + // the committee below quorum. Summed across pods (validators emit this) so the + // series is the network-wide failure rate per cause. + attestationFailedNodeIssueByErrorTypeRate: { + metric: "aztec_validator_attestation_failed_node_issue_count", + unit: "tps", + query: `sum by (aztec_error_type)(rate(aztec_validator_attestation_failed_node_issue_count${NS}[1m]))`, + }, + // Attestations rejected because the proposal itself was bad (invalid_proposal, + // state_mismatch, failed_txs, …) — distinct from the node-side issues above. + attestationFailedBadProposalByErrorTypeRate: { + metric: "aztec_validator_attestation_failed_bad_proposal_count", + unit: "tps", + query: `sum by (aztec_error_type)(rate(aztec_validator_attestation_failed_bad_proposal_count${NS}[1m]))`, + }, + // Successful attestations, the denominator for a failure ratio. + attestationSuccessRate: { + metric: "aztec_validator_attestation_success_count", + unit: "tps", + query: `sum(rate(aztec_validator_attestation_success_count${NS}[1m]))`, + }, checkpointBlockCountMean: { metric: "aztec_sequencer_checkpoint_block_count", unit: "count", @@ -1929,6 +1952,9 @@ async function buildSummary(a: SummaryArgs): Promise> { mempoolMinedP50, mempoolMinedP95, mempoolMinedP99, + attestationFailedNodeIssueCount, + attestationFailedBadProposalCount, + attestationSuccessCount, ] = await Promise.all([ safeInstant( oneShotQuantile( @@ -1967,6 +1993,18 @@ async function buildSummary(a: SummaryArgs): Promise> { safeInstant( oneShotQuantile(0.99, "aztec_mempool_tx_mined_delay_milliseconds_bucket"), ), + // Total attestation outcomes over the observed window. The node-issue count + // is the headline reorg-diagnostic number (e.g. the run #95 "184" of failed + // attestations); success gives a denominator for a failure ratio. + safeInstant( + `sum(increase(aztec_validator_attestation_failed_node_issue_count${NS}[${windowSpec}]))`, + ), + safeInstant( + `sum(increase(aztec_validator_attestation_failed_bad_proposal_count${NS}[${windowSpec}]))`, + ), + safeInstant( + `sum(increase(aztec_validator_attestation_success_count${NS}[${windowSpec}]))`, + ), ]); const reorgs = a.events.filter((e) => e.type === "chainPruned"); @@ -1992,6 +2030,9 @@ async function buildSummary(a: SummaryArgs): Promise> { mempoolTxMinedDelayP50Ms: mempoolMinedP50, mempoolTxMinedDelayP95Ms: mempoolMinedP95, mempoolTxMinedDelayP99Ms: mempoolMinedP99, + attestationFailedNodeIssueCount, + attestationFailedBadProposalCount, + attestationSuccessCount, totalTxsMined, totalTxsFailed: hasInclusionBlockRecords ? inclusionBlocks.reduce((s, b) => s + b.failedCount, 0) From fdab3f2c397848ae9cd2cda8b2a06002a94983a9 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 20:51:40 +0000 Subject: [PATCH 24/39] feat(archiver): attribute prune_count by prune cause MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prune_count was incremented only by the proven/epoch prune (handleEpochPrune), so the pending-chain reorgs that drive world-state's "Chain pruned to block N" — proposed blocks dropped because their slot ended without a checkpoint, because they conflicted with an L1 checkpoint, or because their matching proposed checkpoint never arrived before the deadline (pruneOrphanBlocks) — incremented no metric and were observable only in logs. Add a prune_type attribute (unproven / uncheckpointed / l1_conflict / orphan) to prune_count and tag all four emit sites. Scraped per pod, this distinguishes a single straggler node's local re-sync from a network-wide reorg. --- yarn-project/archiver/src/archiver.ts | 1 + .../archiver/src/modules/instrumentation.ts | 15 +++++++++++++-- .../archiver/src/modules/l1_synchronizer.ts | 3 +++ yarn-project/telemetry-client/src/attributes.ts | 2 ++ yarn-project/telemetry-client/src/metrics.ts | 3 ++- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/yarn-project/archiver/src/archiver.ts b/yarn-project/archiver/src/archiver.ts index 5619c2ce0224..380fbe45bd28 100644 --- a/yarn-project/archiver/src/archiver.ts +++ b/yarn-project/archiver/src/archiver.ts @@ -499,6 +499,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra const prunedBlocks = await this.updater.removeBlocksWithoutProposedCheckpointAfter(pruneAfterBlockNumber); if (prunedBlocks.length > 0) { + this.instrumentation.recordPrune('orphan'); this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, { type: L2BlockSourceEvents.L2PruneUncheckpointed, slotNumber: blockSlot, diff --git a/yarn-project/archiver/src/modules/instrumentation.ts b/yarn-project/archiver/src/modules/instrumentation.ts index 4e031872ca35..d73726d946d4 100644 --- a/yarn-project/archiver/src/modules/instrumentation.ts +++ b/yarn-project/archiver/src/modules/instrumentation.ts @@ -84,7 +84,9 @@ export class ArchiverInstrumentation { this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION); - this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT); + this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT, { + [Attributes.PRUNE_TYPE]: ['unproven', 'uncheckpointed', 'l1_conflict', 'orphan'], + }); this.blockProposalTxTargetCount = createUpDownCounterWithDefault( meter, @@ -150,10 +152,19 @@ export class ArchiverInstrumentation { } public processPrune(duration: number) { - this.pruneCount.add(1); + this.pruneCount.add(1, { [Attributes.PRUNE_TYPE]: 'unproven' }); this.pruneDuration.record(Math.ceil(duration)); } + /** + * Records a pending-chain reorg, where the archiver dropped proposed blocks (and world-state follows by pruning). The + * type distinguishes the cause: 'uncheckpointed' (slot ended without a checkpoint), 'l1_conflict' (blocks conflicting + * with an L1 checkpoint), or 'orphan' (no matching proposed checkpoint arrived before the deadline). + */ + public recordPrune(pruneType: 'uncheckpointed' | 'l1_conflict' | 'orphan') { + this.pruneCount.add(1, { [Attributes.PRUNE_TYPE]: pruneType }); + } + public updateLastProvenCheckpoint(checkpoint: CheckpointData) { const lastBlockNumberInCheckpoint = checkpoint.startBlock + checkpoint.blockCount - 1; this.blockHeight.record(lastBlockNumberInCheckpoint, { [Attributes.STATUS]: 'proven' }); diff --git a/yarn-project/archiver/src/modules/l1_synchronizer.ts b/yarn-project/archiver/src/modules/l1_synchronizer.ts index de484c3be5a6..6a6758467035 100644 --- a/yarn-project/archiver/src/modules/l1_synchronizer.ts +++ b/yarn-project/archiver/src/modules/l1_synchronizer.ts @@ -302,6 +302,7 @@ export class ArchiverL1Synchronizer implements Traceable { const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber); if (prunedBlocks.length > 0) { + this.instrumentation.recordPrune('uncheckpointed'); this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, { type: L2BlockSourceEvents.L2PruneUncheckpointed, slotNumber: firstUncheckpointedBlockSlot, @@ -1077,6 +1078,8 @@ export class ArchiverL1Synchronizer implements Traceable { { prunedBlocks: result.prunedBlocks.map(b => b.toBlockInfo()), prunedSlotNumber, prunedCheckpointNumber }, ); + this.instrumentation.recordPrune('l1_conflict'); + // Emit event for listening services to react to the prune. // Note: slotNumber comes from the first pruned block. If pruned blocks theoretically spanned multiple slots, // only one slot number would be reported (though in practice all blocks in a checkpoint span a single slot). diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index 900b27f915a8..be481ca7b58b 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -73,6 +73,8 @@ export const OK = 'aztec.ok'; export const STATUS = 'aztec.status'; /** Generic error type attribute */ export const ERROR_TYPE = 'aztec.error_type'; +/** The cause of an archiver prune (unproven / uncheckpointed / l1_conflict / orphan) */ +export const PRUNE_TYPE = 'aztec.archiver.prune_type'; /** The type of the transaction */ export const L1_TX_TYPE = 'aztec.l1.tx_type'; /** The L1 address of the entity that sent a transaction to L1 */ diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 520775a3badc..3ea78f33245f 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -340,7 +340,8 @@ export const ARCHIVER_PRUNE_DURATION: MetricDefinition = { }; export const ARCHIVER_PRUNE_COUNT: MetricDefinition = { name: 'aztec.archiver.prune_count', - description: 'Number of prunes detected', + description: + 'Number of prunes detected, dimensioned by prune_type: unproven (epoch prune of checkpoints that will not be proven), uncheckpointed (proposed blocks whose slot ended without a checkpoint), l1_conflict (proposed blocks conflicting with an L1 checkpoint), and orphan (proposed blocks whose matching proposed checkpoint never arrived before the deadline). Only unproven was counted before this dimension was added; the rest are pending-chain reorgs previously observable only in logs.', valueType: ValueType.INT, }; From 85ccaa2f4ab2b5ef8e12966f0e10f3648425ad65 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Fri, 26 Jun 2026 21:10:12 +0000 Subject: [PATCH 25/39] feat(bench): derive reorgCount from per-pod prune_count + prune-type breakdown reorgCount counted `Chain pruned` log events network-wide, which conflates one straggler node repeatedly re-syncing its local view with a genuine network reorg. The 10 TPS sweep showed this: 3 "reorgs" that were all one saturated validator pruning its own view three times while the canonical chain never moved. Re-point reorgCount to count DISTINCT PODS that experienced a pending-chain prune (archiver prune_count, prune_type in uncheckpointed/l1_conflict/orphan), so a straggler reads as 1 and a network reorg as many. Add a prunesByType summary breakdown and a pruneCountByType time series. deepestReorgBlocks stays log-derived (no depth metric). Requires the prune_type-attributed metric (new image); reads 0 on older images, cross-checked by the chainPruned events. --- .../bench_10tps/bench_output.schema.json | 18 +++++++- spartan/scripts/bench_10tps/bench_scrape.ts | 41 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 32e2defe04fe..98af5573ecbe 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -397,11 +397,22 @@ }, "reorgCount": { "type": ["integer", "null"], - "description": "Count of `Chain pruned` events during the run." + "description": "Number of DISTINCT PODS that experienced a pending-chain prune over the window, from the archiver prune_count metric (prune_type in uncheckpointed/l1_conflict/orphan). Counts pods, not events: a single straggler node re-syncing reads as 1, a network-wide reorg as many. Requires the prune_type-attributed metric (new image); reads 0 on older images, so cross-check against the chainPruned events array. Distinct from the raw `Chain pruned` log-event count, which conflates one node's repeated local prunes with a network reorg." + }, + "prunesByType": { + "type": "object", + "additionalProperties": false, + "description": "Total archiver prunes over the window by cause, summed across pods (so this counts prune events, while reorgCount counts pods). 'unproven' is the proven/epoch prune; the rest are pending-chain reorgs. Each value is null when the metric was unavailable.", + "properties": { + "unproven": { "type": ["number", "null"] }, + "uncheckpointed": { "type": ["number", "null"] }, + "l1_conflict": { "type": ["number", "null"] }, + "orphan": { "type": ["number", "null"] } + } }, "deepestReorgBlocks": { "type": ["integer", "null"], - "description": "Max (fromBlock - toBlock) across reorg events. 0 if no reorgs." + "description": "Max (fromBlock - toBlock) across `Chain pruned` log events. 0 if none. Log-derived (no depth metric), so this reflects the deepest local prune observed in logs." } } }, @@ -500,6 +511,9 @@ "attestationSuccessRate": { "$ref": "#/$defs/timeSeries" }, + "pruneCountByType": { + "$ref": "#/$defs/timeSeries" + }, "txCollectorTxsFromMempoolRate": { "$ref": "#/$defs/timeSeries" }, diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index b9f3d0d6fc4e..6b93b5988269 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -583,6 +583,17 @@ const TIME_SERIES_DEFS: Record = { unit: "tps", query: `sum(rate(aztec_validator_attestation_success_count${NS}[1m]))`, }, + // Archiver prunes broken down by cause (prune_type). 'unproven' is the + // proven/epoch prune; 'uncheckpointed'/'l1_conflict'/'orphan' are pending-chain + // reorgs that move a node's proposed tip (world-state then logs "Chain pruned"). + // Summed across pods, so a value here is the network-wide prune rate per cause; + // see summary.reorgCount for how many distinct pods diverged. Requires the + // prune_type-attributed metric (new image); empty on older images. + pruneCountByType: { + metric: "aztec_archiver_prune_count", + unit: "tps", + query: `sum by (aztec_archiver_prune_type)(rate(aztec_archiver_prune_count${NS}[1m]))`, + }, checkpointBlockCountMean: { metric: "aztec_sequencer_checkpoint_block_count", unit: "count", @@ -2013,6 +2024,33 @@ async function buildSummary(a: SummaryArgs): Promise> { return d > max ? d : max; }, 0); + // Pending-chain reorgs from the archiver prune_count metric, counted PER POD so + // a single straggler's local re-sync (one pod) is distinguishable from a + // network-wide reorg (many pods) — the deduped chainPruned log count conflates + // them. Pending-chain prune types are all except 'unproven' (the proven/epoch + // prune). Requires the prune_type-attributed metric (new image); on older + // images this reads 0 — cross-check against the chainPruned events. + const pendingPruneSelector = `aztec_archiver_prune_type=~"uncheckpointed|l1_conflict|orphan"`; + const reorgPodCount = await safeInstant( + `count(sum by (k8s_pod_name)(increase(aztec_archiver_prune_count{k8s_namespace_name="${NAMESPACE}",${pendingPruneSelector}}[${windowSpec}])) > 0)`, + ); + const pruneTypes = [ + "unproven", + "uncheckpointed", + "l1_conflict", + "orphan", + ] as const; + const pruneTypeTotals = await Promise.all( + pruneTypes.map((t) => + safeInstant( + `sum(increase(aztec_archiver_prune_count{k8s_namespace_name="${NAMESPACE}",aztec_archiver_prune_type="${t}"}[${windowSpec}]))`, + ), + ), + ); + const prunesByType = Object.fromEntries( + pruneTypes.map((t, i) => [t, pruneTypeTotals[i]]), + ) as Record<(typeof pruneTypes)[number], number | null>; + return { headlineKpi: inclusionTpsMean === null ? null : inclusionTpsMean / a.targetTps, @@ -2043,7 +2081,8 @@ async function buildSummary(a: SummaryArgs): Promise> { totalSilentSkipDurationMs: hasInclusionBlockRecords ? inclusionBlocks.reduce((s, b) => s + b.silentlySkippedDurationMs, 0) : null, - reorgCount: reorgs.length, + reorgCount: reorgPodCount ?? 0, + prunesByType, deepestReorgBlocks: deepest, }; } From 08591d61e631897bf73a1dc0a9679b5137a2ef06 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Sat, 27 Jun 2026 16:52:00 +0000 Subject: [PATCH 26/39] feat(bench): add throughput-funnel timeseries (gossip / rpc / mined TPS) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a throughputTps series carrying three labeled sub-series (source = gossip | rpc_receive | mined) so the dashboard overlays tx rate at three stages on one chart: gossip propagation (avg per-node accepted tx-topic gossipsub messages — every node accepts each unique tx once, so avg approximates the unique-tx propagation rate; sum would overcount by pod count), RPC ingress (receiveTx on the submission node), and mined (archiver block tx rate, avg across archivers). Query validated against retained Prometheus data. --- .../scripts/bench_10tps/bench_output.schema.json | 4 ++++ spartan/scripts/bench_10tps/bench_scrape.ts | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 98af5573ecbe..9dfca172fc2b 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -429,6 +429,10 @@ "ingressTps": { "$ref": "#/$defs/timeSeries" }, + "throughputTps": { + "$ref": "#/$defs/timeSeries", + "description": "Tx throughput funnel as three labeled series (source = gossip | rpc_receive | mined) for overlaying on one chart: gossip propagation (per-node accepted tx-topic gossipsub, avg across pods), RPC ingress, and mined (avg across archivers)." + }, "mempoolSizeRpc": { "$ref": "#/$defs/timeSeries" }, diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index 6b93b5988269..fddfe53e61a8 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -383,6 +383,22 @@ const TIME_SERIES_DEFS: Record = { unit: "tps", query: `sum(rate(aztec_node_receive_tx_count${NS}[1m]))`, }, + // Throughput funnel — tx rate at three stages, as three labeled series (source= + // gossip|rpc_receive|mined) in one entry so the dashboard overlays them on one + // chart. gossip: per-node accepted tx-topic gossipsub messages, avg across pods + // (every node accepts each unique tx once, so avg ~= unique-tx propagation rate; + // sum would overcount by the pod count). rpc_receive: receiveTx on the single + // submission node (= ingressTps). mined: archiver block tx rate, avg across + // archivers (= inclusionTps). + throughputTps: { + metric: + "gossipsub_accepted_messages_total|aztec_node_receive_tx_count|aztec_archiver_block_tx_count_sum", + unit: "tps", + query: + `label_replace(avg(rate(gossipsub_accepted_messages_total{k8s_namespace_name="${NAMESPACE}",topic="tx"}[1m])), "source", "gossip", "", "") or ` + + `label_replace(sum(rate(aztec_node_receive_tx_count${NS}[1m])), "source", "rpc_receive", "", "") or ` + + `label_replace(avg(rate(aztec_archiver_block_tx_count_sum${NS}[1m])), "source", "mined", "", "")`, + }, // Duration of the RPC node's receiveTx handler (aztec.node.receive_tx.duration, // a histogram). This is the tx-ingest cost on the submission path — the metric // to watch when RPC ingress is the bottleneck (climbs as the RPC saturates). From 912b6b7dde01c844aef163f4fc0a9f4c08fe33e2 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Sat, 27 Jun 2026 17:44:54 +0000 Subject: [PATCH 27/39] feat(bench): add P2P gossip bandwidth timeseries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add p2pBandwidthBytesPerSec carrying four labeled series (recv/sent × avg/max per pod) from the gossipsub RPC byte counters. Per pod rather than summed so the hottest-pod (max) series surfaces gossip bandwidth that isn't evenly spread — at 10 TPS the hottest pod moved ~4 MB/s each way vs ~2.3 MB/s average. No libp2p-level transport metric is exported, so the gossipsub byte counters are the bandwidth signal. Query validated against retained Prometheus data. --- .../scripts/bench_10tps/bench_output.schema.json | 4 ++++ spartan/scripts/bench_10tps/bench_scrape.ts | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 9dfca172fc2b..260ea5b876fa 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -433,6 +433,10 @@ "$ref": "#/$defs/timeSeries", "description": "Tx throughput funnel as three labeled series (source = gossip | rpc_receive | mined) for overlaying on one chart: gossip propagation (per-node accepted tx-topic gossipsub, avg across pods), RPC ingress, and mined (avg across archivers)." }, + "p2pBandwidthBytesPerSec": { + "$ref": "#/$defs/timeSeries", + "description": "Gossipsub P2P wire bandwidth (all topics) as four labeled series (source = recv_avg | recv_max | sent_avg | sent_max): per-pod average and hottest-pod receive/send byte rates. From the gossipsub RPC byte counters (no libp2p-level transport metric is exported)." + }, "mempoolSizeRpc": { "$ref": "#/$defs/timeSeries" }, diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index fddfe53e61a8..c49ccddd707c 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -399,6 +399,21 @@ const TIME_SERIES_DEFS: Record = { `label_replace(sum(rate(aztec_node_receive_tx_count${NS}[1m])), "source", "rpc_receive", "", "") or ` + `label_replace(avg(rate(aztec_archiver_block_tx_count_sum${NS}[1m])), "source", "mined", "", "")`, }, + // Gossipsub P2P wire bandwidth (all topics), as four labeled series: receive and + // send, each as the per-pod average and the hottest pod (max). Per pod, not + // summed, since the question is whether any node's bandwidth is a wall — the max + // series surfaces gossip load that isn't evenly spread. No libp2p-level transport + // metric is exported, so these gossipsub RPC byte counters are the bandwidth + // signal (they dominate P2P traffic). + p2pBandwidthBytesPerSec: { + metric: "gossipsub_rpc_recv_bytes_total|gossipsub_rpc_sent_bytes_total", + unit: "bytes/sec", + query: + `label_replace(avg(rate(gossipsub_rpc_recv_bytes_total${NS}[1m])), "source", "recv_avg", "", "") or ` + + `label_replace(max(rate(gossipsub_rpc_recv_bytes_total${NS}[1m])), "source", "recv_max", "", "") or ` + + `label_replace(avg(rate(gossipsub_rpc_sent_bytes_total${NS}[1m])), "source", "sent_avg", "", "") or ` + + `label_replace(max(rate(gossipsub_rpc_sent_bytes_total${NS}[1m])), "source", "sent_max", "", "")`, + }, // Duration of the RPC node's receiveTx handler (aztec.node.receive_tx.duration, // a histogram). This is the tx-ingest cost on the submission path — the metric // to watch when RPC ingress is the bottleneck (climbs as the RPC saturates). From 2dbd1e8649c8841ca2d8310ba74f766f529545bc Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 29 Jun 2026 07:45:07 +0000 Subject: [PATCH 28/39] chore(bench): move 4-core spot node pool tf to stack/chore-remove-duplicate-tf-blocks The spot_nodes_4core node pool definition belongs with the gke-cluster terraform cleanup, not the bench branch. Removed here and applied to stack/chore-remove-duplicate-tf-blocks. The pool is already applied to the live aztec-gke-private cluster, and the bench full-node resource profile that targets it (cores=4 selector) stays on this branch. --- spartan/terraform/gke-cluster/cluster/main.tf | 48 +------------------ 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/spartan/terraform/gke-cluster/cluster/main.tf b/spartan/terraform/gke-cluster/cluster/main.tf index c94c434e3e27..5d18eda4ecb4 100644 --- a/spartan/terraform/gke-cluster/cluster/main.tf +++ b/spartan/terraform/gke-cluster/cluster/main.tf @@ -15,7 +15,7 @@ resource "google_container_cluster" "primary" { channel = "UNSPECIFIED" } - # NOTE: Enabling workload identity at the cluster level means new node pools may default to GKE_METADATA mode. + # NOTE: Enabling workload identity at the cluster level means new node pools may default to GKE_METADATA mode. dynamic "workload_identity_config" { for_each = var.enable_workload_identity ? [1] : [] content { @@ -344,52 +344,6 @@ resource "google_container_node_pool" "spot_nodes_2core" { } } -# Create 4 core spot instance node pool with autoscaling -resource "google_container_node_pool" "spot_nodes_4core" { - name = "${var.cluster_name}-4core-spot" - location = var.zone - cluster = var.cluster_name - version = var.node_version - # Enable autoscaling - autoscaling { - min_node_count = 0 - max_node_count = 1500 - } - - # Node configuration - node_config { - machine_type = "t2d-standard-4" - spot = true - - service_account = var.service_account - oauth_scopes = [ - "https://www.googleapis.com/auth/cloud-platform" - ] - - labels = { - env = "production" - pool = "spot" - local-ssd = "false" - node-type = "network" - cores = "4" - } - tags = ["aztec-gke-node", "spot"] - - # Spot instance termination handler - taint { - key = "cloud.google.com/gke-spot" - value = "true" - effect = "NO_SCHEDULE" - } - } - - # Management configuration - management { - auto_repair = true - auto_upgrade = false - } -} - # Create 8 core high memory (64 GB) node pool with autoscaling, used for metrics resource "google_container_node_pool" "infra_nodes_8core_highmem" { name = "${var.cluster_name}-infra-8core-hi-mem" From e6fef85e4c8d02bcce7a17b8c79c11f7d20b2bb7 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 29 Jun 2026 09:59:45 +0000 Subject: [PATCH 29/39] feat(bench): make network_bench upload bucket overridable + dispatchable NETWORK_BENCH_BUCKET overrides the upload target (default unchanged: gs://aztec-testnet/network_bench) so the upload path can be validated against a test prefix without polluting the production dashboard index. Add a network_bench_upload dispatch entry so a run JSON can be (re)uploaded manually. --- spartan/bootstrap.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 4e4a52376025..77b9c2c2437d 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -362,7 +362,7 @@ function network_bench_upload { return 0 fi - local bucket="gs://aztec-testnet/network_bench" + local bucket="${NETWORK_BENCH_BUCKET:-gs://aztec-testnet/network_bench}" local run_id=$(jq -r .run.runId "$run_json") local target="${bucket}/${run_id}.json" @@ -531,6 +531,9 @@ case "$cmd" in "hash") echo $(hash) ;; + "network_bench_upload") + network_bench_upload "$1" + ;; test|test_cmds|gke|build|gcp_auth) $cmd ;; From 9d1bd55f2ec6d93f4b8ea289c9db65f4b42f6d18 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 29 Jun 2026 10:25:31 +0000 Subject: [PATCH 30/39] chore(bench): remove private issue-tracker references from comments Strip internal tracker issue IDs and the 'Set A' project label from code and workflow comments; the repo is public so these convey nothing to external readers. Behaviour unchanged (comment-only edits). --- .github/workflows/nightly-bench-inclusion-sweep.yml | 2 +- bootstrap.sh | 2 +- ci.sh | 2 +- spartan/bootstrap.sh | 2 +- spartan/environments/bench-inclusion-sweep.env | 2 +- spartan/scripts/bench_10tps/bench_scrape.ts | 4 ++-- .../deploy-aztec-infra/values/full-node-resources-spot.yaml | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-bench-inclusion-sweep.yml b/.github/workflows/nightly-bench-inclusion-sweep.yml index 23a97474ed18..9a5912db4ac6 100644 --- a/.github/workflows/nightly-bench-inclusion-sweep.yml +++ b/.github/workflows/nightly-bench-inclusion-sweep.yml @@ -1,6 +1,6 @@ name: Nightly Bench Inclusion Sweep -# Set A inclusion sweep (Linear A-1223): deploys ONE network and runs the +# Inclusion sweep: deploys ONE network and runs the # 1/5/10 TPS points SEQUENTIALLY against it, all tagged with a shared sweepId so # the dashboard groups them. The scraper drains the mempool to zero between # points. Each point is the reusable bench-inclusion-point.yml workflow diff --git a/bootstrap.sh b/bootstrap.sh index 8461f10be43b..4f9aa7e096e0 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -938,7 +938,7 @@ case "$cmd" in ;; "ci-network-inclusion-sweep") # Args: [docker_image] - # Runs one Set A inclusion-sweep point (TARGET_TPS) on the given network. + # Runs one inclusion-sweep point (TARGET_TPS) on the given network. # The v4 run JSON (tagged BENCH_SWEEP_ID) is uploaded to GCS inside # bench_inclusion_point; deploy/teardown of each point's namespace is done by # the workflow, so this is normally called with SKIP_NETWORK_DEPLOY=1. diff --git a/ci.sh b/ci.sh index a694ec6e9f8e..70317d31bffa 100755 --- a/ci.sh +++ b/ci.sh @@ -300,7 +300,7 @@ case "$cmd" in ;; network-inclusion-sweep) # Args: [docker_image] - # Runs one inclusion-sweep point (Set A) at TARGET_TPS against an existing + # Runs one inclusion-sweep point at TARGET_TPS against an existing # network, tagged with BENCH_SWEEP_ID. The workflow deploys/tears down each # point's namespace separately, so this is normally called with # SKIP_NETWORK_DEPLOY=1. TARGET_TPS / BENCH_SWEEP_ID / BENCH_SWEEP_LABEL come diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 77b9c2c2437d..527d9649a7d6 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -273,7 +273,7 @@ function bench_10tps { fi } -# One point of the Set A inclusion sweep (A-1223). Same scrape+upload path as +# One point of the inclusion sweep. Same scrape+upload path as # bench_10tps, and the same load model: a fixed 1 TPS of high-value txs (the # measured inclusion lane) plus (TARGET_TPS - 1) TPS of low-value background # traffic to bring total load to the target. So inclusion latency is always diff --git a/spartan/environments/bench-inclusion-sweep.env b/spartan/environments/bench-inclusion-sweep.env index cff4d89d7463..393dadcbf755 100644 --- a/spartan/environments/bench-inclusion-sweep.env +++ b/spartan/environments/bench-inclusion-sweep.env @@ -1,4 +1,4 @@ -# Set A inclusion sweep — nightly benchmark network. +# Inclusion sweep — nightly benchmark network. NAMESPACE=${NAMESPACE:-bench-inclusion-sweep} CLUSTER=aztec-gke-private RESOURCE_PROFILE=prod diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index c49ccddd707c..6bbf7a32a47d 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -701,8 +701,8 @@ const SATURATION_ROLES: Record = { // telemetry-client/src/nodejs_metrics_monitor.ts (nodejs.* prefix, NOT aztec_). // CPU comes from @opentelemetry/host-metrics (process.cpu.utilization), not the // nodejs monitor. NOTE: ELU and especially CPU may be telemetry-gated in the -// bench env — if so these series come back empty (A-1222 acceptance: verify on -// the live env and adjust the metric name / enable the exporter as needed). +// bench env — if so these series come back empty (verify on the live env and +// adjust the metric name / enable the exporter as needed). const SATURATION_METRICS: { key: string; metric: string; unit: string }[] = [ { key: "elu", metric: "nodejs_eventloop_utilization", unit: "ratio" }, { key: "cpu", metric: "process_cpu_utilization", unit: "ratio" }, diff --git a/spartan/terraform/deploy-aztec-infra/values/full-node-resources-spot.yaml b/spartan/terraform/deploy-aztec-infra/values/full-node-resources-spot.yaml index 876cbd04a341..9f3695af922a 100644 --- a/spartan/terraform/deploy-aztec-infra/values/full-node-resources-spot.yaml +++ b/spartan/terraform/deploy-aztec-infra/values/full-node-resources-spot.yaml @@ -1,5 +1,5 @@ -# Full-node profile that REQUIRES spot nodes (Linear A-1223, Set A inclusion -# sweep). Same shape + resources as full-node-resources-prod.yaml, but the +# Full-node profile that REQUIRES spot nodes (inclusion sweep). Same shape +# + resources as full-node-resources-prod.yaml, but the # spot affinity is required (not merely preferred) so full nodes always land on # spot, copying the pattern from prover-resources-dev.yaml. Only full nodes use # this profile; validators / RPC / prover stay on-demand so mid-block spot From 6c7df0e707ca28ae98b7b835373b630b443dcf05 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 29 Jun 2026 10:53:14 +0000 Subject: [PATCH 31/39] feat(bench): measure inclusionTpsMean as steady-state rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The headline inclusion TPS divided total mined by the whole observed window, which includes the warmup ramp and the post-load drain tail — diluting it well below the achieved rate (e.g. 0.84 at a 1 TPS target that actually kept up). Compute it instead from the block log over [first block with txs, load stop]: trims the startup ramp (start at first real inclusion) and the cooldown tail (end at load stop, not drain end). headlineKpi now reads ~1.0 on healthy runs (1/5/10 TPS: 0.97/0.99/0.99) and drops below 1 only on genuine shortfall. The old whole-window figure is retained as inclusionTpsWindowMean. --- .../bench_10tps/bench_output.schema.json | 8 +++- spartan/scripts/bench_10tps/bench_scrape.ts | 39 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 260ea5b876fa..8658c1ad2076 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -307,14 +307,18 @@ "properties": { "headlineKpi": { "type": ["number", "null"], - "description": "inclusionTpsMean / targetTps. The single number on the dashboard top strip." + "description": "inclusionTpsMean / targetTps. The single number on the dashboard top strip. ~1.0 on a healthy run; below 1 only when the network cannot keep up with offered load (warmup/drain no longer dilute it)." }, "targetTps": { "type": "number" }, "inclusionTpsMean": { "type": ["number", "null"], - "description": "Inclusion throughput over the observed inclusion window. Uses exact block-log throughput when block records are available, otherwise falls back to the Prometheus inclusionTps mean." + "description": "Steady-state inclusion throughput from the block log: txs mined between the first block that included txs (warmup ramp trimmed) and load stop (drain tail trimmed), divided by that span. Falls back to inclusionTpsWindowMean when block logs are unavailable." + }, + "inclusionTpsWindowMean": { + "type": ["number", "null"], + "description": "Total txs mined divided by the whole observed window (start to drain end). Diluted by the warmup ramp and post-load drain tail; retained for transparency alongside the steady-state inclusionTpsMean." }, "inclusionTpsPeak": { "type": ["number", "null"], diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index 6bbf7a32a47d..d532cc97afe6 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -1869,6 +1869,9 @@ type SummaryArgs = { targetTps: number; startedAtEpoch: number; inclusionEndedAtEpoch: number; + // Load-stop time (when the generator stopped sending), distinct from + // inclusionEndedAtEpoch (the drain end). Bounds the steady-state window. + loadEndedAtEpoch: number; windowSec: number; histogramWindowSec: number; endedAtEpoch: number; @@ -1944,10 +1947,42 @@ async function buildSummary(a: SummaryArgs): Promise> { ? inclusionBlocks.reduce((s, b) => s + b.successfulCount, 0) : null; const promInclusionTpsMean = meanNonNull(inclusionPoints); - const inclusionTpsMean = + // Whole-observed-window mean: total mined / observed window. Diluted by the + // warmup ramp and the post-load drain tail; kept for transparency. + const inclusionTpsWindowMean = totalTxsMined !== null && a.windowSec > 0 ? totalTxsMined / a.windowSec : promInclusionTpsMean; + // Steady-state rate from the block log: start at the first block that actually + // included txs (trims the warmup ramp) and end at load stop, not the drain end + // (trims the cooldown tail), so the headline reflects sustained-load throughput + // and only drops below target when the network genuinely can't keep up. Falls + // back to the window mean when block logs are unavailable. + const minedBlocks = a.blocks + .map((b) => ({ + epoch: Math.floor(Date.parse(b.minedAt) / 1000), + n: b.successfulCount, + })) + .filter((b) => Number.isFinite(b.epoch)); + const firstInclusionEpoch = minedBlocks + .filter( + (b) => + b.n > 0 && b.epoch >= a.startedAtEpoch && b.epoch <= a.loadEndedAtEpoch, + ) + .reduce((min, b) => Math.min(min, b.epoch), Number.POSITIVE_INFINITY); + const steadySec = Number.isFinite(firstInclusionEpoch) + ? a.loadEndedAtEpoch - firstInclusionEpoch + : 0; + const steadyTxs = Number.isFinite(firstInclusionEpoch) + ? minedBlocks + .filter( + (b) => + b.epoch >= firstInclusionEpoch && b.epoch <= a.loadEndedAtEpoch, + ) + .reduce((s, b) => s + b.n, 0) + : 0; + const inclusionTpsMean = + steadySec > 0 ? steadyTxs / steadySec : inclusionTpsWindowMean; const inclusionTpsPeak = maxNonNull(inclusionPoints); if (!hasInclusionBlockRecords && promInclusionTpsMean !== null) { @@ -2087,6 +2122,7 @@ async function buildSummary(a: SummaryArgs): Promise> { inclusionTpsMean === null ? null : inclusionTpsMean / a.targetTps, targetTps: a.targetTps, inclusionTpsMean, + inclusionTpsWindowMean, inclusionTpsPeak, inclusionLatencyP50Ms: inclLatP50, inclusionLatencyP95Ms: inclLatP95, @@ -2465,6 +2501,7 @@ async function main(): Promise { targetTps: args.targetTps, startedAtEpoch, inclusionEndedAtEpoch: drain.inclusionEndedAtEpoch, + loadEndedAtEpoch: endedAtEpoch, windowSec: observedWindowSec, histogramWindowSec: observedWindowSec, endedAtEpoch: drain.inclusionEndedAtEpoch, From 2b2ac1b2a39ffa97ee0c2f8b3cf4c2650e307de1 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 29 Jun 2026 11:10:59 +0000 Subject: [PATCH 32/39] feat(bench): distinguish chain reorgs from single-node prunes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tag each chainPruned event with podCount (distinct pods that pruned to the same block within the dedupe window). reorgCount now counts only genuine chain reorgs (podCount >= 2 — the canonical chain rewound across nodes); single-node local prunes (a straggler that diverged and re-synced) are counted separately as nodeLocalPruneCount and no longer inflate reorgCount or deepestReorgBlocks. This is log-derived, so it is accurate regardless of node image. Verified on the 1/5/10 TPS runs: 0 chain reorgs, with the 5/10 TPS prunes correctly classified as node-local. --- .../bench_10tps/bench_output.schema.json | 14 ++++- spartan/scripts/bench_10tps/bench_scrape.ts | 60 +++++++++++++------ 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 8658c1ad2076..6be5b1b06caa 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -401,12 +401,16 @@ }, "reorgCount": { "type": ["integer", "null"], - "description": "Number of DISTINCT PODS that experienced a pending-chain prune over the window, from the archiver prune_count metric (prune_type in uncheckpointed/l1_conflict/orphan). Counts pods, not events: a single straggler node re-syncing reads as 1, a network-wide reorg as many. Requires the prune_type-attributed metric (new image); reads 0 on older images, so cross-check against the chainPruned events array. Distinct from the raw `Chain pruned` log-event count, which conflates one node's repeated local prunes with a network reorg." + "description": "Number of genuine CHAIN reorgs: `Chain pruned` events where 2+ distinct pods pruned to the same block (event.podCount >= 2), i.e. the canonical chain rewound network-wide. A single straggler whose local view diverged and re-synced is NOT counted here — see nodeLocalPruneCount. Log-derived, so accurate regardless of node image." + }, + "nodeLocalPruneCount": { + "type": ["integer", "null"], + "description": "Number of single-node local prunes (`Chain pruned` events with podCount <= 1): one node's view diverged and re-synced while the canonical chain held. Informational — does not indicate chain instability or make the headline provisional." }, "prunesByType": { "type": "object", "additionalProperties": false, - "description": "Total archiver prunes over the window by cause, summed across pods (so this counts prune events, while reorgCount counts pods). 'unproven' is the proven/epoch prune; the rest are pending-chain reorgs. Each value is null when the metric was unavailable.", + "description": "Total archiver prunes over the window by cause, summed across pods. 'unproven' is the proven/epoch prune; the rest are pending-chain prunes. From the prune_type-attributed metric; each value is null when that metric was unavailable (older node images).", "properties": { "unproven": { "type": ["number", "null"] }, "uncheckpointed": { "type": ["number", "null"] }, @@ -416,7 +420,7 @@ }, "deepestReorgBlocks": { "type": ["integer", "null"], - "description": "Max (fromBlock - toBlock) across `Chain pruned` log events. 0 if none. Log-derived (no depth metric), so this reflects the deepest local prune observed in logs." + "description": "Max (fromBlock - toBlock) across genuine chain-reorg events (podCount >= 2). 0 when there were no chain reorgs (single-node local prune depths are not counted here; see the chainPruned events for those)." } } }, @@ -711,6 +715,10 @@ "type": "integer", "description": "For chainPruned: the post-prune tip." }, + "podCount": { + "type": "integer", + "description": "For chainPruned: distinct pods that logged this prune (same toBlock within the dedupe window). 1 = a single node diverged locally and re-synced; >= 2 = the canonical chain rewound across multiple nodes (a genuine chain reorg)." + }, "slotNumber": { "type": "integer", "description": "For slotSummary: L2 slot number." diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index d532cc97afe6..5a125b58ce6f 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -1426,6 +1426,10 @@ type ChainPrunedEvent = { source: "log"; fromBlock?: number; toBlock?: number; + // Distinct pods that logged this prune (same toBlock within the dedupe + // window). 1 = a single node's local view diverged and re-synced; many = + // the canonical chain rewound network-wide (a genuine chain reorg). + podCount: number; }; type SlotSummaryEvent = { @@ -1505,10 +1509,12 @@ async function scrapeChainPrunedEvents( at: entry.timestamp, time: Date.parse(entry.timestamp), toBlock: Number(m[1]), + pod: entry.resource?.labels?.pod_name ?? "unknown", }; }) .filter( - (x): x is { at: string; time: number; toBlock: number } => x !== null, + (x): x is { at: string; time: number; toBlock: number; pod: string } => + x !== null, ) .sort((a, b) => a.time - b.time); @@ -1523,6 +1529,22 @@ async function scrapeChainPrunedEvents( } } + // Distinct pods that pruned to the same toBlock within the dedupe window: + // 1 means a lone node diverged locally; many means the canonical chain rewound + // across the network. This is what separates a node-local re-sync from a real + // chain reorg. + const podCountFor = (toBlock: number, time: number) => + new Set( + parsed + .filter( + (p) => + p.toBlock === toBlock && + p.time >= time && + p.time - time < DEDUPE_WINDOW_MS, + ) + .map((p) => p.pod), + ).size; + return deduped.map(({ at, time, toBlock }) => { // fromBlock is reconstructed because server_world_state_synchronizer.ts:459 // doesn't log it structurally. Correlate with the latest block we've seen @@ -1536,6 +1558,7 @@ async function scrapeChainPrunedEvents( source: "log" as const, fromBlock: before || undefined, toBlock, + podCount: podCountFor(toBlock, time), }; }); } @@ -2084,22 +2107,24 @@ async function buildSummary(a: SummaryArgs): Promise> { ), ]); - const reorgs = a.events.filter((e) => e.type === "chainPruned"); - const deepest = reorgs.reduce((max, e) => { - const d = (e.fromBlock ?? 0) - (e.toBlock ?? 0); - return d > max ? d : max; - }, 0); - - // Pending-chain reorgs from the archiver prune_count metric, counted PER POD so - // a single straggler's local re-sync (one pod) is distinguishable from a - // network-wide reorg (many pods) — the deduped chainPruned log count conflates - // them. Pending-chain prune types are all except 'unproven' (the proven/epoch - // prune). Requires the prune_type-attributed metric (new image); on older - // images this reads 0 — cross-check against the chainPruned events. - const pendingPruneSelector = `aztec_archiver_prune_type=~"uncheckpointed|l1_conflict|orphan"`; - const reorgPodCount = await safeInstant( - `count(sum by (k8s_pod_name)(increase(aztec_archiver_prune_count{k8s_namespace_name="${NAMESPACE}",${pendingPruneSelector}}[${windowSpec}])) > 0)`, + // Separate genuine chain reorgs from single-node local divergences using the + // per-event podCount (distinct pods that pruned to the same block). podCount + // >= 2 means multiple nodes rewound to the same block — the canonical chain + // reorged; podCount <= 1 is one straggler whose local view diverged and + // re-synced, which does NOT destabilise the chain or the headline. Events from + // older scrapes without podCount default to node-local. + const chainPrunes = a.events.filter( + (e): e is ChainPrunedEvent => e.type === "chainPruned", ); + const chainReorgs = chainPrunes.filter((e) => (e.podCount ?? 1) >= 2); + const nodeLocalPruneCount = chainPrunes.length - chainReorgs.length; + const deepest = chainReorgs.reduce( + (max, e) => Math.max(max, (e.fromBlock ?? 0) - (e.toBlock ?? 0)), + 0, + ); + + // Prune totals by cause from the prune_type-attributed metric (new image; empty + // on older images). Complements the log-derived reorg classification above. const pruneTypes = [ "unproven", "uncheckpointed", @@ -2148,7 +2173,8 @@ async function buildSummary(a: SummaryArgs): Promise> { totalSilentSkipDurationMs: hasInclusionBlockRecords ? inclusionBlocks.reduce((s, b) => s + b.silentlySkippedDurationMs, 0) : null, - reorgCount: reorgPodCount ?? 0, + reorgCount: chainReorgs.length, + nodeLocalPruneCount, prunesByType, deepestReorgBlocks: deepest, }; From 9c4cc380183d38ec6066413a3a0aa00fab01c7bb Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 29 Jun 2026 13:30:31 +0000 Subject: [PATCH 33/39] feat(bench): a chain reorg requires all validators to prune MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refine reorg detection: tag each chainPruned event with validatorPruneCount (distinct validator pods that pruned to that block) and compare against the validator fleet size (distinct validators that logged l2-block-handled). reorgCount now counts only events where ALL following validators rewound to the same block — a true canonical reorg. A prune reported by a subset of validators (or by non-validator followers) is a local divergence (nodeLocalPruneCount), not a reorg. Verified on 1/5/10 TPS: validatorCount=12, 0 chain reorgs (prunes had 0-1 validators each). --- .../bench_10tps/bench_output.schema.json | 12 ++- spartan/scripts/bench_10tps/bench_scrape.ts | 85 +++++++++++++++---- 2 files changed, 75 insertions(+), 22 deletions(-) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 6be5b1b06caa..02c180241c38 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -401,11 +401,15 @@ }, "reorgCount": { "type": ["integer", "null"], - "description": "Number of genuine CHAIN reorgs: `Chain pruned` events where 2+ distinct pods pruned to the same block (event.podCount >= 2), i.e. the canonical chain rewound network-wide. A single straggler whose local view diverged and re-synced is NOT counted here — see nodeLocalPruneCount. Log-derived, so accurate regardless of node image." + "description": "Number of genuine CHAIN reorgs: `Chain pruned` events where ALL following validators rewound to the same block (event.validatorPruneCount >= validatorCount), i.e. the canonical chain reorged. Prunes reported by only a subset of validators are local divergences and are NOT counted — see nodeLocalPruneCount. Log-derived, so accurate regardless of node image; 0 if the validator fleet size could not be determined." }, "nodeLocalPruneCount": { "type": ["integer", "null"], - "description": "Number of single-node local prunes (`Chain pruned` events with podCount <= 1): one node's view diverged and re-synced while the canonical chain held. Informational — does not indicate chain instability or make the headline provisional." + "description": "Number of `Chain pruned` events that were NOT chain reorgs: a subset of validators (or non-validator followers) diverged locally and re-synced while the canonical chain held. Informational — does not indicate chain instability or make the headline provisional." + }, + "validatorCount": { + "type": ["integer", "null"], + "description": "Number of validator pods following the chain during the run (distinct validators that logged l2-block-handled). The denominator for chain-reorg detection — a reorg requires all of them to have pruned to the same block." }, "prunesByType": { "type": "object", @@ -715,9 +719,9 @@ "type": "integer", "description": "For chainPruned: the post-prune tip." }, - "podCount": { + "validatorPruneCount": { "type": "integer", - "description": "For chainPruned: distinct pods that logged this prune (same toBlock within the dedupe window). 1 = a single node diverged locally and re-synced; >= 2 = the canonical chain rewound across multiple nodes (a genuine chain reorg)." + "description": "For chainPruned: distinct VALIDATOR pods that logged this prune (same toBlock within the dedupe window). A genuine chain reorg requires this to equal the validator fleet size (summary.validatorCount); fewer means a subset of validators (or non-validator followers) diverged locally and re-synced." }, "slotNumber": { "type": "integer", diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index 5a125b58ce6f..eb2fc76d41a2 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -1426,10 +1426,11 @@ type ChainPrunedEvent = { source: "log"; fromBlock?: number; toBlock?: number; - // Distinct pods that logged this prune (same toBlock within the dedupe - // window). 1 = a single node's local view diverged and re-synced; many = - // the canonical chain rewound network-wide (a genuine chain reorg). - podCount: number; + // Distinct VALIDATOR pods that logged this prune (same toBlock within the + // dedupe window). Compared against the validator fleet size to decide whether + // the canonical chain reorged: < fleet = some validators diverged locally and + // re-synced; == fleet = every validator rewound, a genuine chain reorg. + validatorPruneCount: number; }; type SlotSummaryEvent = { @@ -1529,18 +1530,19 @@ async function scrapeChainPrunedEvents( } } - // Distinct pods that pruned to the same toBlock within the dedupe window: - // 1 means a lone node diverged locally; many means the canonical chain rewound - // across the network. This is what separates a node-local re-sync from a real - // chain reorg. - const podCountFor = (toBlock: number, time: number) => + // Distinct VALIDATOR pods that pruned to the same toBlock within the dedupe + // window. A genuine chain reorg means every validator rewound; a lone validator + // (or non-validator follower) pruning is a local divergence that self-heals. + const validatorPrefix = `${NAMESPACE}-validator-`; + const validatorPruneCountFor = (toBlock: number, time: number) => new Set( parsed .filter( (p) => p.toBlock === toBlock && p.time >= time && - p.time - time < DEDUPE_WINDOW_MS, + p.time - time < DEDUPE_WINDOW_MS && + p.pod.startsWith(validatorPrefix), ) .map((p) => p.pod), ).size; @@ -1558,11 +1560,42 @@ async function scrapeChainPrunedEvents( source: "log" as const, fromBlock: before || undefined, toBlock, - podCount: podCountFor(toBlock, time), + validatorPruneCount: validatorPruneCountFor(toBlock, time), }; }); } +// Number of validator pods actively following the chain — distinct validators +// that logged l2-block-handled in the window. Denominator for chain-reorg +// detection: a genuine reorg requires every following validator to have pruned, +// so down validators are excluded (they can't report and shouldn't gate the +// count). Returns 0 on scrape failure, which disables reorg detection (every +// prune is then treated as node-local) rather than false-positiving. +async function scrapeValidatorFleetSize( + startedAt: string, + endedAt: string, +): Promise { + const filter = [ + `resource.labels.namespace_name="${NAMESPACE}"`, + `resource.labels.pod_name=~"${NAMESPACE}-validator-"`, + `jsonPayload.eventName="l2-block-handled"`, + timeFilter(startedAt, endedAt), + ].join(" AND "); + try { + const entries = await gcloudRead(filter); + return new Set( + entries + .map((e) => e.resource?.labels?.pod_name) + .filter((p): p is string => !!p), + ).size; + } catch (err) { + log("validator fleet-size scrape failed", { + err: err instanceof Error ? err.message : String(err), + }); + return 0; + } +} + async function scrapeSlotSummaryEvents( startedAt: string, endedAt: string, @@ -1902,6 +1935,9 @@ type SummaryArgs = { blocks: BlockRecord[]; events: Event[]; inclusionRecords: InclusionRecord[]; + // Number of validator pods following the chain; the denominator for deciding + // whether a prune was a network-wide chain reorg (all validators) vs local. + validatorCount: number; }; type BlockBuildPerTx = { @@ -2107,16 +2143,21 @@ async function buildSummary(a: SummaryArgs): Promise> { ), ]); - // Separate genuine chain reorgs from single-node local divergences using the - // per-event podCount (distinct pods that pruned to the same block). podCount - // >= 2 means multiple nodes rewound to the same block — the canonical chain - // reorged; podCount <= 1 is one straggler whose local view diverged and - // re-synced, which does NOT destabilise the chain or the headline. Events from - // older scrapes without podCount default to node-local. + // A genuine chain reorg requires EVERY following validator to have rewound to + // the same block (validatorPruneCount >= validatorCount). Anything less is a + // local divergence — a subset of validators (or non-validator followers) that + // pruned and re-synced while the canonical chain held — and does not + // destabilise the chain or the headline. If the validator fleet size is + // unknown (scrape failed) no prune is treated as a chain reorg. const chainPrunes = a.events.filter( (e): e is ChainPrunedEvent => e.type === "chainPruned", ); - const chainReorgs = chainPrunes.filter((e) => (e.podCount ?? 1) >= 2); + const chainReorgs = + a.validatorCount > 0 + ? chainPrunes.filter( + (e) => (e.validatorPruneCount ?? 0) >= a.validatorCount, + ) + : []; const nodeLocalPruneCount = chainPrunes.length - chainReorgs.length; const deepest = chainReorgs.reduce( (max, e) => Math.max(max, (e.fromBlock ?? 0) - (e.toBlock ?? 0)), @@ -2175,6 +2216,7 @@ async function buildSummary(a: SummaryArgs): Promise> { : null, reorgCount: chainReorgs.length, nodeLocalPruneCount, + validatorCount: a.validatorCount, prunesByType, deepestReorgBlocks: deepest, }; @@ -2504,6 +2546,12 @@ async function main(): Promise { }); } + const validatorCount = await scrapeValidatorFleetSize( + args.startedAt, + logEndedAt, + ); + log(`Chain-following validator fleet size: ${validatorCount}`); + log("Scraping sequencer state transition logs from gcloud"); let sequencerStateSlots: SequencerStateSlot[] = []; try { @@ -2535,6 +2583,7 @@ async function main(): Promise { blocks, events, inclusionRecords, + validatorCount, }); const payload = { From fddff4b9e8d077f7710d85ff2284fec88a790813 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 29 Jun 2026 19:58:38 +0000 Subject: [PATCH 34/39] feat(bench): add benchmarkType to scrape output and index (schema v5) Adds an optional run.benchmarkType enum (ingress-inclusion, simulated-proving, real-proving, block-capacity) so the dashboard can group a day's sweeps by benchmark kind. Bumps the run schema 4->5 and the index schema 1->2. - bench_scrape.ts: --benchmark-type arg / BENCH_BENCHMARK_TYPE env, emitted in run - bootstrap.sh: inclusion point passes ingress-inclusion; benchmarkType copied into each index.json entry; upload gate now expects schema "5" --- spartan/bootstrap.sh | 10 ++++++---- .../scripts/bench_10tps/bench_output.schema.json | 14 ++++++++++++-- spartan/scripts/bench_10tps/bench_scrape.ts | 8 +++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 527d9649a7d6..12bc328e3969 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -329,6 +329,7 @@ function bench_inclusion_point { --target-tps "$tps" \ --sweep-id "${BENCH_SWEEP_ID:-}" \ --sweep-label "${BENCH_SWEEP_LABEL:-inclusion-sweep}" \ + --benchmark-type "${BENCH_BENCHMARK_TYPE:-ingress-inclusion}" \ --workload sha256_hash_1024 \ --output "$run_json" \ --inclusion-records "$metadata" \ @@ -357,8 +358,8 @@ function network_bench_upload { # Reject anything that's not the schema we've designed the index against. local schema=$(jq -r .schemaVersion "$run_json") - if [[ "$schema" != "4" ]]; then - echo "[network_bench] run JSON has schemaVersion '$schema', expected '4'; skipping upload" + if [[ "$schema" != "5" ]]; then + echo "[network_bench] run JSON has schemaVersion '$schema', expected '5'; skipping upload" return 0 fi @@ -377,6 +378,7 @@ function network_bench_upload { targetTps: .run.targetTps, sweepId: .run.sweepId, sweepLabel: .run.sweepLabel, + benchmarkType: .run.benchmarkType, workload: .run.workload, testDurationSeconds: .run.testDurationSeconds, namespace: .run.namespace, @@ -399,7 +401,7 @@ function network_bench_upload { gcloud storage cp "${bucket}/index.json" "$idx_local" elif echo "$desc_err" | grep -qiE 'not.?found|matched no objects|404'; then echo "[network_bench] no remote index.json yet; seeding empty" - echo '{"schemaVersion":"1","runs":[]}' > "$idx_local" + echo '{"schemaVersion":"2","runs":[]}' > "$idx_local" else echo "[network_bench] cannot read remote index.json:" echo "$desc_err" | head -5 @@ -407,7 +409,7 @@ function network_bench_upload { fi jq --argjson entry "$entry" --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" ' - .schemaVersion = "1" + .schemaVersion = "2" | .generatedAt = $ts | .runs = ((.runs // []) | map(select(.runId != $entry.runId)) + [$entry] | sort_by(.endedAt) | reverse) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 02c180241c38..5002f8426dfe 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -16,8 +16,8 @@ "properties": { "schemaVersion": { "type": "string", - "const": "4", - "description": "Bump when breaking the schema. Old JSONs keep their previous version so the dashboard can render them side-by-side. v3: timeSeries entries carry `series: [{labels, points}]` instead of bare `points`. v4 (additive): adds optional `provingInfra` (hint-gen + proving-queue-by-job_type series) and `saturation` (per-role ELU/CPU/memory, max + avg) sections, plus `run.sweepId`/`run.sweepLabel` so a night's 1/5/10 TPS points group as one sweep. All v3 fields retained; a v3 run is unchanged apart from this version string." + "const": "5", + "description": "Bump when breaking the schema. Old JSONs keep their previous version so the dashboard can render them side-by-side. v3: timeSeries entries carry `series: [{labels, points}]` instead of bare `points`. v4 (additive): adds optional `provingInfra` (hint-gen + proving-queue-by-job_type series) and `saturation` (per-role ELU/CPU/memory, max + avg) sections, plus `run.sweepId`/`run.sweepLabel` so a night's 1/5/10 TPS points group as one sweep. v5 (additive): adds `run.benchmarkType` so the dashboard can group a day's sweeps by the kind of benchmark (ingress-inclusion, simulated-proving, real-proving, block-capacity). All prior fields retained." }, "run": { "$ref": "#/$defs/runMeta" @@ -148,6 +148,16 @@ "type": "string", "description": "v4. Human-readable label for the sweep this run belongs to (e.g. 'inclusion-sweep' or 'proving-sweep'). Optional." }, + "benchmarkType": { + "type": "string", + "enum": [ + "ingress-inclusion", + "simulated-proving", + "real-proving", + "block-capacity" + ], + "description": "v5. The kind of benchmark this run measures. Groups a day's sweeps in the dashboard sidebar (Day -> benchmark type). 'real-proving' typically only runs weekly. Optional; absent runs are treated as 'ingress-inclusion'." + }, "testDurationSeconds": { "type": "integer", "minimum": 0 diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index eb2fc76d41a2..2807d35d2cea 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -64,6 +64,7 @@ type Args = { maxPendingWaitSeconds: number; sweepId: string | undefined; sweepLabel: string | undefined; + benchmarkType: string | undefined; }; function parseArgs(): Args { @@ -104,6 +105,8 @@ function parseArgs(): Args { ), sweepId: get("--sweep-id", env.BENCH_SWEEP_ID ?? "") || undefined, sweepLabel: get("--sweep-label", env.BENCH_SWEEP_LABEL ?? "") || undefined, + benchmarkType: + get("--benchmark-type", env.BENCH_BENCHMARK_TYPE ?? "") || undefined, }; } @@ -2587,7 +2590,7 @@ async function main(): Promise { }); const payload = { - schemaVersion: "4", + schemaVersion: "5", run: { runId: args.runId, startedAt: args.startedAt, @@ -2604,6 +2607,9 @@ async function main(): Promise { targetTps: args.targetTps, ...(args.sweepId !== undefined && { sweepId: args.sweepId }), ...(args.sweepLabel !== undefined && { sweepLabel: args.sweepLabel }), + ...(args.benchmarkType !== undefined && { + benchmarkType: args.benchmarkType, + }), testDurationSeconds: windowSec, workload: args.workload, ...(Object.keys(aztecConfig).length > 0 && { aztecConfig }), From 71dc83bc247935258c83a12f246566ed67a2343d Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Tue, 30 Jun 2026 10:52:53 +0000 Subject: [PATCH 35/39] test(p2p): de-flake mempool mined-delay test with ManualDateProvider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestDateProvider.now() tracks the real clock, so minedDelayMs was the 5s advance plus the real elapsed time between addPendingTxs and handleMinedBlock — which could exceed the < 6000ms bound on a slow/loaded CI box. Switch to ManualDateProvider (frozen clock, advances only via advanceTime) so the delay is deterministically exactly 5000ms. --- .../p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts index c28ee3da302f..af9cb1a1d63f 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts @@ -9,7 +9,7 @@ import { BlockNumber, CheckpointNumber, IndexWithinCheckpoint, SlotNumber } from import { timesAsync } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { createLogger } from '@aztec/foundation/log'; -import { DateProvider, TestDateProvider } from '@aztec/foundation/timer'; +import { DateProvider, ManualDateProvider } from '@aztec/foundation/timer'; import type { AztecAsyncMap } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { RevertCode } from '@aztec/stdlib/avm'; @@ -1580,7 +1580,7 @@ describe('TxPoolV2', () => { it('reports now - receivedAt for a tx that was pending in the pool', async () => { const minedCalls: MinedTxInfo[][] = []; - const dateProvider = new TestDateProvider(); + const dateProvider = new ManualDateProvider(); const { impl, cleanup } = await makeImpl(m => minedCalls.push(m), dateProvider); try { const tx = await mockTx(1); @@ -1593,9 +1593,8 @@ describe('TxPoolV2', () => { const minedTxs = minedCalls[0]; expect(minedTxs).toHaveLength(1); expect(minedTxs[0].txHash).toBe(hashOf(tx)); - // 5s advance dominates; real elapsed during the test is a few ms. - expect(minedTxs[0].minedDelayMs).toBeGreaterThanOrEqual(5000); - expect(minedTxs[0].minedDelayMs).toBeLessThan(6000); + // Frozen clock: the delay is exactly the 5s advance, with no real-time drift. + expect(minedTxs[0].minedDelayMs).toBe(5000); } finally { await cleanup(); } @@ -1603,7 +1602,7 @@ describe('TxPoolV2', () => { it('leaves the delay undefined when receivedAt is unknown (hydrated/restart tx)', async () => { const minedCalls: MinedTxInfo[][] = []; - const dateProvider = new TestDateProvider(); + const dateProvider = new ManualDateProvider(); const { impl, cleanup } = await makeImpl(m => minedCalls.push(m), dateProvider); try { const tx = await mockTx(1); From 46e23e56ef31d2024a50c3421aa4b84f1d2c522e Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Tue, 30 Jun 2026 10:53:58 +0000 Subject: [PATCH 36/39] Updated comment --- yarn-project/telemetry-client/src/metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 3ea78f33245f..a4e59b653bf8 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -341,7 +341,7 @@ export const ARCHIVER_PRUNE_DURATION: MetricDefinition = { export const ARCHIVER_PRUNE_COUNT: MetricDefinition = { name: 'aztec.archiver.prune_count', description: - 'Number of prunes detected, dimensioned by prune_type: unproven (epoch prune of checkpoints that will not be proven), uncheckpointed (proposed blocks whose slot ended without a checkpoint), l1_conflict (proposed blocks conflicting with an L1 checkpoint), and orphan (proposed blocks whose matching proposed checkpoint never arrived before the deadline). Only unproven was counted before this dimension was added; the rest are pending-chain reorgs previously observable only in logs.', + 'Number of prunes detected, dimensioned by prune_type: unproven (epoch prune of checkpoints that will not be proven), uncheckpointed (proposed blocks whose slot ended without a checkpoint), l1_conflict (proposed blocks conflicting with an L1 checkpoint), and orphan (proposed blocks whose matching proposed checkpoint never arrived before the deadline).', valueType: ValueType.INT, }; From 144319ea40cb6d8581cc63a26dc729f5e2a5b3bf Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Tue, 30 Jun 2026 11:07:32 +0000 Subject: [PATCH 37/39] chore(bench): drop the obsolete bench_10tps scenario from spartan bootstrap The single-shot 10 TPS bench (bench_10tps / bench_10tps_cmds) is superseded by the inclusion sweep (bench_inclusion_point, run at 1/5/10 TPS). Removes the dead functions and the case-dispatch entry; the shared scraper at scripts/bench_10tps/bench_scrape.ts is still used by the sweep and is untouched. --- spartan/bootstrap.sh | 56 +++----------------------------------------- 1 file changed, 3 insertions(+), 53 deletions(-) diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 12bc328e3969..344ad7ca7d03 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -182,18 +182,6 @@ function block_capacity_bench_cmds { echo "$(hash):TIMEOUT=${timeout} BENCH_OUTPUT=bench-out/block_capacity.bench.json $root/yarn-project/end-to-end/scripts/run_test.sh simple block_capacity.test.ts" } -function bench_10tps_cmds { - # Mix: 1 tps of high-value + 9 tps of low-value, total still 10 tps. The - # high-value lane is what we measure for the headline client-observed - # inclusion latency (low-value txs pay near-network-min and are allowed to - # fail fee checks, so they would skew the headline if measured). - local high_value_tps=1 - local low_value_tps=9 - local test_duration=${TEST_DURATION_SECONDS:-600} # 10 mins - local timeout=${BENCH_TIMEOUT_SECONDS:-7200} # account for initial committee formation - echo "$(hash):TIMEOUT=${timeout} BENCH_RUN_ID=${BENCH_RUN_ID:-} BENCH_OUTPUT=bench-out/n_tps.10tps.bench.json BENCH_SCENARIO=10tps LOW_VALUE_TPS=${low_value_tps} HIGH_VALUE_TPS=${high_value_tps} TEST_DURATION_SECONDS=${test_duration} $root/yarn-project/end-to-end/scripts/run_test.sh simple n_tps.test.ts" -} - function network_bench { rm -rf bench-out mkdir -p bench-out @@ -236,50 +224,12 @@ function block_capacity_bench { block_capacity_bench_cmds | parallelize 1 } -function bench_10tps { - rm -rf bench-out - mkdir -p bench-out - - local env_file="$1" - source_network_env $env_file - - echo_header "spartan bench-10tps" - gcp_auth - export_admin_api_key - export K8S_ENRICHER=${K8S_ENRICHER:-1} - export BENCH_RUN_ID="${BENCH_RUN_ID:-$(date -u +%Y%m%d)-${COMMIT_HASH:0:10}}" - bench_10tps_cmds | parallelize 1 - - local metadata="/tmp/n_tps_timing_data.json" - local run_json="bench-out/bench-10tps-${BENCH_RUN_ID}.json" - if [[ -f "$metadata" ]]; then - local started=$(jq -r .startedAt < "$metadata") - local ended=$(jq -r .endedAt < "$metadata") - echo "Scraping bench-10tps run ${BENCH_RUN_ID} (started=${started} ended=${ended})" - NAMESPACE="$NAMESPACE" GCP_PROJECT_ID="${GCP_PROJECT_ID:-}" ./scripts/bench_10tps/bench_scrape.ts \ - --run-id "$BENCH_RUN_ID" \ - --started "$started" \ - --ended "$ended" \ - --target-tps 10 \ - --workload sha256_hash_1024 \ - --output "$run_json" \ - --inclusion-records "$metadata" \ - --wait-for-pending-zero \ - --max-pending-wait-seconds "${BENCH_SCRAPE_MAX_PENDING_WAIT_SECONDS:-3600}" \ - || echo "[bench_10tps] scraper failed (non-fatal)" - network_bench_upload "$run_json" || echo "[network_bench] upload failed (non-fatal)" - else - echo "[bench_10tps] no timing metadata at ${metadata}; skipping scraper" - fi -} - -# One point of the inclusion sweep. Same scrape+upload path as -# bench_10tps, and the same load model: a fixed 1 TPS of high-value txs (the +# One point of the inclusion sweep: a fixed 1 TPS of high-value txs (the # measured inclusion lane) plus (TARGET_TPS - 1) TPS of low-value background # traffic to bring total load to the target. So inclusion latency is always # measured for a properly-paying user's tx while the network runs at TARGET_TPS. # Tagged with a shared BENCH_SWEEP_ID so the 1/5/10 points group as one sweep -# (schema v4). Each point runs in its own namespace. +# (schema v5). Each point runs in its own namespace. function bench_inclusion_point_cmds { local tps=${TARGET_TPS:-10} local high_value_tps=1 @@ -490,7 +440,7 @@ case "$cmd" in run_network_tests "$1" "$2" ;; - network_tests|network_tests_1|network_tests_2|network_bench|proving_bench|block_capacity_bench|bench_10tps|bench_inclusion_point) + network_tests|network_tests_1|network_tests_2|network_bench|proving_bench|block_capacity_bench|bench_inclusion_point) env_file="$1" $cmd "$env_file" ;; From 22c8edeb51e0e1c1eb7b09686b54bb00e47dcaf5 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 1 Jul 2026 08:52:17 +0000 Subject: [PATCH 38/39] fix(bench): scraper schema-version check to v5; count checkpoint-mismatch prune - bench_scrape.ts asserted schemaVersion "4" while emitting "5", so the scraper threw on its own output; align the check and refresh the stale header comment (bench_10tps -> bench_inclusion_point). - Archiver: the checkpointed-tip-vs-L1 mismatch rewind (removeCheckpointsAfter in l1_synchronizer) emitted no prune_count. Add a new 'l1_mismatch' prune_type for it (distinct from the proposed-block causes) and surface it in prunesByType + schema. --- .../scripts/bench_10tps/bench_output.schema.json | 3 ++- spartan/scripts/bench_10tps/bench_scrape.ts | 15 ++++++++------- .../archiver/src/modules/instrumentation.ts | 8 +++++--- .../archiver/src/modules/l1_synchronizer.ts | 3 +++ yarn-project/telemetry-client/src/metrics.ts | 2 +- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/spartan/scripts/bench_10tps/bench_output.schema.json b/spartan/scripts/bench_10tps/bench_output.schema.json index 5002f8426dfe..527dc9c10271 100644 --- a/spartan/scripts/bench_10tps/bench_output.schema.json +++ b/spartan/scripts/bench_10tps/bench_output.schema.json @@ -429,7 +429,8 @@ "unproven": { "type": ["number", "null"] }, "uncheckpointed": { "type": ["number", "null"] }, "l1_conflict": { "type": ["number", "null"] }, - "orphan": { "type": ["number", "null"] } + "orphan": { "type": ["number", "null"] }, + "l1_mismatch": { "type": ["number", "null"] } } }, "deepestReorgBlocks": { diff --git a/spartan/scripts/bench_10tps/bench_scrape.ts b/spartan/scripts/bench_10tps/bench_scrape.ts index 2807d35d2cea..581a1d6b414c 100755 --- a/spartan/scripts/bench_10tps/bench_scrape.ts +++ b/spartan/scripts/bench_10tps/bench_scrape.ts @@ -1,10 +1,10 @@ #!/usr/bin/env -S node --experimental-strip-types --no-warnings // -// Scrape a completed bench-10tps run into a schema-conformant JSON payload. -// Contract: bench_output.schema.json (v4). Invoked by the bench_10tps function -// in spartan/bootstrap.sh after n_tps.test.ts finishes. +// Scrape a completed inclusion-sweep run into a schema-conformant JSON payload. +// Contract: bench_output.schema.json (v5). Invoked by the bench_inclusion_point +// function in spartan/bootstrap.sh after n_tps.test.ts finishes. // -// v4 adds two PromQL sections alongside the inclusion timeSeries: +// Alongside the inclusion timeSeries the payload carries two PromQL sections: // - provingInfra: prover-node hint-gen (tx re-execution) + proving-queue // behaviour broken down by job_type. // - saturation: per-role ELU/CPU/memory, each as max (hottest pod) + avg. @@ -618,7 +618,7 @@ const TIME_SERIES_DEFS: Record = { query: `sum(rate(aztec_validator_attestation_success_count${NS}[1m]))`, }, // Archiver prunes broken down by cause (prune_type). 'unproven' is the - // proven/epoch prune; 'uncheckpointed'/'l1_conflict'/'orphan' are pending-chain + // proven/epoch prune; 'uncheckpointed'/'l1_conflict'/'orphan'/'l1_mismatch' are pending-chain // reorgs that move a node's proposed tip (world-state then logs "Chain pruned"). // Summed across pods, so a value here is the network-wide prune rate per cause; // see summary.reorgCount for how many distinct pods diverged. Requires the @@ -2174,6 +2174,7 @@ async function buildSummary(a: SummaryArgs): Promise> { "uncheckpointed", "l1_conflict", "orphan", + "l1_mismatch", ] as const; const pruneTypeTotals = await Promise.all( pruneTypes.map((t) => @@ -2243,9 +2244,9 @@ function assertShape(payload: Record): void { throw new Error(`output missing required top-level key: ${key}`); } } - if (payload.schemaVersion !== "4") { + if (payload.schemaVersion !== "5") { throw new Error( - `schemaVersion must be "4", got ${String(payload.schemaVersion)}`, + `schemaVersion must be "5", got ${String(payload.schemaVersion)}`, ); } const run = payload.run as Record; diff --git a/yarn-project/archiver/src/modules/instrumentation.ts b/yarn-project/archiver/src/modules/instrumentation.ts index d73726d946d4..2ea73a46ca70 100644 --- a/yarn-project/archiver/src/modules/instrumentation.ts +++ b/yarn-project/archiver/src/modules/instrumentation.ts @@ -158,10 +158,12 @@ export class ArchiverInstrumentation { /** * Records a pending-chain reorg, where the archiver dropped proposed blocks (and world-state follows by pruning). The - * type distinguishes the cause: 'uncheckpointed' (slot ended without a checkpoint), 'l1_conflict' (blocks conflicting - * with an L1 checkpoint), or 'orphan' (no matching proposed checkpoint arrived before the deadline). + * type distinguishes the cause: 'uncheckpointed' (slot ended without a checkpoint), 'l1_conflict' (proposed blocks + * conflicting with an L1 checkpoint), 'orphan' (no matching proposed checkpoint arrived before the deadline), or + * 'l1_mismatch' (the local checkpointed tip diverged from L1 — an L1 reorg or a pruned/missed-proof checkpoint — so + * already-checkpointed blocks were rewound). */ - public recordPrune(pruneType: 'uncheckpointed' | 'l1_conflict' | 'orphan') { + public recordPrune(pruneType: 'uncheckpointed' | 'l1_conflict' | 'orphan' | 'l1_mismatch') { this.pruneCount.add(1, { [Attributes.PRUNE_TYPE]: pruneType }); } diff --git a/yarn-project/archiver/src/modules/l1_synchronizer.ts b/yarn-project/archiver/src/modules/l1_synchronizer.ts index 6a6758467035..922bd2f3766a 100644 --- a/yarn-project/archiver/src/modules/l1_synchronizer.ts +++ b/yarn-project/archiver/src/modules/l1_synchronizer.ts @@ -783,6 +783,9 @@ export class ArchiverL1Synchronizer implements Traceable { const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind; await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind)); + if (checkpointsToRemove > 0) { + this.instrumentation.recordPrune('l1_mismatch'); + } this.log.warn( `Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` + diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index a4e59b653bf8..7342f8bd4668 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -341,7 +341,7 @@ export const ARCHIVER_PRUNE_DURATION: MetricDefinition = { export const ARCHIVER_PRUNE_COUNT: MetricDefinition = { name: 'aztec.archiver.prune_count', description: - 'Number of prunes detected, dimensioned by prune_type: unproven (epoch prune of checkpoints that will not be proven), uncheckpointed (proposed blocks whose slot ended without a checkpoint), l1_conflict (proposed blocks conflicting with an L1 checkpoint), and orphan (proposed blocks whose matching proposed checkpoint never arrived before the deadline).', + 'Number of prunes detected, dimensioned by prune_type: unproven (epoch prune of checkpoints that will not be proven), uncheckpointed (proposed blocks whose slot ended without a checkpoint), l1_conflict (proposed blocks conflicting with an L1 checkpoint), orphan (proposed blocks whose matching proposed checkpoint never arrived before the deadline), and l1_mismatch (the local checkpointed tip diverged from L1 — an L1 reorg or a pruned/missed-proof checkpoint — rewinding already-checkpointed blocks).', valueType: ValueType.INT, }; From c1d9ee2d8539f5e0cb854817afe7ab33576f9a3c Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Thu, 2 Jul 2026 19:26:58 +0000 Subject: [PATCH 39/39] fix(archiver): zero-seed l1_mismatch prune_type counter l1_mismatch was added to recordPrune and emitted from the l1 synchronizer, but omitted from the prune_count default-values list, so the prune_count{prune_type="l1_mismatch"} series only appeared after the first such prune instead of from startup like the other causes. Add it to the zero-seed list and to the PRUNE_TYPE attribute doc comment. --- yarn-project/archiver/src/modules/instrumentation.ts | 2 +- yarn-project/telemetry-client/src/attributes.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/archiver/src/modules/instrumentation.ts b/yarn-project/archiver/src/modules/instrumentation.ts index 2ea73a46ca70..20eeff25311e 100644 --- a/yarn-project/archiver/src/modules/instrumentation.ts +++ b/yarn-project/archiver/src/modules/instrumentation.ts @@ -85,7 +85,7 @@ export class ArchiverInstrumentation { this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION); this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT, { - [Attributes.PRUNE_TYPE]: ['unproven', 'uncheckpointed', 'l1_conflict', 'orphan'], + [Attributes.PRUNE_TYPE]: ['unproven', 'uncheckpointed', 'l1_conflict', 'orphan', 'l1_mismatch'], }); this.blockProposalTxTargetCount = createUpDownCounterWithDefault( diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index be481ca7b58b..8a262c03efe6 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -73,7 +73,7 @@ export const OK = 'aztec.ok'; export const STATUS = 'aztec.status'; /** Generic error type attribute */ export const ERROR_TYPE = 'aztec.error_type'; -/** The cause of an archiver prune (unproven / uncheckpointed / l1_conflict / orphan) */ +/** The cause of an archiver prune (unproven / uncheckpointed / l1_conflict / orphan / l1_mismatch) */ export const PRUNE_TYPE = 'aztec.archiver.prune_type'; /** The type of the transaction */ export const L1_TX_TYPE = 'aztec.l1.tx_type';