feat(sequencer): extract pipelined checkpoint out-hash helper, reuse in validators

The proposer-side fix from #23110 (parent checkpointOutHash splice under
pipelining) was inlined as a private method on `CheckpointProposalJob`. The
validator's block re-execution and checkpoint-proposal validation paths in
`proposal_handler.ts` compute the same `previousCheckpointOutHashes` list
through the same archiver-driven query, so they have the same off-by-one
window: if the parent cp lands on L1 between when the validator pulls and
when it re-derives, only the proposer would carry the spliced parent and
attestations would mismatch.

Extract the proposer's logic into a shared `getPreviousCheckpointOutHashes`
helper in `stdlib/src/checkpoint/`. The helper accepts the proposer's
already-loaded `proposedCheckpointData` directly, and falls back on
`L2BlockSource.getProposedCheckpointData(...)` for callers that don't have
it on hand (validator). Wire the helper into the proposer (replacing the
private method) and into both validator sites.

Add a few diagnostics that helped pinpoint this class of bug:
- `prover-node-publisher.ts`: when the L1-recomputed `RootRollupPublicInputs`
  vector mismatches the prover's, decode the differing indices into labels
  (`previousArchiveRoot`, `endArchiveRoot`, `outHash`,
  `checkpointHeaderHashes[i]`, `fees[i].recipient/value`, `constants.*`,
  `blobPublicInputs[*]`), fetch the L1 `CheckpointLog` for any mismatching
  `checkpointHeaderHashes[i]`, and emit a structured error log alongside
  the throw — much easier to triage than the previous opaque dump.
- `BlockRollupPublicInputs.toInspect()` and
  `CheckpointRollupPublicInputs.toInspect()` to keep per-stage orchestrator
  debug logs short.
- Per-stage debug logs in the orchestrator (block-root, block-merge,
  checkpoint-root) consume the new `toInspect()` outputs.
- Lightweight checkpoint builder logs `headerHash` and the size of
  `previousCheckpointOutHashes` at debug.
- Epoch proving job's per-checkpoint start log trimmed to the fields that
  are actually useful for cross-comparison.

Author: spalladino

yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts

@@ -290,6 +290,14 @@ export class LightweightCheckpointBuilder {
       totalManaUsed,
     });
 
+    this.logger.debug(`Completed checkpoint ${this.checkpointNumber}`, {
+      checkpointNumber: this.checkpointNumber,
+      headerHash: header.hash().toString(),
+      checkpointOutHash: checkpointOutHash.toString(),
+      numPreviousCheckpointOutHashes: this.previousCheckpointOutHashes.length,
+      ...header.toInspect(),
+    });
+
     return new Checkpoint(newArchive, header, blocks, this.checkpointNumber, this.feeAssetPriceModifier);
   }
 

yarn-project/prover-client/src/orchestrator/orchestrator.ts

@@ -896,7 +896,11 @@ export class ProvingOrchestrator implements EpochProver {
         },
       ),
       async result => {
-        this.logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
+        this.logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`, {
+          blockNumber: provingState.blockNumber,
+          checkpointIndex: provingState.parentCheckpoint.index,
+          ...result.inputs.toInspect(),
+        });
 
         const leafLocation = provingState.setBlockRootRollupProof(result);
         const checkpointProvingState = provingState.parentCheckpoint;
@@ -1015,6 +1019,11 @@ export class ProvingOrchestrator implements EpochProver {
         signal => this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber),
       ),
       async result => {
+        this.logger.debug(`Completed block merge rollup proof for checkpoint ${provingState.index}`, {
+          checkpointIndex: provingState.index,
+          mergeLocation: location,
+          ...result.inputs.toInspect(),
+        });
         provingState.setBlockMergeRollupProof(location, result);
         await this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
       },
@@ -1067,7 +1076,10 @@ export class ProvingOrchestrator implements EpochProver {
           return;
         }
 
-        this.logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
+        this.logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}`, {
+          checkpointIndex: provingState.index,
+          ...result.inputs.toInspect(),
+        });
 
         const leafLocation = provingState.setCheckpointRootRollupProof(result);
         const epochProvingState = provingState.parentEpoch;

yarn-project/prover-node/src/job/epoch-proving-job.ts

@@ -191,11 +191,12 @@ export class EpochProvingJob implements Traceable {
         const previousHeader = previousBlockHeaders[checkpointIndex];
         const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
 
-        this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
+        this.log.debug(`Starting processing checkpoint ${checkpoint.number}`, {
           number: checkpoint.number,
           checkpointHash: checkpoint.hash().toString(),
-          lastArchive: checkpoint.header.lastArchiveRoot,
-          previousHeader: previousHeader.hash(),
+          headerHash: checkpoint.header.hash().toString(),
+          numL1ToL2Messages: l1ToL2Messages.length,
+          previousBlockNumber: previousHeader.globalVariables.blockNumber,
           uuid: this.uuid,
         });
 

yarn-project/prover-node/src/prover-node-publisher.ts

@@ -203,10 +203,14 @@ export class ProverNodePublisher {
     const argsPublicInputs = [...publicInputs.toFields()];
 
     if (!areArraysEqual(rollupPublicInputs, argsPublicInputs, (a, b) => a.equals(b))) {
-      const fmt = (inputs: Fr[] | readonly string[]) => inputs.map(x => x.toString()).join(', ');
-      throw new Error(
-        `Root rollup public inputs mismatch:\nRollup:  ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
-      );
+      throw await reportPublicInputsMismatch({
+        rollupPublicInputs,
+        argsPublicInputs,
+        fromCheckpoint,
+        toCheckpoint,
+        rollupContract: this.rollupContract,
+        log: this.log,
+      });
     }
   }
 
@@ -372,3 +376,100 @@ export class ProverNodePublisher {
     };
   }
 }
+
+/**
+ * Decodes a `Root rollup public inputs mismatch`, fetches the on-chain CheckpointLog for any
+ * mismatching `checkpointHeaderHashes[i]`, emits a structured error log, and returns a thrown-ready
+ * Error with a human-readable summary.
+ *
+ * Layout of `RootRollupPublicInputs.toFields()`:
+ *   [0]                   previousArchiveRoot
+ *   [1]                   endArchiveRoot
+ *   [2]                   outHash
+ *   [3 .. 3+N-1]          checkpointHeaderHashes[i] for i in 0..N-1   (N = MAX_CHECKPOINTS_PER_EPOCH)
+ *   [3+N .. 3+3N-1]       fees[i] = (recipient, value) for i in 0..N-1
+ *   [3+3N .. 3+3N+4]      EpochConstantData (chainId, version, vkTreeRoot, protocolContractsHash, proverId)
+ *   [3+3N+5 ..]           blobPublicInputs (FinalBlobAccumulator)
+ */
+async function reportPublicInputsMismatch(input: {
+  rollupPublicInputs: readonly Fr[];
+  argsPublicInputs: readonly Fr[];
+  fromCheckpoint: CheckpointNumber;
+  toCheckpoint: CheckpointNumber;
+  rollupContract: RollupContract;
+  log: Logger;
+}): Promise<Error> {
+  const { rollupPublicInputs, argsPublicInputs, fromCheckpoint, toCheckpoint, rollupContract, log } = input;
+  const N = MAX_CHECKPOINTS_PER_EPOCH;
+  const constantsStart = 3 + 3 * N;
+  const blobStart = constantsStart + 5;
+  const constantLabels = ['chainId', 'version', 'vkTreeRoot', 'protocolContractsHash', 'proverId'];
+
+  const diffs: { index: number; label: string; rollup: Fr; computed: Fr; checkpointIndex?: number }[] = [];
+  const len = Math.max(rollupPublicInputs.length, argsPublicInputs.length);
+  for (let i = 0; i < len; i++) {
+    const a = rollupPublicInputs[i] ?? Fr.ZERO;
+    const b = argsPublicInputs[i] ?? Fr.ZERO;
+    if (a.equals(b)) {
+      continue;
+    }
+    let label: string;
+    let checkpointIndex: number | undefined;
+    if (i === 0) {
+      label = 'previousArchiveRoot';
+    } else if (i === 1) {
+      label = 'endArchiveRoot';
+    } else if (i === 2) {
+      label = 'outHash';
+    } else if (i < 3 + N) {
+      checkpointIndex = i - 3;
+      label = `checkpointHeaderHashes[${checkpointIndex}]`;
+    } else if (i < 3 + 3 * N) {
+      const feePairIndex = i - (3 + N);
+      const feeIndex = Math.floor(feePairIndex / 2);
+      const sub = feePairIndex % 2 === 0 ? 'recipient' : 'value';
+      label = `fees[${feeIndex}].${sub}`;
+    } else if (i < blobStart) {
+      label = `constants.${constantLabels[i - constantsStart]}`;
+    } else {
+      label = `blobPublicInputs[${i - blobStart}]`;
+    }
+    diffs.push({ index: i, label, rollup: a, computed: b, checkpointIndex });
+  }
+
+  // For each mismatching checkpointHeaderHash, fetch the L1 CheckpointLog so the operator can
+  // see what was published on-chain alongside the prover's recomputed hash.
+  const onChainCheckpoints = await Promise.all(
+    diffs
+      .filter(d => d.checkpointIndex !== undefined)
+      .map(async d => {
+        const checkpointNumber = CheckpointNumber(fromCheckpoint + d.checkpointIndex!);
+        try {
+          const cp = await rollupContract.getCheckpoint(checkpointNumber);
+          return { checkpointIndex: d.checkpointIndex!, checkpointNumber, headerHash: cp.headerHash.toString() };
+        } catch (err) {
+          return { checkpointIndex: d.checkpointIndex!, checkpointNumber, error: (err as Error).message };
+        }
+      }),
+  );
+
+  log.error(`Root rollup public inputs mismatch`, undefined, {
+    fromCheckpoint,
+    toCheckpoint,
+    numDiffs: diffs.length,
+    diffs: diffs.map(d => ({
+      index: d.index,
+      label: d.label,
+      rollup: d.rollup.toString(),
+      computed: d.computed.toString(),
+    })),
+    onChainCheckpoints,
+  });
+
+  const fmt = (inputs: readonly Fr[]) => inputs.map(x => x.toString()).join(', ');
+  const summary = diffs.map(d => `[${d.index} ${d.label}] L1=${d.rollup} prover=${d.computed}`).join('\n');
+  return new Error(
+    `Root rollup public inputs mismatch (${diffs.length} fields differ):\n${summary}\n` +
+      `Rollup:  ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
+  );
+}

yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts

@@ -33,13 +33,13 @@ import {
   MaliciousCommitteeAttestationsAndSigners,
   type ValidateCheckpointResult,
 } from '@aztec/stdlib/block';
-import { type Checkpoint, type ProposedCheckpointData, validateCheckpoint } from '@aztec/stdlib/checkpoint';
 import {
-  computeQuorum,
-  getEpochAtSlot,
-  getSlotStartBuildTimestamp,
-  getTimestampForSlot,
-} from '@aztec/stdlib/epoch-helpers';
+  type Checkpoint,
+  type ProposedCheckpointData,
+  getPreviousCheckpointOutHashes,
+  validateCheckpoint,
+} from '@aztec/stdlib/checkpoint';
+import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import { Gas } from '@aztec/stdlib/gas';
 import {
   type BlockBuilderOptions,
@@ -416,37 +416,6 @@ export class CheckpointProposalJob implements Traceable {
     }
   }
 
-  /**
-   * Returns the out hashes of all checkpoints in `targetEpoch` that precede the one being built.
-   * Under pipelining, the parent checkpoint may not be on L1 yet at build time, so the on-chain
-   * archiver is missing it; in that case we splice in the parent's `checkpointOutHash` from the
-   * proposed-checkpoint payload (when it is in the same epoch) so the resulting `epochOutHash`
-   * matches what other validators and L1 will compute once the parent lands.
-   */
-  private async collectPreviousCheckpointOutHashes(): Promise<Fr[]> {
-    const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
-    const checkpointed = (await this.l2BlockSource.getCheckpointsData({ epoch: this.targetEpoch }))
-      .filter(c => c.checkpointNumber < this.checkpointNumber)
-      .map(c => ({ checkpointNumber: c.checkpointNumber, checkpointOutHash: c.checkpointOutHash }));
-
-    const shouldSpliceParent =
-      this.epochCache.isProposerPipeliningEnabled() &&
-      this.proposedCheckpointData !== undefined &&
-      this.proposedCheckpointData.checkpointNumber === parentCheckpointNumber &&
-      getEpochAtSlot(this.proposedCheckpointData.header.slotNumber, this.epochCache.getL1Constants()) ===
-        this.targetEpoch &&
-      !checkpointed.some(c => c.checkpointNumber === parentCheckpointNumber);
-
-    if (shouldSpliceParent) {
-      checkpointed.push({
-        checkpointNumber: parentCheckpointNumber,
-        checkpointOutHash: this.proposedCheckpointData!.checkpointOutHash,
-      });
-    }
-
-    return checkpointed.sort((a, b) => a.checkpointNumber - b.checkpointNumber).map(c => c.checkpointOutHash);
-  }
-
   /**
    * Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
    * Polls until the archiver has synced L1 past the parent's slot, then verifies:
@@ -620,11 +589,19 @@ export class CheckpointProposalJob implements Traceable {
       const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
 
       // Collect the out hashes of all the checkpoints before this one in the same epoch.
-      // Under pipelining, the parent checkpoint may not be on L1 yet at build time, so
-      // `getCheckpointsData` would miss it. Splice in the parent's checkpointOutHash from the
-      // proposed-checkpoint payload so the resulting `epochOutHash` matches what the validators
-      // (and L1) compute once the parent lands on L1.
-      const previousCheckpointOutHashes = await this.collectPreviousCheckpointOutHashes();
+      // Under pipelining the parent checkpoint may not be on L1 yet at build time, so the helper
+      // splices in the parent's checkpointOutHash from the locally-known proposed checkpoint so
+      // the resulting `epochOutHash` matches what validators (and L1) compute once the parent
+      // lands on L1.
+      const previousCheckpointOutHashes = await getPreviousCheckpointOutHashes({
+        blockSource: this.l2BlockSource,
+        epoch: this.targetEpoch,
+        checkpointNumber: this.checkpointNumber,
+        l1Constants: this.epochCache.getL1Constants(),
+        pipeliningEnabled: this.epochCache.isProposerPipeliningEnabled(),
+        proposedCheckpointData: this.proposedCheckpointData,
+        log: this.log,
+      });
 
       // Get the fee asset price modifier from the oracle
       const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier();

yarn-project/stdlib/src/checkpoint/index.ts

@@ -2,5 +2,6 @@ export * from './checkpoint.js';
 export * from './checkpoint_data.js';
 export * from './checkpoint_info.js';
 export * from './digest.js';
+export * from './previous_checkpoint_out_hashes.js';
 export * from './published_checkpoint.js';
 export * from './validate.js';

yarn-project/stdlib/src/checkpoint/previous_checkpoint_out_hashes.ts

@@ -0,0 +1,61 @@
+import { CheckpointNumber, type EpochNumber } from '@aztec/foundation/branded-types';
+import type { Fr } from '@aztec/foundation/curves/bn254';
+import type { Logger } from '@aztec/foundation/log';
+
+import type { L2BlockSource } from '../block/l2_block_source.js';
+import { type L1RollupConstants, getEpochAtSlot } from '../epoch-helpers/index.js';
+import type { ProposedCheckpointData } from './checkpoint_data.js';
+
+/**
+ * Returns the out hashes (in epoch order) of all checkpoints in `epoch` that precede
+ * `checkpointNumber`, used to compute the `epochOutHash` baked into that checkpoint's header.
+ *
+ * Under proposer pipelining the parent checkpoint may not be confirmed on L1 by the time the
+ * builder runs, so the on-chain archiver query (`getCheckpointsData`) is missing it and the
+ * resulting `epochOutHash` would diverge from what other validators (and L1) compute after the
+ * parent lands. To avoid that, the parent's `checkpointOutHash` is spliced in from the locally
+ * known proposed checkpoint when:
+ *   - pipelining is enabled,
+ *   - the archiver lookup is genuinely short (its last entry is not already cp `N-1`),
+ *   - and the proposed cp `N-1` is in the same target epoch (otherwise we're at an epoch boundary
+ *     and the previous-epoch cps must be excluded).
+ *
+ * Callers may either pass the already-loaded `proposedCheckpointData` (the proposer has it on
+ * hand) or leave it undefined, in which case it's fetched via `getProposedCheckpointData`.
+ */
+export async function getPreviousCheckpointOutHashes(input: {
+  blockSource: Pick<L2BlockSource, 'getCheckpointsData' | 'getProposedCheckpointData'>;
+  epoch: EpochNumber;
+  checkpointNumber: CheckpointNumber;
+  l1Constants: Pick<L1RollupConstants, 'epochDuration'>;
+  pipeliningEnabled: boolean;
+  proposedCheckpointData?: ProposedCheckpointData;
+  log?: Logger;
+}): Promise<Fr[]> {
+  const { blockSource, epoch, checkpointNumber, l1Constants, pipeliningEnabled, log } = input;
+
+  const checkpointed = (await blockSource.getCheckpointsData({ epoch }))
+    .filter(c => c.checkpointNumber < checkpointNumber)
+    .sort((a, b) => a.checkpointNumber - b.checkpointNumber);
+
+  if (!pipeliningEnabled || checkpointNumber === 0) {
+    return checkpointed.map(c => c.checkpointOutHash);
+  }
+
+  const parentCheckpoint = CheckpointNumber(checkpointNumber - 1);
+  if (checkpointed.at(-1)?.checkpointNumber === parentCheckpoint) {
+    return checkpointed.map(c => c.checkpointOutHash);
+  }
+
+  const proposedParent =
+    input.proposedCheckpointData ?? (await blockSource.getProposedCheckpointData({ number: parentCheckpoint }));
+  if (!proposedParent || proposedParent.checkpointNumber !== parentCheckpoint) {
+    return checkpointed.map(c => c.checkpointOutHash);
+  }
+  if (getEpochAtSlot(proposedParent.header.slotNumber, l1Constants) !== epoch) {
+    return checkpointed.map(c => c.checkpointOutHash);
+  }
+
+  log?.debug(`Splicing pipelined parent cp ${parentCheckpoint} outHash for cp ${checkpointNumber}`);
+  return [...checkpointed.map(c => c.checkpointOutHash), proposedParent.checkpointOutHash];
+}

yarn-project/stdlib/src/rollup/block_rollup_public_inputs.ts

@@ -119,6 +119,19 @@ export class BlockRollupPublicInputs {
     return this.toBuffer();
   }
 
+  toInspect() {
+    return {
+      previousArchiveRoot: this.previousArchive.root.toString(),
+      newArchiveRoot: this.newArchive.root.toString(),
+      blockHeadersHash: this.blockHeadersHash.toString(),
+      inHash: this.inHash.toString(),
+      outHash: this.outHash.toString(),
+      timestamp: this.timestamp.toString(),
+      accumulatedFees: this.accumulatedFees.toString(),
+      accumulatedManaUsed: this.accumulatedManaUsed.toString(),
+    };
+  }
+
   static get schema() {
     return bufferSchemaFor(BlockRollupPublicInputs);
   }

yarn-project/stdlib/src/rollup/checkpoint_rollup_public_inputs.ts

@@ -101,6 +101,16 @@ export class CheckpointRollupPublicInputs {
     return this.toBuffer();
   }
 
+  toInspect() {
+    return {
+      checkpointHeaderHash: this.checkpointHeaderHashes[0].toString(),
+      previousArchiveRoot: this.previousArchive.root.toString(),
+      newArchiveRoot: this.newArchive.root.toString(),
+      previousOutHashRoot: this.previousOutHash.root.toString(),
+      newOutHashRoot: this.newOutHash.root.toString(),
+    };
+  }
+
   /** Creates an instance from a hex string. */
   static get schema() {
     return bufferSchemaFor(CheckpointRollupPublicInputs);