fix(sequencer): only warn about missing proposed checkpoint once overdue (#23807)

## Motivation

The orphan-block guard in `checkSync` (added in #23606) was logging at
`warn` on every non-proposer validator, ~once per second for a full
slot, every slot. Under pipelining a node receives and re-executes a
block proposal for the next checkpoint up to one slot before the
matching checkpoint proposal arrives, so the world-state tip
legitimately sits in an as-yet-unproposed checkpoint for that whole
window. That is the happy path, not the abnormal "proposer published
blocks but never the checkpoint" case the guard is meant to flag.
Observed on `next-net`: 118 warnings in ~59s on a healthy validator for
a single slot.

## Approach

The condition that distinguishes "checkpoint hasn't arrived yet" from
"checkpoint will never arrive" is purely temporal — which is exactly
what the archiver already computes in `pruneOrphanProposedBlocks` to
decide when to prune an orphan block. The guard now reuses that same
deadline: it still refuses to build (`return undefined`) whenever the
orphan-shaped state holds, but only escalates to `warn` once the
enclosing checkpoint is overdue by that deadline; within the normal
pipelining window it logs at `debug`. The warn therefore fires at the
same instant the archiver would prune the orphan.

## Changes

- **sequencer-client**: Add `isProposedCheckpointOverdue`, mirroring the
archiver's orphan-prune deadline (`start of slot after the block's build
slot + grace`, grace derived from `blockDurationMs` as the node wiring
does). Gate the existing guard's log level on it — `warn` when overdue,
`debug` otherwise. Control flow is unchanged.
- **sequencer-client (tests)**: Thread a real `blockSlot` through the
orphan-guard test setup and split the warning test into an overdue case
(expects `warn`) and a within-window case (expects no `warn`).

Author: spalladino

yarn-project/sequencer-client/src/sequencer/sequencer.test.ts

@@ -42,6 +42,7 @@ import {
 } from '@aztec/stdlib/interfaces/server';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 import { CheckpointHeader } from '@aztec/stdlib/rollup';
+import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable';
 import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees';
 import { BlockHeader, GlobalVariables, type Tx } from '@aztec/stdlib/tx';
 import type { FullNodeCheckpointsBuilder, ValidatorClient } from '@aztec/validator-client';
@@ -1335,6 +1336,7 @@ describe('sequencer', () => {
     // checkpointed and proposed-checkpoint tips sit at the given checkpoint numbers.
     const setupSyncedToBlock = (opts: {
       blockNumber: BlockNumber;
+      blockSlot: SlotNumber;
       blockCheckpointNumber: CheckpointNumber;
       checkpointedCheckpointNumber: CheckpointNumber;
       proposedCheckpointTipNumber: CheckpointNumber;
@@ -1376,7 +1378,9 @@ describe('sequencer', () => {
       l1ToL2MessageSource.getL2Tips.mockResolvedValue(tips);
       p2p.getStatus.mockResolvedValue({ syncedToL2Block: { number: opts.blockNumber, hash } } as any);
       l2BlockSource.getBlockData.mockResolvedValue({
-        header: BlockHeader.empty({ globalVariables: GlobalVariables.empty({ blockNumber: opts.blockNumber }) }),
+        header: BlockHeader.empty({
+          globalVariables: GlobalVariables.empty({ blockNumber: opts.blockNumber, slotNumber: opts.blockSlot }),
+        }),
         archive: AppendOnlyTreeSnapshot.empty(),
         blockHash: BlockHash.ZERO,
         checkpointNumber: opts.blockCheckpointNumber,
@@ -1385,16 +1389,23 @@ describe('sequencer', () => {
       l2BlockSource.getProposedCheckpointData.mockResolvedValue(opts.proposedCheckpointData);
     };
 
-    it('returns undefined and warns when the proposed block has no matching proposed checkpoint', async () => {
+    // The orphan block sits at slot 3; with pipelining offset 1 and a grace of MIN_EXECUTION_TIME (no
+    // blockDurationMs configured) its enclosing checkpoint is due at l1GenesisTime + 3 * slotDuration + 2.
+    const orphanCheckpointDueSeconds = () => Number(l1Constants.l1GenesisTime) + 3 * slotDuration + MIN_EXECUTION_TIME;
+
+    it('returns undefined and warns once the missing proposed checkpoint is overdue', async () => {
       // Local tip is a block at checkpoint 3, but the checkpointed and proposed-checkpoint tips are
-      // still at checkpoint 2 and no proposed checkpoint 3 exists: an orphan block-only tip.
+      // still at checkpoint 2 and no proposed checkpoint 3 exists: an orphan block-only tip whose
+      // enclosing checkpoint should have been proposed by now.
       setupSyncedToBlock({
         blockNumber: BlockNumber(3),
+        blockSlot: SlotNumber(3),
         blockCheckpointNumber: CheckpointNumber(3),
         checkpointedCheckpointNumber: CheckpointNumber(2),
         proposedCheckpointTipNumber: CheckpointNumber(2),
         proposedCheckpointData: undefined,
       });
+      dateProvider.setTime((orphanCheckpointDueSeconds() + 1) * 1000);
       const warnSpy = jest.spyOn(sequencer.getLogger(), 'warn');
 
       const result = await sequencer.checkSyncForTest({ ts: 1000n, slot: SlotNumber(2) });
@@ -1411,9 +1422,31 @@ describe('sequencer', () => {
       );
     });
 
+    it('returns undefined without warning while the proposed checkpoint is not yet overdue', async () => {
+      // Same orphan-shaped tip, but we are still within the normal pipelining window: the block proposal
+      // for checkpoint 3 has arrived ahead of its checkpoint proposal, which is not yet due. This is the
+      // happy-path steady state and must not warn.
+      setupSyncedToBlock({
+        blockNumber: BlockNumber(3),
+        blockSlot: SlotNumber(3),
+        blockCheckpointNumber: CheckpointNumber(3),
+        checkpointedCheckpointNumber: CheckpointNumber(2),
+        proposedCheckpointTipNumber: CheckpointNumber(2),
+        proposedCheckpointData: undefined,
+      });
+      dateProvider.setTime((orphanCheckpointDueSeconds() - 1) * 1000);
+      const warnSpy = jest.spyOn(sequencer.getLogger(), 'warn');
+
+      const result = await sequencer.checkSyncForTest({ ts: 1000n, slot: SlotNumber(2) });
+
+      expect(result).toBeUndefined();
+      expect(warnSpy).not.toHaveBeenCalled();
+    });
+
     it('proceeds when a matching proposed checkpoint exists for the block', async () => {
       setupSyncedToBlock({
         blockNumber: BlockNumber(3),
+        blockSlot: SlotNumber(3),
         blockCheckpointNumber: CheckpointNumber(3),
         checkpointedCheckpointNumber: CheckpointNumber(2),
         proposedCheckpointTipNumber: CheckpointNumber(3),

yarn-project/sequencer-client/src/sequencer/sequencer.ts

@@ -20,7 +20,7 @@ import type {
 } from '@aztec/stdlib/block';
 import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
 import type { ChainConfig } from '@aztec/stdlib/config';
-import { getEpochAtSlot, getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
+import { getEpochAtSlot, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 import {
   type ResolvedSequencerConfig,
   type SequencerConfig,
@@ -30,6 +30,7 @@ import {
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
 import { pickFromSchema } from '@aztec/stdlib/schemas';
+import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable';
 import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
 import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, type ValidatorClient } from '@aztec/validator-client';
 
@@ -727,7 +728,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
       (l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber ||
         proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber)
     ) {
-      this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, {
+      const logCtx = {
         blockCheckpointNumber: blockData.checkpointNumber,
         checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
         proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number,
@@ -736,7 +737,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
         blockSlot: blockData.header.getSlot(),
         syncedL2Slot,
         ...args,
-      });
+      };
+
+      // Under pipelining the block proposal for a checkpoint leads its checkpoint proposal by up to one
+      // slot, so a world-state tip sitting in an as-yet-unproposed checkpoint is the expected steady state
+      // until that checkpoint is due. Only treat it as abnormal — and warn — once the checkpoint is overdue
+      // by the same deadline the archiver uses to prune the orphan block (see pruneOrphanProposedBlocks).
+      // Before then this is normal pipelining and we wait it out quietly.
+      if (this.isProposedCheckpointOverdue(blockData.header.getSlot())) {
+        this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, logCtx);
+      } else {
+        this.log.debug(`Waiting for proposed checkpoint to catch up with reexecuted block`, logCtx);
+      }
       return undefined;
     }
 
@@ -787,6 +799,21 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     };
   }
 
+  /**
+   * Whether the enclosing checkpoint of a reexecuted block is overdue: past the deadline by which a
+   * well-behaved proposer should have published it. Mirrors the archiver's orphan-prune deadline (the
+   * start of the slot after the block's build slot, plus a grace period) so the sequencer only warns
+   * about a missing proposed checkpoint once the archiver itself would prune the orphan block. The grace
+   * is derived from the block build duration the same way the archiver defaults it at node wiring.
+   */
+  private isProposedCheckpointOverdue(blockSlot: SlotNumber): boolean {
+    const expectedBySlot = SlotNumber(Number(blockSlot) - PROPOSER_PIPELINING_SLOT_OFFSET + 1);
+    const graceSeconds =
+      this.config.blockDurationMs !== undefined ? Math.ceil(this.config.blockDurationMs / 1000) : MIN_EXECUTION_TIME;
+    const expectedByTime = getTimestampForSlot(expectedBySlot, this.l1Constants) + BigInt(graceSeconds);
+    return BigInt(this.dateProvider.nowInSeconds()) >= expectedByTime;
+  }
+
   /**
    * Checks if we are the proposer for the next slot.
    * @returns True if we can propose, and the proposer address (undefined if anyone can propose)