fix(prover-node): don't treat prune-induced faults as genuine failures

Two prune-vs-failure gaps remained after decoupling checkpoint failure from
epoch failure:

- Session classification only checked isFailed(), missing a control-plane
  cancel that reaches start()'s catch before the reconcile marks the session
  'cancelled'. Such a cancelled-but-not-failed prover was misclassified as the
  session's own 'failed', triggering a spurious full-snapshot upload for a
  prune. Treat a cancelled prover as prune-ambiguous too.

- A data-plane fork fault reaches the checkpoint-level onFailed upload
  indistinguishable from a genuine sub-tree failure (no cancel has landed
  yet). Gate the upload on the archiver: a pruned checkpoint's last block is no
  longer canonical there, so skip the expensive world-state + archiver snapshot
  when it has been pruned out.

Author: PhilWindle

yarn-project/prover-node/src/job/epoch-session.test.ts

@@ -321,6 +321,29 @@ describe('EpochSession', () => {
       await expect(startResult).resolves.toBe('stopped');
     });
 
+    it('ends the session in "stopped" (not "failed") when a prover was cancelled by a prune (isCancelled, not isFailed)', async () => {
+      // A control-plane prune cancels the prover: its blockProofs reject with "cancelled" and it reports
+      // isCancelled()===true / isFailed()===false. If that rejection reaches start()'s catch before the
+      // reconcile marks the session 'cancelled', it must NOT be classified 'failed' (which would trigger a
+      // spurious full-snapshot upload) — a cancelled prover is prune-ambiguous, so classify 'stopped'.
+      const cancelledProver = makeStubProver(cp, {
+        blockProofsError: new Error('Checkpoint cancelled'),
+        isFailed: false,
+        isCancelled: true,
+      });
+      const session = new EpochSession(
+        makeSpec(),
+        [cancelledProver],
+        makeDeps({
+          hooks: { topTreeProveOverride: () => cancelledProver.whenBlockProofsReady().then(() => synthProof) },
+        }),
+      );
+      const state = await session.start();
+      expect(state).toBe('stopped');
+      expect(session.hasFailed()).toBe(false);
+      expect(publishingService.submit).not.toHaveBeenCalled();
+    });
+
     it('a top-tree prove that rejects with healthy provers ends the session in "failed" without submitting', async () => {
       // The provers are healthy but the top-tree (root) prove itself rejects — the session's own work
       // failed, and since no prover failed this is definitively not a prune, so it ends in 'failed'.
@@ -447,7 +470,10 @@ class TestEpochSession extends EpochSession {
  * path where CheckpointProver.executeCheckpoint catches an internal failure and rejects
  * its blockProofs promise.
  */
-function makeStubProver(checkpoint: Checkpoint, opts: { blockProofsError?: Error } = {}): CheckpointProver {
+function makeStubProver(
+  checkpoint: Checkpoint,
+  opts: { blockProofsError?: Error; isFailed?: boolean; isCancelled?: boolean } = {},
+): CheckpointProver {
   const id = CheckpointProver.idFor(checkpoint);
   // By default whenBlockProofsReady never resolves in these tests; the prove override
   // bypasses any path that would actually await it.
@@ -468,9 +494,10 @@ function makeStubProver(checkpoint: Checkpoint, opts: { blockProofsError?: Error
     previousArchiveSiblingPath: makeTuple(ARCHIVE_HEIGHT, () => Fr.ZERO),
     txs: new Map(),
     whenBlockProofsReady: () => blockProofs,
-    isCancelled: () => false,
-    // A prover configured with a blockProofsError is one whose block proofs rejected — i.e. failed.
-    isFailed: () => opts.blockProofsError !== undefined,
+    isCancelled: () => opts.isCancelled ?? false,
+    // A prover configured with a blockProofsError is one whose block proofs rejected — i.e. failed,
+    // unless the caller decouples the two (e.g. to model a cancelled-but-not-failed prune).
+    isFailed: () => opts.isFailed ?? opts.blockProofsError !== undefined,
     cancel: () => {},
     whenDone: () => Promise.resolve(),
     getAbortSignal: () => new AbortController().signal,

yarn-project/prover-node/src/job/epoch-session.ts

@@ -229,15 +229,16 @@ export class EpochSession implements Traceable {
         ...this.spec,
       });
       // Distinguish the two ways an attempt can fault:
-      //  - a checkpoint prover in the set has failed (a sub-tree fault or prune-induced fork fault):
-      //    end in the non-declaring terminal 'stopped'. This is not the session's own failure and may
-      //    be a prune, so the reconciler does not upload it; a re-add installs a fresh prover.
-      //  - no prover failed, yet top-tree proving or L1 submission failed: this is the session's own,
-      //    genuine failure — and, because every prover succeeded, it is definitively NOT a prune. End
-      //    in terminal 'failed' so the reconciler retains it (no pointless re-prove) and uploads a
-      //    race-free post-mortem.
+      //  - a checkpoint prover in the set has failed OR was cancelled (a sub-tree fault, a prune-induced
+      //    fork fault, or a control-plane cancel that reached this catch before the reconcile marked the
+      //    session 'cancelled'): end in the non-declaring terminal 'stopped'. This is not the session's own
+      //    failure and may be a prune, so the reconciler does not upload it; a re-add installs a fresh prover.
+      //  - no prover failed or was cancelled, yet top-tree proving or L1 submission failed: this is the
+      //    session's own, genuine failure — and, because every prover is healthy and un-cancelled, it is
+      //    definitively NOT a prune. End in terminal 'failed' so the reconciler retains it (no pointless
+      //    re-prove) and uploads a race-free post-mortem.
       if (!this.isTerminal()) {
-        this.state = this.checkpoints.some(c => c.isFailed()) ? 'stopped' : 'failed';
+        this.state = this.checkpoints.some(c => c.isFailed() || c.isCancelled()) ? 'stopped' : 'failed';
       }
     } finally {
       clearTimeout(this.deadlineTimeoutHandler);

yarn-project/prover-node/src/prover-node.test.ts

@@ -374,6 +374,46 @@ describe('ProverNode', () => {
     expect(uploadSpy).toHaveBeenCalledWith(prover);
   });
 
+  describe('isCheckpointCanonical', () => {
+    it('is true when the archiver holds a block at the checkpoint tip with a matching archive root', async () => {
+      const archiveRoot = Fr.random();
+      const checkpoint = makeCheckpoint(3, 3, 3, archiveRoot);
+      l2BlockSource.getBlock.mockResolvedValue({ archive: { root: archiveRoot } } as unknown as L2Block);
+
+      await expect(proverNode.callIsCheckpointCanonical(checkpoint)).resolves.toBe(true);
+    });
+
+    it('is false when the checkpoint tip block was pruned out (archiver returns nothing)', async () => {
+      const checkpoint = makeCheckpoint(3, 3, 3);
+      l2BlockSource.getBlock.mockResolvedValue(undefined);
+
+      await expect(proverNode.callIsCheckpointCanonical(checkpoint)).resolves.toBe(false);
+    });
+
+    it('is false when the tip block was replaced by a reorg (archive root differs)', async () => {
+      const checkpoint = makeCheckpoint(3, 3, 3, Fr.random());
+      l2BlockSource.getBlock.mockResolvedValue({ archive: { root: Fr.random() } } as unknown as L2Block);
+
+      await expect(proverNode.callIsCheckpointCanonical(checkpoint)).resolves.toBe(false);
+    });
+  });
+
+  it('tryUploadCheckpointFailure skips the upload for a checkpoint pruned out of the canonical chain', async () => {
+    // A prune-induced fork fault reaches onFailed just like a genuine sub-tree failure, but the pruned
+    // checkpoint no longer exists on-chain — the expensive full snapshot must not be produced for it.
+    (proverNode as any).config.proverNodeFailedEpochStore = 'file:///tmp/does-not-matter';
+    const checkpoint = makeCheckpoint(3, 3, 3);
+    l2BlockSource.getBlock.mockResolvedValue(undefined);
+    const failedProver = { id: 'prover-3', checkpoint } as unknown as Parameters<
+      typeof proverNode.tryUploadCheckpointFailure
+    >[0];
+
+    await expect(proverNode.tryUploadCheckpointFailure(failedProver)).resolves.toBeUndefined();
+    expect(l2BlockSource.getBlock).toHaveBeenCalledWith({ number: checkpoint.blocks.at(-1)!.number });
+    // World-state snapshotting is the first thing the real upload path touches; it must never be reached.
+    expect(worldState.getSnapshot).not.toHaveBeenCalled();
+  });
+
   // ---------------- forwarders ----------------
 
   it('startProof forwards to the session manager and returns the job id', async () => {
@@ -772,6 +812,10 @@ class TestProverNode extends ProverNode {
     return this.isProvenBlockLastOfItsEpoch(provenBlock, provenEpoch, l1Constants as any);
   }
 
+  public callIsCheckpointCanonical(checkpoint: Checkpoint) {
+    return this.isCheckpointCanonical(checkpoint);
+  }
+
   public getLastExpiredEpoch(): EpochNumber | undefined {
     return this.lastExpiredEpoch;
   }

yarn-project/prover-node/src/prover-node.ts

@@ -638,13 +638,25 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
    * Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
    * proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
    * block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
-   * failed-epoch store is configured. Swallows its own errors so a fire-and-forget caller can't leak.
+   * failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
+   * to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
    */
   public async tryUploadCheckpointFailure(prover: CheckpointProver): Promise<string | undefined> {
     if (!this.config.proverNodeFailedEpochStore) {
       return undefined;
     }
     try {
+      // A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
+      // prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
+      // committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
+      // Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
+      // the snapshot (full world-state + archiver) is expensive to produce and store.
+      if (!(await this.isCheckpointCanonical(prover.checkpoint))) {
+        this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
+          checkpointNumber: prover.checkpoint.number,
+        });
+        return undefined;
+      }
       const data = SessionManager.buildProvingData([prover]);
       return await uploadEpochProofFailure(
         this.config.proverNodeFailedEpochStore,
@@ -664,6 +676,20 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
 
   // ---------------- helpers ----------------
 
+  /**
+   * True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
+   * block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
+   * so this returns false. Protected for direct unit-test access.
+   */
+  protected async isCheckpointCanonical(checkpoint: Checkpoint): Promise<boolean> {
+    const lastBlock = checkpoint.blocks.at(-1);
+    if (!lastBlock) {
+      return false;
+    }
+    const onChain = await this.l2BlockSource.getBlock({ number: lastBlock.number });
+    return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
+  }
+
   @memoize
   private getL1Constants(): Promise<L1RollupConstants> {
     return this.l2BlockSource.getL1Constants();