feat: unknown payload sync for incoming blocks (supersedes #9102)

When a gossip block claims its parent was FULL but the parent's envelope
hasn't been seen, the block is queued and the parent envelope is fetched
via sendExecutionPayloadEnvelopesByRoot.

Changes:
- Add PARENT_PAYLOAD_UNKNOWN block error code and gossip validation
- Add PendingPayloadEnvelope type for tracking missing envelopes
- BlockInputSync: pendingPayloads map, triggerPayloadSearch,
  fetchPayloadEnvelope, onUnknownPayloadEnvelope handler
- Subscribe to unknownEnvelopeBlockRoot and executionPayloadAvailable events
- Add metrics for payload fetch tracking
- Handle PARENT_PAYLOAD_UNKNOWN in processBlock error recovery

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: ensi321

packages/beacon-node/src/chain/errors/blockError.ts

@@ -120,7 +120,7 @@ export type BlockErrorType =
   | {code: BlockErrorCode.TOO_MANY_KZG_COMMITMENTS; blobKzgCommitmentsLen: number; commitmentLimit: number}
   | {code: BlockErrorCode.BID_PARENT_ROOT_MISMATCH; bidParentRoot: RootHex; blockParentRoot: RootHex}
   | {code: BlockErrorCode.PARENT_EXECUTION_INVALID; parentRoot: RootHex}
-  | {code: BlockErrorCode.PARENT_PAYLOAD_UNKNOWN; parentBlockHash: RootHex};
+  | {code: BlockErrorCode.PARENT_PAYLOAD_UNKNOWN; parentRoot: RootHex; parentBlockHash: RootHex};
 
 export class BlockGossipError extends GossipActionError<BlockErrorType> {}
 

packages/beacon-node/src/chain/validation/block.ts

@@ -103,6 +103,7 @@ export async function validateGossipBlock(
     if (chain.forkChoice.getBlockHexAndBlockHash(parentRoot, parentBlockHashHex) === null) {
       throw new BlockGossipError(GossipAction.IGNORE, {
         code: BlockErrorCode.PARENT_PAYLOAD_UNKNOWN,
+        parentRoot,
         parentBlockHash: parentBlockHashHex,
       });
     }

packages/beacon-node/src/metrics/metrics/lodestar.ts

@@ -680,6 +680,23 @@ export function createLodestarMetrics(
           labelNames: ["code", "client"],
         }),
       },
+      pendingPayloads: register.gauge({
+        name: "lodestar_sync_unknown_block_pending_payloads_size",
+        help: "Current size of BlockInputSync pending payloads cache",
+      }),
+      payloadRequests: register.gauge<{source: BlockInputSource}>({
+        name: "lodestar_sync_unknown_block_payload_requests_total",
+        help: "Total number of payload envelope fetch requests triggered",
+        labelNames: ["source"],
+      }),
+      payloadFetchSuccess: register.gauge({
+        name: "lodestar_sync_unknown_block_payload_fetch_success_total",
+        help: "Total number of successful payload envelope fetches",
+      }),
+      payloadFetchError: register.gauge({
+        name: "lodestar_sync_unknown_block_payload_fetch_error_total",
+        help: "Total number of errored payload envelope fetches",
+      }),
       peerBalancer: {
         peersMetaCount: register.gauge({
           name: "lodestar_sync_unknown_block_peer_balancer_peers_meta_count",

packages/beacon-node/src/network/processor/gossipHandlers.ts

@@ -198,6 +198,23 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     } catch (e) {
       if (e instanceof BlockGossipError) {
         logger.debug("Gossip block has error", {slot, root: blockShortHex, code: e.type.code});
+        if (e.type.code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN && blockInput) {
+          logger.debug("Gossip block has parent payload unknown", {slot, root: blockShortHex, code: e.type.code});
+          // Track the child block for processing after parent envelope arrives
+          chain.emitter.emit(ChainEvent.blockUnknownParent, {
+            blockInput,
+            peer: peerIdStr,
+            source: BlockInputSource.gossip,
+          });
+          // Trigger parent envelope fetch
+          chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {
+            rootHex: e.type.parentRoot,
+            peer: peerIdStr,
+            source: BlockInputSource.gossip,
+          });
+          throw e;
+        }
+
         if (e.type.code === BlockErrorCode.PARENT_UNKNOWN && blockInput) {
           chain.emitter.emit(ChainEvent.blockUnknownParent, {
             blockInput,

packages/beacon-node/src/sync/types.ts

@@ -1,5 +1,6 @@
 import {RootHex, Slot} from "@lodestar/types";
 import {IBlockInput} from "../chain/blocks/blockInput/index.js";
+import {PeerIdStr} from "../util/peerId.js";
 
 export enum PendingBlockType {
   /**
@@ -55,3 +56,12 @@ export function getBlockInputSyncCacheItemRootHex(block: BlockInputSyncCacheItem
 export function getBlockInputSyncCacheItemSlot(block: BlockInputSyncCacheItem): Slot | string {
   return isPendingBlockInput(block) ? block.blockInput.slot : "unknown";
 }
+
+export type PendingPayloadEnvelope = {
+  status: "pending" | "fetching";
+  blockRootHex: RootHex;
+  slot: Slot;
+  attempts: number;
+  peerIdStrings: Set<PeerIdStr>;
+  timeAddedSec: number;
+};

packages/beacon-node/src/sync/unknownBlock.ts

@@ -1,11 +1,13 @@
 import {ChainForkConfig} from "@lodestar/config";
+import {PayloadStatus} from "@lodestar/fork-choice";
 import {ForkSeq} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {RootHex} from "@lodestar/types";
-import {Logger, prettyPrintIndices, pruneSetToMax, sleep} from "@lodestar/utils";
+import {Logger, fromHex, prettyPrintIndices, pruneSetToMax, sleep} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
+import {PayloadEnvelopeInputSource} from "../chain/blocks/payloadEnvelopeInput/index.js";
 import {BlockError, BlockErrorCode} from "../chain/errors/index.js";
 import {ChainEvent, ChainEventData, IBeaconChain} from "../chain/index.js";
 import {Metrics} from "../metrics/index.js";
@@ -22,6 +24,7 @@ import {
   PendingBlockInput,
   PendingBlockInputStatus,
   PendingBlockType,
+  PendingPayloadEnvelope,
   getBlockInputSyncCacheItemRootHex,
   getBlockInputSyncCacheItemSlot,
   isPendingBlockInput,
@@ -32,6 +35,8 @@ import {getAllDescendantBlocks, getDescendantBlocks, getUnknownAndAncestorBlocks
 const MAX_ATTEMPTS_PER_BLOCK = 5;
 const MAX_KNOWN_BAD_BLOCKS = 500;
 const MAX_PENDING_BLOCKS = 100;
+const MAX_PENDING_PAYLOADS = 100;
+const MAX_ATTEMPTS_PER_PAYLOAD = 5;
 
 enum FetchResult {
   SuccessResolved = "success_resolved",
@@ -78,6 +83,7 @@ export class BlockInputSync {
    * block RootHex -> PendingBlock. To avoid finding same root at the same time
    */
   private readonly pendingBlocks = new Map<RootHex, BlockInputSyncCacheItem>();
+  private readonly pendingPayloads = new Map<RootHex, PendingPayloadEnvelope>();
   private readonly knownBadBlocks = new Set<RootHex>();
   private readonly maxPendingBlocks;
   private subscribedToNetworkEvents = false;
@@ -101,6 +107,9 @@ export class BlockInputSync {
       metrics.blockInputSync.knownBadBlocks.addCollect(() =>
         metrics.blockInputSync.knownBadBlocks.set(this.knownBadBlocks.size)
       );
+      metrics.blockInputSync.pendingPayloads?.addCollect(() =>
+        metrics.blockInputSync.pendingPayloads?.set(this.pendingPayloads.size)
+      );
     }
   }
 
@@ -116,6 +125,7 @@ export class BlockInputSync {
       this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
       this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
       this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
+      this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownPayloadEnvelope);
       this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
       this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
       this.subscribedToNetworkEvents = true;
@@ -127,6 +137,7 @@ export class BlockInputSync {
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
     this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
+    this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownPayloadEnvelope);
     this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
     this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
     this.subscribedToNetworkEvents = false;
@@ -183,6 +194,21 @@ export class BlockInputSync {
     }
   };
 
+  /**
+   * Process an unknownEnvelopeBlockRoot event - fetch missing payload envelope for a known block.
+   */
+  private onUnknownPayloadEnvelope = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]): void => {
+    try {
+      const {rootHex: blockRootHex, peer} = data;
+      const block = this.chain.forkChoice.getBlockHexDefaultStatus(blockRootHex);
+      if (!block) return;
+      this.addPendingPayload(blockRootHex, block.slot, peer);
+      this.metrics?.blockInputSync.payloadRequests?.inc({source: data.source});
+    } catch (e) {
+      this.logger.debug("Error handling unknownPayloadEnvelope event", {}, e as Error);
+    }
+  };
+
   private addByRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr): void => {
     let pendingBlock = this.pendingBlocks.get(rootHex);
     if (!pendingBlock) {
@@ -248,6 +274,7 @@ export class BlockInputSync {
       const peerSyncMeta = this.network.getConnectedPeerSyncMeta(peerId);
       this.peerBalancer.onPeerConnected(data.peer, peerSyncMeta);
       this.triggerUnknownBlockSearch();
+      this.triggerPayloadSearch();
     } catch (e) {
       this.logger.debug("Error handling peerConnected event", {}, e as Error);
     }
@@ -258,6 +285,105 @@ export class BlockInputSync {
     this.peerBalancer.onPeerDisconnected(peerId);
   };
 
+  addPendingPayload(rootHex: RootHex, slot: number, peer?: PeerIdStr): void {
+    const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+    if (payloadInput?.hasPayloadEnvelope()) return;
+    if (this.chain.forkChoice.getBlockHex(rootHex, PayloadStatus.FULL)) return;
+    if (this.pendingPayloads.size >= MAX_PENDING_PAYLOADS) return;
+
+    let pending = this.pendingPayloads.get(rootHex);
+    if (!pending) {
+      pending = {
+        status: "pending",
+        blockRootHex: rootHex,
+        slot,
+        attempts: 0,
+        peerIdStrings: new Set(),
+        timeAddedSec: Date.now() / 1000,
+      };
+      this.pendingPayloads.set(rootHex, pending);
+    }
+    if (peer) pending.peerIdStrings.add(peer);
+    this.triggerPayloadSearch();
+  }
+
+  private triggerPayloadSearch = (): void => {
+    if (this.pendingPayloads.size === 0) return;
+    if (this.network.getConnectedPeers().length === 0) return;
+
+    const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
+
+    for (const [rootHex, pending] of this.pendingPayloads) {
+      if (pending.slot <= finalizedSlot) {
+        this.pendingPayloads.delete(rootHex);
+        continue;
+      }
+      if (
+        this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.hasPayloadEnvelope() ||
+        this.chain.forkChoice.getBlockHex(rootHex, PayloadStatus.FULL)
+      ) {
+        this.pendingPayloads.delete(rootHex);
+        continue;
+      }
+      if (!this.chain.forkChoice.hasBlockHexUnsafe(rootHex)) continue;
+      if (pending.status !== "pending") continue;
+      if (pending.attempts >= MAX_ATTEMPTS_PER_PAYLOAD) {
+        this.pendingPayloads.delete(rootHex);
+        continue;
+      }
+      this.fetchPayloadEnvelope(pending).catch((e) => {
+        this.logger.debug("Unexpected error - fetchPayloadEnvelope", {root: pending.blockRootHex}, e);
+      });
+    }
+  };
+
+  private async fetchPayloadEnvelope(pending: PendingPayloadEnvelope): Promise<void> {
+    pending.status = "fetching";
+    pending.attempts++;
+    try {
+      const peerMeta = this.peerBalancer.bestPeerForPendingColumns(new Set(), new Set());
+      if (!peerMeta) {
+        pending.status = "pending";
+        return;
+      }
+      const {peerId: peer} = peerMeta;
+
+      const envelopes = await this.network.sendExecutionPayloadEnvelopesByRoot(peer, [fromHex(pending.blockRootHex)]);
+      if (envelopes.length === 0) {
+        pending.status = "pending";
+        return;
+      }
+
+      const envelope = envelopes[0];
+      const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(pending.blockRootHex);
+      if (!payloadInput) {
+        this.logger.debug("PayloadEnvelopeInput missing for fetched envelope", {root: pending.blockRootHex});
+        pending.status = "pending";
+        return;
+      }
+
+      payloadInput.addPayloadEnvelope({
+        envelope,
+        source: PayloadEnvelopeInputSource.byRoot,
+        seenTimestampSec: Date.now() / 1000,
+        peerIdStr: peer,
+      });
+
+      if (payloadInput.isComplete()) {
+        await this.chain.processExecutionPayload(payloadInput);
+      }
+
+      this.pendingPayloads.delete(pending.blockRootHex);
+      this.metrics?.blockInputSync.payloadFetchSuccess?.inc();
+      // Re-trigger block search since pending blocks may now be processable
+      this.triggerUnknownBlockSearch();
+    } catch (e) {
+      this.logger.debug("Error fetching payload envelope", {root: pending.blockRootHex}, e as Error);
+      pending.status = "pending";
+      this.metrics?.blockInputSync.payloadFetchError?.inc();
+    }
+  }
+
   /**
    * Gather tip parent blocks with unknown parent and do a search for all of them
    */
@@ -456,6 +582,12 @@ export class BlockInputSync {
             pendingBlock.status = PendingBlockInputStatus.downloaded;
             break;
 
+          case BlockErrorCode.PARENT_PAYLOAD_UNKNOWN:
+            this.logger.debug("Block parent payload unknown", errorData, res.err);
+            this.addPendingPayload(res.err.type.parentRoot, pendingBlock.blockInput.slot - 1);
+            pendingBlock.status = PendingBlockInputStatus.downloaded;
+            break;
+
           case BlockErrorCode.EXECUTION_ENGINE_ERROR:
             // Removing the block(s) without penalizing the peers, hoping for EL to
             // recover on a latter download + verify attempt
@@ -671,6 +803,7 @@ export class BlockInputSync {
     for (const block of badPendingBlocks) {
       const rootHex = getBlockInputSyncCacheItemRootHex(block);
       this.pendingBlocks.delete(rootHex);
+      this.pendingPayloads.delete(rootHex);
       this.chain.seenBlockInputCache.prune(rootHex);
       this.logger.debug("Removing bad/unknown/incomplete BlockInputSyncCacheItem", {
         slot,