fix(pxe): retry settled-read witness fetches and self-diagnose misses in kernel reset

A settled note-hash/nullifier read request whose membership witness the node cannot serve at the tx
anchor block currently fails the whole proving job on the first miss. The anchor block is data the node
has already synced, so a miss is either a transient node-side visibility race or a genuinely absent
leaf. Retry the hint-building batch a bounded number of times for the former, and enrich the final
error for the latter with the anchor block number/hash plus a latest-view insertion probe that tells a
stale anchor apart from node inconsistency and from a bogus read request.

Author: vezenovm

yarn-project/pxe/src/private_kernel/hints/private_kernel_reset_private_inputs_builder.test.ts

@@ -8,10 +8,12 @@ import {
   NOTE_HASH_TREE_HEIGHT,
   VK_TREE_HEIGHT,
 } from '@aztec/constants';
+import { BlockNumber } from '@aztec/foundation/branded-types';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import type { GrumpkinScalar } from '@aztec/foundation/curves/grumpkin';
 import { MembershipWitness } from '@aztec/foundation/trees';
 import { AztecAddress } from '@aztec/stdlib/aztec-address';
+import { BlockHash } from '@aztec/stdlib/block';
 import {
   type DimensionName,
   type PrivateKernelResetDimensions,
@@ -146,15 +148,37 @@ describe('PrivateKernelResetPrivateInputsBuilder', () => {
         expect(oracle.getNoteHashMembershipWitness).toHaveBeenCalledTimes(numSettled);
       });
 
+      it('retries and recovers when a settled read witness is transiently missing', async () => {
+        kernel.addSettledNoteHashReadRequest();
+
+        oracle.getNoteHashMembershipWitness.mockResolvedValueOnce(undefined);
+
+        const builder = makeResetBuilder();
+        expect(builder.needsReset()).toBe(true);
+
+        const result = await builder.build(oracle);
+        expectDimensions(builder, result.dimensions, { NOTE_HASH_SETTLED_READ: 1 });
+        expect(oracle.getNoteHashMembershipWitness).toHaveBeenCalledTimes(2);
+      });
+
       it('throws when settled read request has no matching membership witness', async () => {
         kernel.addSettledNoteHashReadRequest();
 
         oracle.getNoteHashMembershipWitness.mockResolvedValue(undefined);
+        oracle.getAnchorBlockNumber.mockReturnValue(BlockNumber(42));
+        oracle.getAnchorBlockHash.mockResolvedValue(BlockHash.random());
+        oracle.findLeafInsertionBlock.mockResolvedValue(BlockNumber(45));
 
         const builder = makeResetBuilder();
         expect(builder.needsReset()).toBe(true);
 
-        await expect(builder.build(oracle)).rejects.toThrow('Read request is reading an unknown note hash.');
+        const err = await builder.build(oracle).then(
+          () => undefined,
+          (e: Error) => e,
+        );
+        expect(err?.message).toContain('Read request is reading an unknown note hash');
+        expect(err?.message).toContain('Anchor block 42');
+        expect(err?.message).toContain('latest view: leaf inserted in block 45');
       });
 
       it('throws when pending read request has no matching note hash', () => {
@@ -212,15 +236,37 @@ describe('PrivateKernelResetPrivateInputsBuilder', () => {
         expect(oracle.getNullifierMembershipWitness).toHaveBeenCalledTimes(numSettled);
       });
 
+      it('retries and recovers when a settled read witness is transiently missing', async () => {
+        kernel.addSettledNullifierReadRequest();
+
+        oracle.getNullifierMembershipWitness.mockResolvedValueOnce(undefined);
+
+        const builder = makeResetBuilder();
+        expect(builder.needsReset()).toBe(true);
+
+        const result = await builder.build(oracle);
+        expectDimensions(builder, result.dimensions, { NULLIFIER_SETTLED_READ: 1 });
+        expect(oracle.getNullifierMembershipWitness).toHaveBeenCalledTimes(2);
+      });
+
       it('throws when settled read request has no matching membership witness', async () => {
         kernel.addSettledNullifierReadRequest();
 
         oracle.getNullifierMembershipWitness.mockResolvedValue(undefined);
+        oracle.getAnchorBlockNumber.mockReturnValue(BlockNumber(42));
+        oracle.getAnchorBlockHash.mockResolvedValue(BlockHash.random());
+        oracle.findLeafInsertionBlock.mockResolvedValue(undefined);
 
         const builder = makeResetBuilder();
         expect(builder.needsReset()).toBe(true);
 
-        await expect(builder.build(oracle)).rejects.toThrow('Cannot find the leaf for nullifier');
+        const err = await builder.build(oracle).then(
+          () => undefined,
+          (e: Error) => e,
+        );
+        expect(err?.message).toContain('Cannot find the leaf for nullifier');
+        expect(err?.message).toContain('Anchor block 42');
+        expect(err?.message).toContain('latest view: leaf not found');
       });
 
       it('throws when pending read request has no matching nullifier', () => {

yarn-project/pxe/src/private_kernel/hints/private_kernel_reset_private_inputs_builder.ts

@@ -10,7 +10,9 @@ import {
 import { makeTuple } from '@aztec/foundation/array';
 import { padArrayEnd } from '@aztec/foundation/collection';
 import type { Fr } from '@aztec/foundation/curves/bn254';
+import { createLogger } from '@aztec/foundation/log';
 import { assertLength } from '@aztec/foundation/serialize';
+import { sleep } from '@aztec/foundation/sleep';
 import { MembershipWitness } from '@aztec/foundation/trees';
 import { privateKernelResetDimensionsConfig } from '@aztec/noir-protocol-circuits-types/client';
 import {
@@ -37,16 +39,44 @@ import {
   getNullifierReadRequestResetActions,
   privateKernelResetDimensionNames,
 } from '@aztec/stdlib/kernel';
+import { MerkleTreeId } from '@aztec/stdlib/trees';
 import { type PrivateCallExecutionResult, collectNested } from '@aztec/stdlib/tx';
 import { VkData } from '@aztec/stdlib/vks';
 
 import type { PrivateKernelOracle } from '../private_kernel_oracle.js';
 
+// Settled-read membership witnesses are anchored at the tx's anchor block, whose data the node has already
+// synced, so a missing witness is either a transient node-side visibility race (a retry succeeds) or a
+// genuinely absent leaf (retries cannot mask it: the final error carries a latest-view probe to tell the
+// two apart). 3 total attempts, 300ms apart.
+const SETTLED_WITNESS_ATTEMPTS = 3;
+const SETTLED_WITNESS_RETRY_INTERVAL_MS = 300;
+
+const log = createLogger('pxe:private-kernel-reset');
+
+/** Thrown by the settled-read witness resolvers when the node cannot provide a witness at the anchor block. */
+class MissingSettledWitnessError extends Error {
+  constructor(
+    message: string,
+    /** The tree the leaf was expected in. */
+    public readonly treeId: MerkleTreeId,
+    /** The leaf (siloed nullifier or siloed note hash) whose witness is missing. */
+    public readonly leaf: Fr,
+  ) {
+    super(message);
+    this.name = 'MissingSettledWitnessError';
+  }
+}
+
 function getNullifierMembershipWitnessResolver(oracle: PrivateKernelOracle) {
   return async (nullifier: Fr) => {
     const res = await oracle.getNullifierMembershipWitness(nullifier);
     if (!res) {
-      throw new Error(`Cannot find the leaf for nullifier ${nullifier}.`);
+      throw new MissingSettledWitnessError(
+        `Cannot find the leaf for nullifier ${nullifier}.`,
+        MerkleTreeId.NULLIFIER_TREE,
+        nullifier,
+      );
     }
 
     const { index, siblingPath, leafPreimage } = res;
@@ -57,6 +87,75 @@ function getNullifierMembershipWitnessResolver(oracle: PrivateKernelOracle) {
   };
 }
 
+function getNoteHashMembershipWitnessResolver(oracle: PrivateKernelOracle) {
+  return async (noteHash: Fr) => {
+    const witness = await oracle.getNoteHashMembershipWitness(noteHash);
+    if (!witness) {
+      throw new MissingSettledWitnessError(
+        `Read request is reading an unknown note hash ${noteHash}.`,
+        MerkleTreeId.NOTE_HASH_TREE,
+        noteHash,
+      );
+    }
+    return witness;
+  };
+}
+
+/**
+ * Appends the anchor block and a latest-view insertion probe to a missing-witness error, so a failure
+ * self-reports whether the anchor predates the leaf (stale anchor), the node lost visibility of a leaf it
+ * should serve (node inconsistency), or the leaf does not exist at all (bad read request).
+ */
+async function enrichMissingWitnessError(err: MissingSettledWitnessError, oracle: PrivateKernelOracle) {
+  const anchorBlockNumber = oracle.getAnchorBlockNumber();
+  let anchorBlockHash = 'unavailable';
+  let latestView = 'latest-view probe failed';
+  try {
+    anchorBlockHash = (await oracle.getAnchorBlockHash()).toString();
+  } catch {
+    // Keep the fallback description.
+  }
+  try {
+    const insertionBlock = await oracle.findLeafInsertionBlock(err.treeId, err.leaf);
+    latestView = insertionBlock === undefined ? 'leaf not found' : `leaf inserted in block ${insertionBlock}`;
+  } catch {
+    // Keep the fallback description.
+  }
+  return new Error(
+    `${err.message} Anchor block ${anchorBlockNumber} (hash ${anchorBlockHash}); latest view: ${latestView}.`,
+  );
+}
+
+/**
+ * Runs a settled-read hint-building batch, retrying the whole batch when a witness the anchor block must
+ * contain is transiently missing. Any error other than {@link MissingSettledWitnessError} rethrows untouched.
+ */
+async function buildSettledReadHintsWithRetry<T>(
+  oracle: PrivateKernelOracle,
+  description: string,
+  buildHints: () => Promise<T>,
+): Promise<T> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await buildHints();
+    } catch (err) {
+      if (!(err instanceof MissingSettledWitnessError)) {
+        throw err;
+      }
+      if (attempt >= SETTLED_WITNESS_ATTEMPTS) {
+        throw await enrichMissingWitnessError(err, oracle);
+      }
+      log.warn(`Missing settled ${description} witness at anchor block, retrying`, {
+        attempt,
+        maxAttempts: SETTLED_WITNESS_ATTEMPTS,
+        anchorBlockNumber: oracle.getAnchorBlockNumber(),
+        leaf: err.leaf.toString(),
+      });
+      await sleep(SETTLED_WITNESS_RETRY_INTERVAL_MS);
+    }
+  }
+}
+
 async function getMasterSecretKeysAndKeyTypeDomainSeparators(
   keyValidationRequests: ClaimedLengthArray<
     ScopedKeyValidationRequestAndSeparator,
@@ -160,22 +259,26 @@ export class PrivateKernelResetPrivateInputsBuilder {
     const [previousVkMembershipWitness, noteHashReadRequestHints, nullifierReadRequestHints, keyValidationHints] =
       await Promise.all([
         oracle.getVkMembershipWitness(this.previousKernelOutput.verificationKey.keyAsFields),
-        buildNoteHashReadRequestHintsFromResetActions<
-          typeof MAX_NOTE_HASH_READ_REQUESTS_PER_TX,
-          typeof MAX_NOTE_HASH_READ_REQUESTS_PER_TX
-        >(
-          oracle,
-          this.previousKernel.validationRequests.noteHashReadRequests,
-          this.previousKernel.end.noteHashes,
-          this.noteHashResetActions,
+        buildSettledReadHintsWithRetry(oracle, 'note hash', () =>
+          buildNoteHashReadRequestHintsFromResetActions<
+            typeof MAX_NOTE_HASH_READ_REQUESTS_PER_TX,
+            typeof MAX_NOTE_HASH_READ_REQUESTS_PER_TX
+          >(
+            { getNoteHashMembershipWitness: getNoteHashMembershipWitnessResolver(oracle) },
+            this.previousKernel.validationRequests.noteHashReadRequests,
+            this.previousKernel.end.noteHashes,
+            this.noteHashResetActions,
+          ),
         ),
-        buildNullifierReadRequestHintsFromResetActions<
-          typeof MAX_NULLIFIER_READ_REQUESTS_PER_TX,
-          typeof MAX_NULLIFIER_READ_REQUESTS_PER_TX
-        >(
-          { getNullifierMembershipWitness: getNullifierMembershipWitnessResolver(oracle) },
-          this.previousKernel.validationRequests.nullifierReadRequests,
-          this.nullifierResetActions,
+        buildSettledReadHintsWithRetry(oracle, 'nullifier', () =>
+          buildNullifierReadRequestHintsFromResetActions<
+            typeof MAX_NULLIFIER_READ_REQUESTS_PER_TX,
+            typeof MAX_NULLIFIER_READ_REQUESTS_PER_TX
+          >(
+            { getNullifierMembershipWitness: getNullifierMembershipWitnessResolver(oracle) },
+            this.previousKernel.validationRequests.nullifierReadRequests,
+            this.nullifierResetActions,
+          ),
         ),
         getMasterSecretKeysAndKeyTypeDomainSeparators(
           this.previousKernel.validationRequests.scopedKeyValidationRequestsAndSeparators,

yarn-project/pxe/src/private_kernel/private_kernel_oracle.ts

@@ -5,6 +5,7 @@ import {
   UPDATES_VALUE_SIZE,
   VK_TREE_HEIGHT,
 } from '@aztec/constants';
+import type { BlockNumber } from '@aztec/foundation/branded-types';
 import type { Fr } from '@aztec/foundation/curves/bn254';
 import type { GrumpkinScalar } from '@aztec/foundation/curves/grumpkin';
 import { MembershipWitness } from '@aztec/foundation/trees';
@@ -13,12 +14,13 @@ import { getVKIndex, getVKSiblingPath } from '@aztec/noir-protocol-circuits-type
 import { ProtocolContractAddress } from '@aztec/protocol-contracts';
 import type { FunctionSelector } from '@aztec/stdlib/abi';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { BlockHash } from '@aztec/stdlib/block';
 import { type ContractInstanceWithAddress, computeSaltedInitializationHash } from '@aztec/stdlib/contract';
 import { DelayedPublicMutableValues, DelayedPublicMutableValuesWithHash } from '@aztec/stdlib/delayed-public-mutable';
 import { computePublicDataTreeLeafSlot } from '@aztec/stdlib/hash';
 import type { AztecNode } from '@aztec/stdlib/interfaces/client';
 import { UpdatedClassIdHints } from '@aztec/stdlib/kernel';
-import type { NullifierMembershipWitness } from '@aztec/stdlib/trees';
+import type { MerkleTreeId, NullifierMembershipWitness } from '@aztec/stdlib/trees';
 import type { BlockHeader } from '@aztec/stdlib/tx';
 import type { VerificationKeyAsFields } from '@aztec/stdlib/vks';
 
@@ -112,6 +114,26 @@ export class PrivateKernelOracle {
     return this.node.getNullifierMembershipWitness(await this.blockHeader.hash(), nullifier);
   }
 
+  /** The block number of the anchor block all membership queries of this oracle are anchored at. */
+  getAnchorBlockNumber(): BlockNumber {
+    return this.blockHeader.getBlockNumber();
+  }
+
+  /** The hash of the anchor block, i.e. the reference block used for the node witness queries above. */
+  getAnchorBlockHash(): Promise<BlockHash> {
+    return this.blockHeader.hash();
+  }
+
+  /**
+   * Diagnostic lookup: the block that inserted the given leaf as of the node's latest view, if any.
+   * Used to enrich missing-witness errors so a failure self-reports whether the leaf is missing entirely
+   * or landed in a block past the anchor.
+   */
+  async findLeafInsertionBlock(treeId: MerkleTreeId, leaf: Fr): Promise<BlockNumber | undefined> {
+    const [result] = await this.node.findLeavesIndexes('latest', treeId, [leaf]);
+    return result?.l2BlockNumber;
+  }
+
   /** Returns the root of our note hash merkle tree. */
   getNoteHashTreeRoot(): Fr {
     return this.blockHeader.state.partial.noteHashTree.root;