Merge branch 'spy/deployment-config' of https://github.com/AztecProtocol/aztec-packages into spy/deployment-config

Author: spypsy

yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts

@@ -5,6 +5,7 @@ import { EthAddress } from '@aztec/aztec.js/addresses';
 import { waitForProven } from '@aztec/aztec.js/contracts';
 import { generateClaimSecret } from '@aztec/aztec.js/ethereum';
 import { Fr } from '@aztec/aztec.js/fields';
+import { waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging';
 import { RollupCheatCodes } from '@aztec/aztec/testing';
 import { FeeAssetHandlerContract, RegistryContract, RollupContract } from '@aztec/ethereum/contracts';
 import { deployRollupForUpgrade } from '@aztec/ethereum/deploy-aztec-l1-contracts';
@@ -13,6 +14,7 @@ import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'
 import { L1TxUtils, createL1TxUtils } from '@aztec/ethereum/l1-tx-utils';
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
 import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import {
   GovernanceAbi,
@@ -42,7 +44,6 @@ import { shouldCollectMetrics } from '../fixtures/fixtures.js';
 import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js';
 import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNodes, createProverNode } from '../fixtures/setup_p2p_test.js';
 import { setupSharedBlobStorage } from '../fixtures/utils.js';
-import { waitForL1ToL2MessageSeen } from '../shared/wait_for_l1_to_l2_message.js';
 import { TestWallet } from '../test-wallet/test_wallet.js';
 import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES } from './p2p_network.js';
 
@@ -53,7 +54,7 @@ const BOOT_NODE_UDP_PORT = 4500;
 const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'add-rollup-old-'));
 const DATA_DIR_NEW = fs.mkdtempSync(path.join(os.tmpdir(), 'add-rollup-new-'));
 
-jest.setTimeout(1000 * 60 * 10);
+jest.setTimeout(1000 * 60 * 20);
 
 /**
  * This test emulates the addition of a new rollup to the registry and tests that cross-chain messages work.
@@ -80,6 +81,14 @@ describe('e2e_p2p_add_rollup', () => {
         ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES,
         listenAddress: '127.0.0.1',
         governanceProposerRoundSize: 10,
+        enableProposerPipelining: true,
+        // Allow validators to build empty checkpoints under pipelining so the chain keeps
+        // advancing while we wait for L1->L2 messages to land in the next checkpoint's inbox tree.
+        minTxsPerBlock: 0,
+        // Pipelining starts cycle for checkpoint N+1 during slot N, but the inbox tree for
+        // checkpoint N is only sealed when checkpoint N is published. inboxLag: 2 sources
+        // L1->L2 messages from checkpoint N-1 (already sealed), avoiding L1ToL2MessagesNotReadyError.
+        inboxLag: 2,
       },
       startProverNode: false, // Start one later using p2p.
     });
@@ -307,7 +316,10 @@ describe('e2e_p2p_add_rollup', () => {
       });
 
       const makeMessageConsumable = async (msgHash: Fr) => {
-        await waitForL1ToL2MessageSeen(node, msgHash, { timeoutSeconds: 10 });
+        // Wait until the message is ready to be consumed (the rollup has reached the message's checkpoint).
+        // Using waitForL1ToL2MessageReady rather than isL1ToL2MessageSynced because with `inboxLag > 0`
+        // a synced message is not yet present in the latest checkpoint's inbox tree.
+        await waitForL1ToL2MessageReady(node, msgHash, { timeoutSeconds: 120 });
 
         const { receipt } = await testContract.methods
           .create_l2_to_l1_message_arbitrary_recipient_private(contentOutFromRollup, ethRecipient)
@@ -578,6 +590,21 @@ describe('e2e_p2p_add_rollup', () => {
     // The new rollup should have no checkpoints
     expect(await newRollup.getCheckpointNumber()).toBe(CheckpointNumber(0));
 
+    // Wait for the new rollup to publish its first checkpoint AND for `nodes[0]` to have synced
+    // it locally, before the second bridging step. The bridge wallet uses
+    // `syncChainTip: 'checkpointed'`, which falls back to the genesis block when no checkpoint
+    // exists. After warping ~500 epochs forward, txs anchored at genesis would expire before
+    // being included. We poll the node's local view (not just the L1 rollup contract) so the PXE
+    // and the assertion observe the same chain state.
+    t.logger.info(`Waiting for new rollup to publish its first checkpoint`);
+    await retryUntil(
+      async () => Number(await nodes[0].getCheckpointNumber('checkpointed')) > 0,
+      'newRollup first checkpoint synced by node',
+      300,
+      2,
+    );
+    t.logger.info(`New rollup published its first checkpoint`);
+
     // Bridge into and out of the new rollup to ensure that it works.
     await bridging(
       nodes[0],

yarn-project/end-to-end/src/e2e_p2p/data_withholding_slash.test.ts

@@ -157,8 +157,36 @@ describe('e2e_p2p_data_withholding_slash', () => {
     t.logger.warn('L2 txs mined');
 
     t.logger.warn('Stopping nodes');
+    // removeInitialNode sends a dummy L1 tx and awaits its receipt to sync the
+    // dateProvider, so it must run while L1 mining is still active.
     await t.removeInitialNode();
-    // Now stop the nodes,
+
+    // Pause L1 block production while we tear down and recreate validators. With
+    // `aztecProofSubmissionEpochs=0`, epoch 8 becomes prunable as soon as epoch 9 begins
+    // (~32s after slot 17). The stop/wipe/recreate cycle takes longer than that, so L1
+    // would otherwise race past the prune deadline before the recreated nodes come up.
+    // When that happens, the recreated archivers detect the prune during their initial
+    // sync (`handleEpochPrune` emits `L2PruneUnproven`), but the `EpochPruneWatcher`
+    // listener is only attached after `archiver.waitForInitialSync()` resolves
+    // (see `aztec-node/server.ts`), so the event is dropped and `DATA_WITHHOLDING` is
+    // never emitted. By freezing L1 here, the recreated archivers ingest checkpoint 1
+    // cleanly during initial sync, the watcher starts and attaches its listener, and
+    // then we resume L1 below so the prune fires while the listener is live.
+    const ethCheatCodes = t.ctx.cheatCodes.eth;
+    await ethCheatCodes.setAutomine(false);
+    await ethCheatCodes.setIntervalMining(0);
+
+    // Fail fast if we paused too late — i.e. if L1 already crossed into epoch 9 before
+    // we got here. In that case the recreated nodes would still see the prune during
+    // initial sync and the test would flake exactly the same way.
+    const epochAtPause = await rollup.getCurrentEpoch();
+    expect(Number(epochAtPause)).toBeLessThan(9);
+
+    // Now stop the validator nodes. With L1 paused, any in-flight L1 submissions from
+    // the validator sequencers would hang `sequencer.stop()` (it awaits pending L1
+    // submissions). Since `minTxsPerBlock=1` and no txs are queued for slot 18+, the
+    // sequencers don't submit further L1 transactions after the slot-17 checkpoint
+    // (already published before `waitForTx` returned), so this is safe.
     await t.stopNodes(nodes);
     // And remove the data directories (which forms the crux of the "attack")
     for (let i = 0; i < NUM_VALIDATORS; i++) {
@@ -186,6 +214,16 @@ describe('e2e_p2p_data_withholding_slash', () => {
     // Wait for P2P mesh to be fully formed before proceeding
     await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS);
 
+    // Resume L1 block production. Warp L1 forward to current wall-clock time so the
+    // epoch-8 deadline is crossed immediately on the next L1 block, then re-enable
+    // interval mining. By now each recreated archiver has block 1 stored locally and
+    // its `EpochPruneWatcher` listener is attached, so the next sync iteration emits
+    // `L2PruneUnproven` for epoch 8 to a live listener → `DATA_WITHHOLDING`.
+    const resumeTimestamp = Math.floor(t.ctx.dateProvider.now() / 1000);
+    await ethCheatCodes.setNextBlockTimestamp(resumeTimestamp);
+    await ethCheatCodes.mine();
+    await ethCheatCodes.setIntervalMining(t.ctx.aztecNodeConfig.ethereumSlotDuration);
+
     const offenses = await awaitOffenseDetected({
       epochDuration: t.ctx.aztecNodeConfig.aztecEpochDuration,
       logger: t.logger,

yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts

@@ -290,6 +290,14 @@ export class LightweightCheckpointBuilder {
       totalManaUsed,
     });
 
+    this.logger.debug(`Completed checkpoint ${this.checkpointNumber}`, {
+      checkpointNumber: this.checkpointNumber,
+      headerHash: header.hash().toString(),
+      checkpointOutHash: checkpointOutHash.toString(),
+      numPreviousCheckpointOutHashes: this.previousCheckpointOutHashes.length,
+      ...header.toInspect(),
+    });
+
     return new Checkpoint(newArchive, header, blocks, this.checkpointNumber, this.feeAssetPriceModifier);
   }
 

yarn-project/prover-client/src/orchestrator/orchestrator.ts

@@ -896,7 +896,11 @@ export class ProvingOrchestrator implements EpochProver {
         },
       ),
       async result => {
-        this.logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
+        this.logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`, {
+          blockNumber: provingState.blockNumber,
+          checkpointIndex: provingState.parentCheckpoint.index,
+          ...result.inputs.toInspect(),
+        });
 
         const leafLocation = provingState.setBlockRootRollupProof(result);
         const checkpointProvingState = provingState.parentCheckpoint;
@@ -1015,6 +1019,11 @@ export class ProvingOrchestrator implements EpochProver {
         signal => this.prover.getBlockMergeRollupProof(inputs, signal, provingState.epochNumber),
       ),
       async result => {
+        this.logger.debug(`Completed block merge rollup proof for checkpoint ${provingState.index}`, {
+          checkpointIndex: provingState.index,
+          mergeLocation: location,
+          ...result.inputs.toInspect(),
+        });
         provingState.setBlockMergeRollupProof(location, result);
         await this.checkAndEnqueueNextBlockMergeRollup(provingState, location);
       },
@@ -1067,7 +1076,10 @@ export class ProvingOrchestrator implements EpochProver {
           return;
         }
 
-        this.logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
+        this.logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}`, {
+          checkpointIndex: provingState.index,
+          ...result.inputs.toInspect(),
+        });
 
         const leafLocation = provingState.setCheckpointRootRollupProof(result);
         const epochProvingState = provingState.parentEpoch;

yarn-project/prover-node/src/job/epoch-proving-job.ts

@@ -191,11 +191,12 @@ export class EpochProvingJob implements Traceable {
         const previousHeader = previousBlockHeaders[checkpointIndex];
         const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
 
-        this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
+        this.log.debug(`Starting processing checkpoint ${checkpoint.number}`, {
           number: checkpoint.number,
           checkpointHash: checkpoint.hash().toString(),
-          lastArchive: checkpoint.header.lastArchiveRoot,
-          previousHeader: previousHeader.hash(),
+          headerHash: checkpoint.header.hash().toString(),
+          numL1ToL2Messages: l1ToL2Messages.length,
+          previousBlockNumber: previousHeader.globalVariables.blockNumber,
           uuid: this.uuid,
         });
 

yarn-project/prover-node/src/prover-node-publisher.ts

@@ -203,10 +203,14 @@ export class ProverNodePublisher {
     const argsPublicInputs = [...publicInputs.toFields()];
 
     if (!areArraysEqual(rollupPublicInputs, argsPublicInputs, (a, b) => a.equals(b))) {
-      const fmt = (inputs: Fr[] | readonly string[]) => inputs.map(x => x.toString()).join(', ');
-      throw new Error(
-        `Root rollup public inputs mismatch:\nRollup:  ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
-      );
+      throw await reportPublicInputsMismatch({
+        rollupPublicInputs,
+        argsPublicInputs,
+        fromCheckpoint,
+        toCheckpoint,
+        rollupContract: this.rollupContract,
+        log: this.log,
+      });
     }
   }
 
@@ -372,3 +376,100 @@ export class ProverNodePublisher {
     };
   }
 }
+
+/**
+ * Decodes a `Root rollup public inputs mismatch`, fetches the on-chain CheckpointLog for any
+ * mismatching `checkpointHeaderHashes[i]`, emits a structured error log, and returns a thrown-ready
+ * Error with a human-readable summary.
+ *
+ * Layout of `RootRollupPublicInputs.toFields()`:
+ *   [0]                   previousArchiveRoot
+ *   [1]                   endArchiveRoot
+ *   [2]                   outHash
+ *   [3 .. 3+N-1]          checkpointHeaderHashes[i] for i in 0..N-1   (N = MAX_CHECKPOINTS_PER_EPOCH)
+ *   [3+N .. 3+3N-1]       fees[i] = (recipient, value) for i in 0..N-1
+ *   [3+3N .. 3+3N+4]      EpochConstantData (chainId, version, vkTreeRoot, protocolContractsHash, proverId)
+ *   [3+3N+5 ..]           blobPublicInputs (FinalBlobAccumulator)
+ */
+async function reportPublicInputsMismatch(input: {
+  rollupPublicInputs: readonly Fr[];
+  argsPublicInputs: readonly Fr[];
+  fromCheckpoint: CheckpointNumber;
+  toCheckpoint: CheckpointNumber;
+  rollupContract: RollupContract;
+  log: Logger;
+}): Promise<Error> {
+  const { rollupPublicInputs, argsPublicInputs, fromCheckpoint, toCheckpoint, rollupContract, log } = input;
+  const N = MAX_CHECKPOINTS_PER_EPOCH;
+  const constantsStart = 3 + 3 * N;
+  const blobStart = constantsStart + 5;
+  const constantLabels = ['chainId', 'version', 'vkTreeRoot', 'protocolContractsHash', 'proverId'];
+
+  const diffs: { index: number; label: string; rollup: Fr; computed: Fr; checkpointIndex?: number }[] = [];
+  const len = Math.max(rollupPublicInputs.length, argsPublicInputs.length);
+  for (let i = 0; i < len; i++) {
+    const a = rollupPublicInputs[i] ?? Fr.ZERO;
+    const b = argsPublicInputs[i] ?? Fr.ZERO;
+    if (a.equals(b)) {
+      continue;
+    }
+    let label: string;
+    let checkpointIndex: number | undefined;
+    if (i === 0) {
+      label = 'previousArchiveRoot';
+    } else if (i === 1) {
+      label = 'endArchiveRoot';
+    } else if (i === 2) {
+      label = 'outHash';
+    } else if (i < 3 + N) {
+      checkpointIndex = i - 3;
+      label = `checkpointHeaderHashes[${checkpointIndex}]`;
+    } else if (i < 3 + 3 * N) {
+      const feePairIndex = i - (3 + N);
+      const feeIndex = Math.floor(feePairIndex / 2);
+      const sub = feePairIndex % 2 === 0 ? 'recipient' : 'value';
+      label = `fees[${feeIndex}].${sub}`;
+    } else if (i < blobStart) {
+      label = `constants.${constantLabels[i - constantsStart]}`;
+    } else {
+      label = `blobPublicInputs[${i - blobStart}]`;
+    }
+    diffs.push({ index: i, label, rollup: a, computed: b, checkpointIndex });
+  }
+
+  // For each mismatching checkpointHeaderHash, fetch the L1 CheckpointLog so the operator can
+  // see what was published on-chain alongside the prover's recomputed hash.
+  const onChainCheckpoints = await Promise.all(
+    diffs
+      .filter(d => d.checkpointIndex !== undefined)
+      .map(async d => {
+        const checkpointNumber = CheckpointNumber(fromCheckpoint + d.checkpointIndex!);
+        try {
+          const cp = await rollupContract.getCheckpoint(checkpointNumber);
+          return { checkpointIndex: d.checkpointIndex!, checkpointNumber, headerHash: cp.headerHash.toString() };
+        } catch (err) {
+          return { checkpointIndex: d.checkpointIndex!, checkpointNumber, error: (err as Error).message };
+        }
+      }),
+  );
+
+  log.error(`Root rollup public inputs mismatch`, undefined, {
+    fromCheckpoint,
+    toCheckpoint,
+    numDiffs: diffs.length,
+    diffs: diffs.map(d => ({
+      index: d.index,
+      label: d.label,
+      rollup: d.rollup.toString(),
+      computed: d.computed.toString(),
+    })),
+    onChainCheckpoints,
+  });
+
+  const fmt = (inputs: readonly Fr[]) => inputs.map(x => x.toString()).join(', ');
+  const summary = diffs.map(d => `[${d.index} ${d.label}] L1=${d.rollup} prover=${d.computed}`).join('\n');
+  return new Error(
+    `Root rollup public inputs mismatch (${diffs.length} fields differ):\n${summary}\n` +
+      `Rollup:  ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
+  );
+}