feat(prover-node): checkpoint-driven optimistic proving

yarn-project/end-to-end/bootstrap.sh

@@ -33,10 +33,12 @@ function test_cmds {
     echo "$prefix:NAME=e2e_prover_full_fake FAKE_PROOFS=1 $run_test_script simple e2e_prover/full"
   fi
   echo "$prefix:TIMEOUT=15m:NAME=e2e_block_building $(set_dump_avm e2e_block_building) $run_test_script simple e2e_block_building"
+  echo "$prefix:TIMEOUT=15m:NAME=e2e_epochs/epochs_long_proving_time $run_test_script simple src/e2e_epochs/epochs_long_proving_time.test.ts"
 
   local tests=(
     # List all standalone and nested tests, except for the ones listed above.
-    src/e2e_!(prover)/*.test.ts
+    src/e2e_!(prover|epochs)/*.test.ts
+    src/e2e_epochs/!(epochs_long_proving_time).test.ts
     src/e2e_p2p/reqresp/*.test.ts
     src/e2e_!(block_building).test.ts
   )

yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts

@@ -6,7 +6,9 @@ import { jest } from '@jest/globals';
 
 import { EpochsTestContext } from './epochs_test.js';
 
-jest.setTimeout(1000 * 60 * 10);
+jest.setTimeout(1000 * 60 * 15);
+
+const MAX_JOB_COUNT = 20;
 
 describe('e2e_epochs/epochs_long_proving_time', () => {
   let logger: Logger;
@@ -24,11 +26,14 @@ describe('e2e_epochs/epochs_long_proving_time', () => {
     const { aztecSlotDuration } = EpochsTestContext.getSlotDurations({ aztecEpochDuration });
     const epochDurationInSeconds = aztecSlotDuration * aztecEpochDuration;
     const proverTestDelayMs = (epochDurationInSeconds * 1000 * 3) / 4;
+    // Each epoch takes ~3 epochs to prove, so the broker needs to keep results for
+    // at least that many epochs to avoid rejecting jobs as stale.
     test = await EpochsTestContext.setup({
       aztecEpochDuration,
       aztecProofSubmissionEpochs: 1000, // Effectively don't re-org
       proverTestDelayMs,
-      proverNodeMaxPendingJobs: 1, // We test for only a single job at once
+      proverNodeMaxPendingJobs: MAX_JOB_COUNT, // Prove multiple epochs concurrently
+      proverBrokerMaxEpochsToKeepResultsFor: 10,
       enableProposerPipelining: true,
     });
     ({ logger, monitor, L1_BLOCK_TIME_IN_S } = test);
@@ -59,10 +64,7 @@ describe('e2e_epochs/epochs_long_proving_time', () => {
     // At least 3 epochs should have passed after the proven one (though we add a -1 just in case)
     expect(monitor.checkpointNumber).toBeGreaterThanOrEqual(targetProvenEpochs * test.epochDuration * 3 - 1);
 
-    // We expect maxJobCount to equal 1, since the prover node epoch monitor defines an epoch as ready to be proven
-    // only if the previous one has already been proven. We can relax this check if we want to support multiple epochs
-    // to be proven in parallel, in which case we should update the assertion below.
-    expect(maxJobCount).toEqual(1);
-    logger.info(`Test succeeded`);
+    expect(maxJobCount).toBeLessThanOrEqual(MAX_JOB_COUNT);
+    logger.info(`Test succeeded, max prover jobs ${maxJobCount}`);
   });
 });

yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts

@@ -46,18 +46,18 @@ describe('e2e_epochs/epochs_multi_proof', () => {
     // This prevents the race condition where multiple provers submit to L1 at the same time
     test.proverNodes.forEach((proverAztecNode, index) => {
       const proverManager = proverAztecNode.getProverNode()!.getProver();
-      const origCreateEpochProver = proverManager.createEpochProver.bind(proverManager);
-      proverManager.createEpochProver = () => {
-        const epochProver = origCreateEpochProver();
-        const origFinalizeEpoch = epochProver.finalizeEpoch.bind(epochProver);
-        epochProver.finalizeEpoch = async () => {
-          const result = await origFinalizeEpoch();
+      const origCreateTopTree = proverManager.createTopTreeOrchestrator.bind(proverManager);
+      proverManager.createTopTreeOrchestrator = () => {
+        const topTree = origCreateTopTree();
+        const origProve = topTree.prove.bind(topTree);
+        topTree.prove = async (...args: Parameters<typeof origProve>) => {
+          const result = await origProve(...args);
           const sleepTime = index * 1000 * test.constants.ethereumSlotDuration;
-          logger.warn(`Delaying finalizeEpoch for prover node ${index} by ${sleepTime}ms`);
+          logger.warn(`Delaying top-tree prove for prover node ${index} by ${sleepTime}ms`);
           await sleep(sleepTime);
           return result;
         };
-        return epochProver;
+        return topTree;
       };
     });