From d2a8980435394240df3c9c59be23640d55039c23 Mon Sep 17 00:00:00 2001 From: Amin Sammara <84764772+aminsammara@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:52:50 +0300 Subject: [PATCH 01/35] feat(cli): support funding accounts in validator-keys new/set-funding-account (#24476) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Wires a real `--funding-account` option into `aztec validator-keys new` (previously commented out with a TODO, so operators had to hand-edit the keystore JSON), and adds a dedicated `set-funding-account` subcommand for existing keystores. Per review feedback, `add` does **not** take `--funding-account`: the funding account is a keystore-level field (the only one `KeystoreManager.createFundingSigner` reads), so setting it from `add` would be a global mutation disguised as a per-validator flag. Use `set-funding-account` instead: ``` aztec validator-keys set-funding-account [--remote-signer ] [--password ] ``` ## Behavior - The funding account value accepts either a 32-byte private key or a 20-byte address. - An address requires `--remote-signer` (a local funder needs its private key to sign funding txs); it is stored as `{ address, remoteSignerUrl }`. - With `--password`, a plaintext funder key is encrypted to a JSON V3 file and replaced with a `{ path, password }` reference, mirroring how attester/publisher keys are handled. - The value is written at the keystore top level (`keystore.fundingAccount`). Validator-level funding exists in the schema but is never consumed at runtime, so it is intentionally not emitted. - `set-funding-account` replaces an existing funding account with a warning. ## Changes - `index.ts` — real `--funding-account` option on `new`; new `set-funding-account` subcommand. - `set_funding_account.ts` — new command: validate, resolve, optionally encrypt, write `keystore.fundingAccount`. - `utils.ts` — `validateFundingAccountOptions` (normalize + validate key/address, require remote-signer for address form). - `shared.ts` — `resolveFundingAccount` and `encryptFundingAccountToFile`; extracted the ETH JSON V3 encryption helper for reuse. - `new.ts` — validate, resolve, optionally encrypt, set `keystore.fundingAccount`. - `valkeys.test.ts` — 17 new tests covering validation, private-key/address/password paths on `new`, and set/replace on `set-funding-account`. ## Testing - `yarn workspace @aztec/cli test src/cmds/validator_keys/valkeys.test.ts` — 60 passed. - `yarn build`, `yarn format cli`, `yarn lint cli` — all clean. (cherry picked from commit 3aa68f6d748f199e323b5882bd3121b3a4451201) --- .../cli/src/cmds/validator_keys/index.ts | 34 ++- .../cli/src/cmds/validator_keys/new.ts | 23 +- .../validator_keys/set_funding_account.ts | 55 +++++ .../cli/src/cmds/validator_keys/shared.ts | 38 ++- .../cli/src/cmds/validator_keys/utils.ts | 45 +++- .../src/cmds/validator_keys/valkeys.test.ts | 228 +++++++++++++++++- .../src/keystore_manager.test.ts | 38 +++ 7 files changed, 453 insertions(+), 8 deletions(-) create mode 100644 yarn-project/cli/src/cmds/validator_keys/set_funding_account.ts diff --git a/yarn-project/cli/src/cmds/validator_keys/index.ts b/yarn-project/cli/src/cmds/validator_keys/index.ts index 8588f40c55a7..0c6088a6cbf7 100644 --- a/yarn-project/cli/src/cmds/validator_keys/index.ts +++ b/yarn-project/cli/src/cmds/validator_keys/index.ts @@ -33,8 +33,10 @@ export function injectCommands(program: Command, log: LogFn) { 'Coinbase ETH address to use when proposing. Defaults to attester address.', parseEthereumAddress, ) - // TODO: add funding account back in when implemented - // .option('--funding-account ', 'ETH private key (or address for remote signer setup) to fund publishers') + .option( + '--funding-account ', + 'ETH funding account used to top up publisher EOAs. Provide a private key, or an address together with --remote-signer.', + ) .option('--remote-signer ', 'Default remote signer URL for accounts in this file') .option('--ikm ', 'Initial keying material for BLS (alternative to mnemonic)', value => parseHex(value, 32)) .option('--bls-path ', `EIP-2334 path (default ${defaultBlsPath})`) @@ -99,8 +101,6 @@ export function injectCommands(program: Command, log: LogFn) { 'Coinbase ETH address to use when proposing. Defaults to attester address.', parseEthereumAddress, ) - // TODO: add funding account back in when implemented - // .option('--funding-account ', 'ETH private key (or address for remote signer setup) to fund publishers') .option('--remote-signer ', 'Default remote signer URL for accounts in this file') .option('--ikm ', 'Initial keying material for BLS (alternative to mnemonic)', value => parseHex(value, 32)) .option('--bls-path ', `EIP-2334 path (default ${defaultBlsPath})`) @@ -117,6 +117,32 @@ export function injectCommands(program: Command, log: LogFn) { await addValidatorKeys(existing, options, log); }); + group + .command('set-funding-account') + .summary('Set the funding account of an existing keystore') + .description( + 'Sets the keystore-level ETH funding account used to top up publisher EOAs, replacing any existing one', + ) + .argument('', 'Path to existing keystore JSON') + .argument( + '', + 'Funding account: a private key, or an address (needs --remote-signer unless the keystore already defines one)', + ) + .option( + '--remote-signer ', + 'Remote signer URL for the funding account (required with an address unless the keystore already defines one)', + ) + .option( + '--password ', + 'Password for writing the funding key as an encrypted ETH JSON V3 file. Empty string allowed', + ) + .option('--encrypted-keystore-dir ', 'Output directory for the encrypted funding key file') + .option('--json', 'Echo resulting JSON to stdout') + .action(async (existing: string, fundingAccount: string, options) => { + const { setFundingAccount } = await import('./set_funding_account.js'); + await setFundingAccount(existing, fundingAccount, options, log); + }); + group .command('staker') .summary('Generate staking JSON from keystore') diff --git a/yarn-project/cli/src/cmds/validator_keys/new.ts b/yarn-project/cli/src/cmds/validator_keys/new.ts index 208ebc7a8748..bc8ebc9826bd 100644 --- a/yarn-project/cli/src/cmds/validator_keys/new.ts +++ b/yarn-project/cli/src/cmds/validator_keys/new.ts @@ -9,12 +9,14 @@ import { wordlist } from '@scure/bip39/wordlists/english.js'; import { readFile, writeFile } from 'fs/promises'; import { basename, dirname, join } from 'path'; import { createPublicClient, fallback, http } from 'viem'; -import { generateMnemonic, mnemonicToAccount } from 'viem/accounts'; +import { generateMnemonic, mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { buildValidatorEntries, + encryptFundingAccountToFile, logValidatorSummaries, maybePrintJson, + resolveFundingAccount, resolveKeystoreOutputPath, writeBlsBn254ToFile, writeEthJsonV3ToFile, @@ -23,6 +25,7 @@ import { import { processAttesterAccounts } from './staker.js'; import { validateBlsPathOptions, + validateFundingAccountOptions, validatePublisherOptions, validateRemoteSignerOptions, validateStakerOutputOptions, @@ -51,6 +54,7 @@ export type NewValidatorKeystoreOptions = { json?: boolean; feeRecipient: AztecAddress; coinbase?: EthAddress; + fundingAccount?: string; remoteSigner?: string; stakerOutput?: boolean; gseAddress?: EthAddress; @@ -147,6 +151,8 @@ export async function newValidatorKeystore(options: NewValidatorKeystoreOptions, validatePublisherOptions(options); // validate remote signer options validateRemoteSignerOptions(options); + // validate funding account option + validateFundingAccountOptions(options); const { dataDir, @@ -156,6 +162,7 @@ export async function newValidatorKeystore(options: NewValidatorKeystoreOptions, publishers, json, coinbase, + fundingAccount, accountIndex = 0, addressIndex = 0, feeRecipient, @@ -213,17 +220,26 @@ export async function newValidatorKeystore(options: NewValidatorKeystoreOptions, remoteSigner, }); + let resolvedFundingAccount = fundingAccount ? resolveFundingAccount(fundingAccount, remoteSigner) : undefined; + // If password provided, write ETH JSON V3 and BLS BN254 keystores and replace plaintext if (shouldEncryptKeystores) { const encryptedKeystoreOutDir = encryptedKeystoreDir && encryptedKeystoreDir.length > 0 ? encryptedKeystoreDir : keystoreOutDir; await writeEthJsonV3ToFile(validators, { outDir: encryptedKeystoreOutDir, password: ethPassword }); await writeBlsBn254ToFile(validators, { outDir: encryptedKeystoreOutDir, password: blsPassword }); + if (resolvedFundingAccount) { + resolvedFundingAccount = await encryptFundingAccountToFile(resolvedFundingAccount, { + outDir: encryptedKeystoreOutDir, + password: ethPassword, + }); + } } const keystore = { schemaVersion: 1, validators, + ...(resolvedFundingAccount ? { fundingAccount: resolvedFundingAccount } : {}), }; await writeKeystoreFile(outputPath, keystore); @@ -285,6 +301,11 @@ export async function newValidatorKeystore(options: NewValidatorKeystoreOptions, // print a concise summary of public keys (addresses and BLS pubkeys) if no --json options was selected if (!json) { logValidatorSummaries(log, summaries); + if (fundingAccount) { + const funderAddress = + fundingAccount.length === 66 ? privateKeyToAccount(fundingAccount as `0x${string}`).address : fundingAccount; + log(`funding account: ${funderAddress}`); + } } if (mnemonic && remoteSigner && !json) { diff --git a/yarn-project/cli/src/cmds/validator_keys/set_funding_account.ts b/yarn-project/cli/src/cmds/validator_keys/set_funding_account.ts new file mode 100644 index 000000000000..21258abb56b1 --- /dev/null +++ b/yarn-project/cli/src/cmds/validator_keys/set_funding_account.ts @@ -0,0 +1,55 @@ +import type { LogFn } from '@aztec/foundation/log'; +import { loadKeystoreFile } from '@aztec/node-keystore/loader'; +import type { KeyStore } from '@aztec/node-keystore/types'; + +import { dirname } from 'path'; +import { privateKeyToAccount } from 'viem/accounts'; + +import { encryptFundingAccountToFile, maybePrintJson, resolveFundingAccount, writeKeystoreFile } from './shared.js'; +import { validateFundingAccountOptions } from './utils.js'; + +export type SetFundingAccountOptions = { + remoteSigner?: string; + password?: string; + encryptedKeystoreDir?: string; + json?: boolean; +}; + +/** + * Sets the top-level funding account of an existing keystore, replacing any previous one. The + * account may be a private key, or an address paired with a remote signer URL. With a password, + * a plaintext key is encrypted to an ETH JSON V3 file and stored as a { path, password } reference. + */ +export async function setFundingAccount( + existing: string, + fundingAccount: string, + options: SetFundingAccountOptions, + log: LogFn, +) { + const { remoteSigner, password, encryptedKeystoreDir, json } = options; + + const keystore: KeyStore = loadKeystoreFile(existing); + + const validated = { fundingAccount, remoteSigner }; + validateFundingAccountOptions(validated, !!keystore.remoteSigner); + + let resolved = resolveFundingAccount(validated.fundingAccount!, remoteSigner); + if (password !== undefined) { + const outDir = encryptedKeystoreDir && encryptedKeystoreDir.length > 0 ? encryptedKeystoreDir : dirname(existing); + resolved = await encryptFundingAccountToFile(resolved, { outDir, password }); + } + + if (keystore.fundingAccount) { + log('Replacing existing funding account in keystore'); + } + keystore.fundingAccount = resolved; + + await writeKeystoreFile(existing, keystore); + + if (!json) { + const value = validated.fundingAccount!; + const funderAddress = value.length === 66 ? privateKeyToAccount(value as `0x${string}`).address : value; + log(`Set funding account ${funderAddress} in ${existing}`); + } + maybePrintJson(log, !!json, keystore as unknown as Record); +} diff --git a/yarn-project/cli/src/cmds/validator_keys/shared.ts b/yarn-project/cli/src/cmds/validator_keys/shared.ts index 981acc155e1b..64d162454505 100644 --- a/yarn-project/cli/src/cmds/validator_keys/shared.ts +++ b/yarn-project/cli/src/cmds/validator_keys/shared.ts @@ -3,7 +3,7 @@ import { asyncPool } from '@aztec/foundation/async-pool'; import { deriveBlsPrivateKey } from '@aztec/foundation/crypto/bls'; import { createBn254Keystore } from '@aztec/foundation/crypto/bls/bn254_keystore'; import { computeBn254G1PublicKeyCompressed } from '@aztec/foundation/crypto/bn254'; -import type { EthAddress } from '@aztec/foundation/eth-address'; +import { EthAddress } from '@aztec/foundation/eth-address'; import type { LogFn } from '@aztec/foundation/log'; import type { EthAccount, EthPrivateKey, ValidatorKeyStore } from '@aztec/node-keystore/types'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; @@ -125,6 +125,21 @@ export function deriveEthAttester( : (('0x' + Buffer.from(acct.getHdKey().privateKey!).toString('hex')) as EthPrivateKey); } +/** + * Resolve a `--funding-account` value into a keystore `EthAccount`. A 66-char value is a private key + * (used verbatim). A 42-char value is an address: with an explicit `remoteSigner` URL it becomes an + * `{ address, remoteSignerUrl }` pair; without one it is stored as a bare address that falls back to + * the keystore-level remote signer at runtime. Callers must validate the value first (see + * `validateFundingAccountOptions`). + */ +export function resolveFundingAccount(fundingAccount: string, remoteSigner?: string): EthAccount { + if (fundingAccount.length === 66) { + return fundingAccount as EthPrivateKey; + } + const address = EthAddress.fromString(fundingAccount); + return remoteSigner ? ({ address, remoteSignerUrl: remoteSigner } as EthAccount) : address; +} + export async function buildValidatorEntries(input: BuildValidatorsInput) { const { validatorCount, @@ -330,6 +345,27 @@ export async function writeEthJsonV3Keystore( return outPath; } +/** + * If `account` is a plaintext ETH private key, encrypt it to a JSON V3 file and return a + * { path, password } reference; otherwise return it unchanged. + */ +async function maybeEncryptEthAccount(account: any, label: string, options: { outDir: string; password: string }) { + if (typeof account === 'string' && account.startsWith('0x') && account.length === 66) { + const fileBase = `${label}_${account.slice(2, 10)}`; + const p = await writeEthJsonV3Keystore(options.outDir, fileBase, options.password, account); + return { path: p, password: options.password }; + } + return account; +} + +/** Encrypt a plaintext funding-account key to a JSON V3 file, replacing it with a { path, password } reference. */ +export async function encryptFundingAccountToFile( + account: EthAccount, + options: { outDir: string; password: string }, +): Promise { + return (await maybeEncryptEthAccount(account, 'funding', options)) as EthAccount; +} + /** Replace plaintext ETH keys in validators with { path, password } pointing to JSON V3 files. */ export async function writeEthJsonV3ToFile( validators: ValidatorKeyStore[], diff --git a/yarn-project/cli/src/cmds/validator_keys/utils.ts b/yarn-project/cli/src/cmds/validator_keys/utils.ts index 8b8dca71f191..1e83594f9b6a 100644 --- a/yarn-project/cli/src/cmds/validator_keys/utils.ts +++ b/yarn-project/cli/src/cmds/validator_keys/utils.ts @@ -1,4 +1,4 @@ -import type { EthAddress } from '@aztec/foundation/eth-address'; +import { EthAddress } from '@aztec/foundation/eth-address'; import { ethPrivateKeySchema } from '@aztec/node-keystore/schemas'; import type { EthPrivateKey } from '@aztec/node-keystore/types'; @@ -79,3 +79,46 @@ export function validatePublisherOptions(options: { publishers?: string[]; publi options.publishers = normalizedKeys as EthPrivateKey[]; } } + +/** + * Validates and normalizes the `--funding-account` option in place. The value may be a private key + * (used as a local signer) or an ETH address. An address needs a remote signer to sign funding txs: + * either `--remote-signer`, or a keystore that already defines one (pass `hasKeystoreRemoteSigner`), + * which a bare address inherits at runtime. + */ +export function validateFundingAccountOptions( + options: { fundingAccount?: string; remoteSigner?: string }, + hasKeystoreRemoteSigner = false, +) { + if (!options.fundingAccount) { + return; + } + + let value = options.fundingAccount.trim(); + if (!value.startsWith('0x')) { + value = '0x' + value; + } + + if (value.length === 66) { + try { + ethPrivateKeySchema.parse(value); + } catch (error) { + throw new Error(`Invalid funding account private key: ${error instanceof Error ? error.message : String(error)}`); + } + } else if (value.length === 42) { + try { + EthAddress.fromString(value); + } catch (error) { + throw new Error(`Invalid funding account address: ${error instanceof Error ? error.message : String(error)}`); + } + if (!options.remoteSigner && !hasKeystoreRemoteSigner) { + throw new Error( + '--funding-account as an address requires --remote-signer, or a keystore that already defines a remote signer', + ); + } + } else { + throw new Error('Invalid funding account: expected a 32-byte private key or a 20-byte address'); + } + + options.fundingAccount = value; +} diff --git a/yarn-project/cli/src/cmds/validator_keys/valkeys.test.ts b/yarn-project/cli/src/cmds/validator_keys/valkeys.test.ts index 24abb2c06169..e647115f7597 100644 --- a/yarn-project/cli/src/cmds/validator_keys/valkeys.test.ts +++ b/yarn-project/cli/src/cmds/validator_keys/valkeys.test.ts @@ -13,6 +13,7 @@ import { mnemonicToAccount } from 'viem/accounts'; import { addValidatorKeys } from './add.js'; import { generateBlsKeypair } from './generate_bls_keypair.js'; import { newValidatorKeystore } from './new.js'; +import { setFundingAccount } from './set_funding_account.js'; import { buildValidatorEntries, computeBlsPublicKeyCompressed, @@ -24,7 +25,7 @@ import { writeEthJsonV3ToFile, writeKeystoreFile, } from './shared.js'; -import { validatePublisherOptions } from './utils.js'; +import { validateFundingAccountOptions, validatePublisherOptions } from './utils.js'; const TEST_MNEMONIC = 'abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about'; @@ -402,6 +403,44 @@ describe('validator keys utilities', () => { }); }); + describe('validateFundingAccountOptions', () => { + const validPrivateKey = '0x' + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'; + const validAddress = '0x' + '01'.repeat(20); + + it('accepts a private key and leaves it normalized', () => { + const options = { fundingAccount: validPrivateKey }; + expect(() => validateFundingAccountOptions(options)).not.toThrow(); + expect(options.fundingAccount).toBe(validPrivateKey); + }); + + it('adds a missing 0x prefix', () => { + const options = { fundingAccount: validPrivateKey.slice(2) }; + expect(() => validateFundingAccountOptions(options)).not.toThrow(); + expect(options.fundingAccount).toBe(validPrivateKey); + }); + + it('accepts an address when a remote signer is set', () => { + const options = { fundingAccount: validAddress, remoteSigner: 'http://localhost:9000' }; + expect(() => validateFundingAccountOptions(options)).not.toThrow(); + expect(options.fundingAccount).toBe(validAddress); + }); + + it('throws for an address without a remote signer', () => { + const options = { fundingAccount: validAddress }; + expect(() => validateFundingAccountOptions(options)).toThrow(/requires --remote-signer/); + }); + + it('throws for a malformed value', () => { + const options = { fundingAccount: '0x1234' }; + expect(() => validateFundingAccountOptions(options)).toThrow(/Invalid funding account/); + }); + + it('is a no-op when unset', () => { + const options = {}; + expect(() => validateFundingAccountOptions(options)).not.toThrow(); + }); + }); + describe('newValidatorKeystore', () => { it('creates a keystore file and logs a summary', async () => { const path = join(tmp, 'created.json'); @@ -946,6 +985,99 @@ describe('validator keys utilities', () => { expect(Array.isArray(validator.publisher)).toBe(true); expect(validator.publisher).toEqual([publisherKey1, publisherKey2]); }); + + it('writes a top-level funding account from a private key', async () => { + const path = join(tmp, 'with-funding-key.json'); + const fundingKey = '0x' + 'ab'.repeat(32); + await newValidatorKeystore( + { + dataDir: tmp, + file: 'with-funding-key.json', + count: 1, + mnemonic: TEST_MNEMONIC, + fundingAccount: fundingKey, + feeRecipient: ('0x' + '11'.repeat(32)) as unknown as AztecAddress, + }, + s => s, + ); + const keystore: KeyStore = loadKeystoreFile(path); + expect(keystore.fundingAccount).toBe(fundingKey); + }); + + it('writes a remote-signer funding account from an address', async () => { + const path = join(tmp, 'with-funding-address.json'); + const fundingAddress = '0x' + '02'.repeat(20); + const remoteSigner = 'http://localhost:9000'; + await newValidatorKeystore( + { + dataDir: tmp, + file: 'with-funding-address.json', + count: 1, + mnemonic: TEST_MNEMONIC, + fundingAccount: fundingAddress, + remoteSigner, + feeRecipient: ('0x' + '12'.repeat(32)) as unknown as AztecAddress, + }, + s => s, + ); + const keystore: KeyStore = loadKeystoreFile(path); + const funder = keystore.fundingAccount as any; + expect(funder.remoteSignerUrl).toBe(remoteSigner); + expect(funder.address.toString().toLowerCase()).toBe(fundingAddress); + }); + + it('rejects a funding-account address without a remote signer', async () => { + await expect( + newValidatorKeystore( + { + dataDir: tmp, + file: 'funding-address-no-signer.json', + count: 1, + mnemonic: TEST_MNEMONIC, + fundingAccount: '0x' + '03'.repeat(20), + feeRecipient: ('0x' + '13'.repeat(32)) as unknown as AztecAddress, + }, + s => s, + ), + ).rejects.toThrow(/requires --remote-signer/); + }); + + it('rejects a malformed funding account', async () => { + await expect( + newValidatorKeystore( + { + dataDir: tmp, + file: 'funding-malformed.json', + count: 1, + mnemonic: TEST_MNEMONIC, + fundingAccount: '0xdead', + feeRecipient: ('0x' + '14'.repeat(32)) as unknown as AztecAddress, + }, + s => s, + ), + ).rejects.toThrow(/Invalid funding account/); + }); + + it('encrypts a plaintext funding account when a password is provided', async () => { + const path = join(tmp, 'with-funding-encrypted.json'); + await newValidatorKeystore( + { + dataDir: tmp, + file: 'with-funding-encrypted.json', + count: 1, + mnemonic: TEST_MNEMONIC, + fundingAccount: '0x' + 'cd'.repeat(32), + password: 'funding-test-pw', + encryptedKeystoreDir: tmp, + feeRecipient: ('0x' + '15'.repeat(32)) as unknown as AztecAddress, + }, + s => s, + ); + const keystore: KeyStore = loadKeystoreFile(path); + const funder = keystore.fundingAccount as any; + expect(typeof funder.path).toBe('string'); + expect(existsSync(funder.path)).toBe(true); + }); }); describe('materialization helpers (invoked directly)', () => { @@ -1021,6 +1153,100 @@ describe('validator keys utilities', () => { }); }); + describe('setFundingAccount', () => { + const writeBaseKeystore = (path: string, extra: Record = {}) => { + const baseKeystore = { + schemaVersion: 1, + validators: [{ attester: '0x' + '0a'.repeat(32), feeRecipient: ('0x' + '06'.repeat(32)) as unknown as string }], + ...extra, + }; + writeFileSync(path, JSON.stringify(baseKeystore, null, 2), 'utf-8'); + }; + + it('sets a funding account on a keystore that has none', async () => { + const existing = join(tmp, 'set-funding.json'); + writeBaseKeystore(existing); + const fundingKey = '0x' + 'ab'.repeat(32); + + const logs: string[] = []; + await setFundingAccount(existing, fundingKey, {}, s => logs.push(s)); + + const updated: KeyStore = loadKeystoreFile(existing); + expect(updated.fundingAccount).toBe(fundingKey); + expect(logs.some(l => l.includes('Replacing existing funding account'))).toBe(false); + expect(logs.some(l => l.includes('Set funding account'))).toBe(true); + }); + + it('replaces an existing funding account and warns', async () => { + const existing = join(tmp, 'replace-funding.json'); + writeBaseKeystore(existing, { fundingAccount: '0x' + 'aa'.repeat(32) }); + const newFundingKey = '0x' + 'bb'.repeat(32); + + const logs: string[] = []; + await setFundingAccount(existing, newFundingKey, {}, s => logs.push(s)); + + const updated: KeyStore = loadKeystoreFile(existing); + expect(updated.fundingAccount).toBe(newFundingKey); + expect(logs.some(l => l.includes('Replacing existing funding account'))).toBe(true); + }); + + it('sets a remote-signer funding account from an address', async () => { + const existing = join(tmp, 'set-funding-address.json'); + writeBaseKeystore(existing); + const fundingAddress = '0x' + '02'.repeat(20); + const remoteSigner = 'http://localhost:9000'; + + await setFundingAccount(existing, fundingAddress, { remoteSigner }, s => s); + + const updated: KeyStore = loadKeystoreFile(existing); + const funder = updated.fundingAccount as any; + expect(funder.remoteSignerUrl).toBe(remoteSigner); + expect(funder.address.toString().toLowerCase()).toBe(fundingAddress); + }); + + it('inherits the keystore remote signer for an address when --remote-signer is omitted', async () => { + const existing = join(tmp, 'set-funding-inherit-signer.json'); + writeBaseKeystore(existing, { remoteSigner: 'http://localhost:9000' }); + const fundingAddress = '0x' + '02'.repeat(20); + + await setFundingAccount(existing, fundingAddress, {}, s => s); + + const updated: KeyStore = loadKeystoreFile(existing); + // Stored as a bare address (no inline remoteSignerUrl); resolved via the keystore-level signer at runtime. + const funder = updated.fundingAccount as any; + expect(funder.remoteSignerUrl).toBeUndefined(); + expect(funder.toString().toLowerCase()).toBe(fundingAddress); + }); + + it('rejects an address without a remote signer', async () => { + const existing = join(tmp, 'set-funding-no-signer.json'); + writeBaseKeystore(existing); + + await expect(setFundingAccount(existing, '0x' + '03'.repeat(20), {}, s => s)).rejects.toThrow( + /requires --remote-signer/, + ); + }); + + it('rejects a malformed funding account', async () => { + const existing = join(tmp, 'set-funding-malformed.json'); + writeBaseKeystore(existing); + + await expect(setFundingAccount(existing, '0xdead', {}, s => s)).rejects.toThrow(/Invalid funding account/); + }); + + it('encrypts a plaintext funding key when a password is provided', async () => { + const existing = join(tmp, 'set-funding-encrypted.json'); + writeBaseKeystore(existing); + + await setFundingAccount(existing, '0x' + 'cd'.repeat(32), { password: '', encryptedKeystoreDir: tmp }, s => s); + + const updated: KeyStore = loadKeystoreFile(existing); + const funder = updated.fundingAccount as any; + expect(typeof funder.path).toBe('string'); + expect(existsSync(funder.path)).toBe(true); + }); + }); + describe('generateBlsKeypair', () => { it('writes to file and logs a write message when out is provided', async () => { const out = join(tmp, 'bls.json'); diff --git a/yarn-project/node-keystore/src/keystore_manager.test.ts b/yarn-project/node-keystore/src/keystore_manager.test.ts index a0bcf268d5cf..8452f1dbe0ac 100644 --- a/yarn-project/node-keystore/src/keystore_manager.test.ts +++ b/yarn-project/node-keystore/src/keystore_manager.test.ts @@ -1605,5 +1605,43 @@ describe('KeystoreManager', () => { expect(signer).toBeUndefined(); }); + + it('resolves a bare-address fundingAccount via the keystore-level remote signer', async () => { + const fundingAddress = EthAddress.random(); + const keystore: KeyStore = { + schemaVersion: 1, + validators: [ + { + attester: EthAddress.random(), + feeRecipient: await AztecAddress.random(), + }, + ], + remoteSigner: 'http://localhost:9000', + fundingAccount: fundingAddress, + }; + + const manager = new KeystoreManager(keystore); + const signer = manager.createFundingSigner(); + + expect(signer).toBeInstanceOf(RemoteSigner); + expect(signer!.address.equals(fundingAddress)).toBeTruthy(); + }); + + it('throws for a bare-address fundingAccount when no remote signer is configured', async () => { + const keystore: KeyStore = { + schemaVersion: 1, + validators: [ + { + attester: EthAddress.random(), + feeRecipient: await AztecAddress.random(), + }, + ], + fundingAccount: EthAddress.random(), + }; + + const manager = new KeystoreManager(keystore); + + expect(() => manager.createFundingSigner()).toThrow(/No remote signer configuration found/); + }); }); }); From 78cdd03140294b45385ad1b33812102765e5b801 Mon Sep 17 00:00:00 2001 From: aminsammara Date: Tue, 21 Jul 2026 09:44:59 +0000 Subject: [PATCH 02/35] chore(spartan): lower mainnet inactivity slash target to 70% Reduce the mainnet SLASH_INACTIVITY_TARGET_PERCENTAGE default from 0.8 (80%) to 0.7 (70%) in spartan/environments/network-defaults.yml. Only the networks.mainnet preset is changed; devnet/testnet and the top-level slasher anchor (all 0.9) are untouched. (cherry picked from commit 71f8e3d74d5098454bb2f83d3cf70be43023a4de) --- spartan/environments/network-defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spartan/environments/network-defaults.yml b/spartan/environments/network-defaults.yml index de5b1a7bb592..6b6bb4bf127b 100644 --- a/spartan/environments/network-defaults.yml +++ b/spartan/environments/network-defaults.yml @@ -365,7 +365,7 @@ networks: ENABLE_AUTO_SHUTDOWN: false # Slasher penalties - more lenient initially SLASH_DATA_WITHHOLDING_PENALTY: 0 - SLASH_INACTIVITY_TARGET_PERCENTAGE: 0.8 + SLASH_INACTIVITY_TARGET_PERCENTAGE: 0.7 SLASH_INACTIVITY_CONSECUTIVE_EPOCH_THRESHOLD: 2 SLASH_INACTIVITY_PENALTY: 2000e18 SLASH_PROPOSE_INVALID_ATTESTATIONS_PENALTY: 2000e18 From f4a3e63b8949d03addaa96b09651647fa120b978 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 8 Jul 2026 06:57:24 -0300 Subject: [PATCH 03/35] docs(e2e): update READMEs for the consolidated suite layout (#24518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates every README in the e2e test section to describe the suite as it will look once the nine open round-2 consolidation PRs land. This is docs-only — no code, config, or `bootstrap.sh` changes. ## What changed per README - **`end-to-end/README.md`** (root): repointed the `LOG_LEVEL` example off the deleted `proving/empty_blocks.test.ts` to `proving/default_node.test.ts`; linked the new p2p README from the category table (the p2p row now reads "yes" and points at `src/p2p/README.md`); and renamed the `setupBlockProducer` submission-window literal `1024` to the named constant `NO_REORG_SUBMISSION_EPOCHS`. - **`end-to-end/src/single-node/README.md`**: rewrote the `cross-chain/` row for the merged layout (`CrossChainMessagingTest` + `message_test_helpers.ts`, and the `l1_to_l2` / `l1_to_l2_inbox_drift` / `l2_to_l1` / `token_bridge` files); folded `public_payments`/`sponsored_payments` into `private_payments.parallel` and fixed the stale `failures.parallel` name to `failures`; replaced `slasher_config`/`sequencer_config` with the merged `runtime_config`; replaced `world_state_pruning`/`empty_blocks` with the merged `default_node`; added the previously-missing `snapshot_sync` to the `sync/` row; and applied the `NO_REORG_SUBMISSION_EPOCHS` rename. - **`end-to-end/src/multi-node/README.md`**: named `equivocation_offenses` in the slashing row (in place of the separate `duplicate_proposal`/`duplicate_attestation` files) and applied the `NO_REORG_SUBMISSION_EPOCHS` rename in the `MOCK_GOSSIP_MULTI_VALIDATOR_OPTS` preset. - **`end-to-end/src/p2p/README.md`** (new): created with the content added by #24500. - **`end-to-end/src/automine/README.md`** and **`end-to-end/src/infra/README.md`**: untouched — both are behavior-level (no file names) or cover a directory none of the nine PRs change, so they remain accurate as-landed. --------- Co-authored-by: AztecBot (cherry picked from commit 1cc5f91856dba05994eb98e0a3d092048b2cf84c) --- yarn-project/end-to-end/README.md | 52 ++++++++++++------- .../end-to-end/src/multi-node/README.md | 3 +- .../end-to-end/src/single-node/README.md | 24 +++++---- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/yarn-project/end-to-end/README.md b/yarn-project/end-to-end/README.md index 085fac046e10..5b97d49fbb5f 100644 --- a/yarn-project/end-to-end/README.md +++ b/yarn-project/end-to-end/README.md @@ -14,15 +14,21 @@ Run a single test (spawns its own in-process anvil): ```bash yarn test:e2e src/automine/token/access_control.parallel.test.ts -yarn test:e2e src/single-node/block-building/block_building.test.ts -t 'rejects double spend' +yarn test:e2e src/single-node/block-building/block_building.test.ts -t 'rejects a private then private double-spend' ``` Turn up logging with `LOG_LEVEL` (`verbose` is the useful default; `debug:sequencer,archiver` scopes it): ```bash -LOG_LEVEL=verbose yarn test:e2e src/single-node/proving/empty_blocks.test.ts +LOG_LEVEL=verbose yarn test:e2e src/single-node/proving/default_node.test.ts ``` +Each run spawns anvil on port 8545, so two tests can only run side by side if each gets its own +`ANVIL_PORT` (p2p tests additionally bind fixed p2p ports and can never run concurrently — see +[`src/p2p/README.md`](src/p2p/README.md)). To shake flakiness out of a test, +`scripts/deflaker.sh yarn test:e2e ` reruns it up to 100 times and stops at the first failure +(output lands in `scripts/deflaker.log`). + Compose-based tests (those under `src/composed/`) need a running local network — see [Compose / HA / web3signer tests](#compose--ha--web3signer-tests). @@ -37,7 +43,7 @@ its environment, so a test file only describes the scenario, not the wiring. | [`automine/`](src/automine/README.md) | One node, deterministic `AutomineSequencer` — one block per tx, no committee/prover/validator. Fast. | it exercises contract or protocol behavior that doesn't depend on real block-building or consensus (transfers, nested calls, note discovery, tx semantics). | yes | | [`single-node/`](src/single-node/README.md) | One node, production sequencer (interval block production), optional prover. | it asserts on sequencer, proving, partial-proof, L1-reorg, recovery, fee, or cross-chain behavior on a single sequencer. | yes | | [`multi-node/`](src/multi-node/README.md) | N validators on an in-memory mock-gossip bus. | it needs a committee: consensus, attestations, slashing, governance, or multi-validator block production. | yes | -| `p2p/` | Real libp2p transport between nodes. | the networking transport itself is under test (gossip, rediscovery, req/resp). | — | +| [`p2p/`](src/p2p/README.md) | Real libp2p transport between nodes. | the networking transport itself is under test (gossip, rediscovery, req/resp). | yes | | [`infra/`](src/infra/README.md) | Targets a deployed/external network (local anvil or a public testnet). | its concern is deployment or network targeting, not a specific protocol behavior. | yes | A handful of tests live **outside** this package, next to the code they test — see @@ -78,7 +84,7 @@ Each category centralizes its environment in a base class. The hierarchy: (both wrap `fixtures/setup.ts:setup()`), but fixes the automine topology and makes `AUTOMINE_E2E_OPTS` the default. Exposes `markProvenAndWarp`, `registerContract`, `applyManualParentChild`. - `p2p/p2p_network.ts` → **`P2PNetworkTest`**. Real libp2p; node creation goes through - `setup_p2p_test.ts`. + `fixtures/setup_p2p_test.ts`. - `infra/` has no shared base — its tests target a network selected by `L1_CHAIN_ID` (local anvil in CI, a public testnet with credentials). @@ -91,8 +97,8 @@ Categories expose thin factories over their base's static `setup`, named by what calls the factory instead of spreading option presets: - `single-node/setup.ts`: `setupWithProver` (fake in-process prover — the single-node default) and - `setupBlockProducer` (no prover; raises `aztecProofSubmissionEpochs` to `1024` so unproven blocks - aren't pruned, and points the PXE at `syncChainTip: 'proposed'`). + `setupBlockProducer` (no prover; raises `aztecProofSubmissionEpochs` to `NO_REORG_SUBMISSION_EPOCHS` + (1024) so unproven blocks aren't pruned, and points the PXE at `syncChainTip: 'proposed'`). - `automine` tests call `AutomineTestContext.setup({ numberOfAccounts })` directly. ### The harness pattern (domain setup on top of a category) @@ -136,15 +142,20 @@ CI splits each `it` in a `.parallel.test.ts` file into its own docker job, runni ### CI test discovery — `bootstrap.sh` -`end-to-end/bootstrap.sh` enumerates tests in two arrays, and a test must appear in the relevant one or it -**won't run in CI**: +`end-to-end/bootstrap.sh` enumerates tests in two arrays, and a test must resolve through the relevant one +or it **won't run in CI**: -- `test_cmds` (~line 37) — the standard run. -- `compat_test_cmds` (~line 290) — the forward/legacy-compat run (a subset). +- `test_cmds` — the standard run. Covers each category with a recursive glob (e.g. + `src/automine/!(simulation)/**/*.test.ts`, `src/multi-node/**/*.test.ts`), so a new file or sub-folder + inside an existing category is picked up automatically; only a new top-level category needs its own glob + line. Tests with bespoke handling sit outside the globs: the `single-node/prover/` lanes at the top of + the function (real proofs and custom resources under `CI_FULL`, `FAKE_PROOFS=1` otherwise) and + `avm_simulator` (below). +- `compat_test_cmds` — the forward/legacy-compat run (a subset). This one enumerates **single-level leaf + globs** (e.g. `src/automine/token/*.test.ts`), so a new sub-folder whose tests should run against legacy + contract artifacts needs its own line here. -Each leaf folder needs its own single-level glob line (e.g. `src/automine/token/*.test.ts`) in each array; -globs are not recursive, so every sub-folder is listed explicitly. Folders that organize by behavior get -one line per leaf. Bespoke handling to be aware of: +Bespoke handling to be aware of: - **`avm_simulator`** (`automine/simulation/avm_simulator.test.ts`) has a dedicated line in `test_cmds` that sets `DUMP_AVM_INPUTS_TO_DIR` (feeds the downstream `avm_check_circuit` job) and is therefore @@ -153,8 +164,8 @@ one line per leaf. Bespoke handling to be aware of: - **`kernelless_simulation`** is excluded from `compat_test_cmds` only. After editing the arrays, confirm every `*.test.ts` resolves through exactly one line (no duplicate, no -omission). Per-test bash `TIMEOUT` overrides live in the `case` block in `test_cmds` and must stay in sync -with the test's `jest.setTimeout`. +omission — anything excluded via `!(...)` must be matched by its dedicated line). Per-test bash `TIMEOUT` +overrides live in the `case` block in `test_cmds` and must stay in sync with the test's `jest.setTimeout`. ### Flaky tests — `.test_patterns.yml` @@ -190,11 +201,12 @@ These run in their own package's test lane (both packages already run anvil-back ### Support directories (not test categories) -- `fixtures/` — the shared `setup()`, option presets (`fixtures.ts`), `CrossChainTestHarness`, - `l1_to_l2_messaging`, and common utils. -- `shared/` — shared test bodies and `timing_env.mjs`, a **custom jest `testEnvironment`** referenced from - this package's `package.json`. `yarn prepare` / the package-json check will try to revert it to the - default — don't let it. +- `fixtures/` — the shared `setup()`, option presets (`fixtures.ts`), the named node-level waiters + (`wait_helpers.ts`), the span instrumentation (`timing.ts` — `testSpan`, zero-cost unless + `TEST_TIMING_FILE` is set), `l1_to_l2_messaging`, and common utils. +- `shared/` — shared test bodies, the `CrossChainTestHarness`, and `timing_env.mjs`, a **custom jest + `testEnvironment`** referenced from this package's `package.json`. `yarn prepare` / the package-json + check will try to revert it to the default — don't let it. - `simulators/` — in-TS reference models (`TokenSimulator`, `LendingSimulator`) used to assert contract behavior. - `test-wallet/`, `bench/`, `spartan/`, `quality_of_service/`, `forward-compatibility/` — helpers, diff --git a/yarn-project/end-to-end/src/multi-node/README.md b/yarn-project/end-to-end/src/multi-node/README.md index 9751a9856240..522079aa206f 100644 --- a/yarn-project/end-to-end/src/multi-node/README.md +++ b/yarn-project/end-to-end/src/multi-node/README.md @@ -31,7 +31,8 @@ copy-pasted: `getPrivateKeyFromIndex(i + 3)`), passed as `initialValidators`. - `MOCK_GOSSIP_MULTI_VALIDATOR_OPTS` — a tight committee on the mock bus with no prover (`{ mockGossipSubNetwork, skipInitialSequencer, startProverNode: false, aztecProofSubmissionEpochs: - 1024, numberOfAccounts: 0 }`). Tests that want a prover leave `startProverNode` explicit. + NO_REORG_SUBMISSION_EPOCHS, numberOfAccounts: 0 }`, the constant re-exported from `../single-node/setup.ts`). + Tests that want a prover leave `startProverNode` explicit. - `SLASHER_ENABLED_MULTI_VALIDATOR_OPTS` — the same committee with the slasher turned on, used by the offense-detection tests. - `defaultSlashingPenalties(unit?)` / `withOnlyOffense(offense, unit?)` — build the per-offense diff --git a/yarn-project/end-to-end/src/single-node/README.md b/yarn-project/end-to-end/src/single-node/README.md index 0c4eca43beeb..16b4f7b7b212 100644 --- a/yarn-project/end-to-end/src/single-node/README.md +++ b/yarn-project/end-to-end/src/single-node/README.md @@ -19,6 +19,11 @@ All tests use `SingleNodeTestContext` (`single_node_test_context.ts`), which own `MultiNodeTestContext` (in `../multi-node/`) extends this base with the N-validator topology, so the multi-node category inherits the same environment and waiters. +Prefer these named waiters — and the node-level ones in `../fixtures/wait_helpers.ts` — over hand-rolled +`retryUntil` / `.on` / `sleep` polling in test bodies; the full catalog, including the helpers this base +class defines, is listed under [Helper surface](../multi-node/README.md#helper-surface) in the multi-node +README. + ## Setup factories `setup.ts` holds thin factories over `SingleNodeTestContext.setup`, named by the prover mode a test @@ -27,9 +32,10 @@ wants. Tests call the factory rather than the static method directly: - `setupWithProver(opts)` — a single sequencer plus the context's fake in-process prover node. This is the default the proving / partial-proofs / l1-reorgs / recovery / misc suites use. - `setupBlockProducer(opts)` — a single production sequencer with **no prover node**, used by the - block-building / sequencer / sync suites. It raises `aztecProofSubmissionEpochs` to `1024` so unproven - blocks are not pruned without a prover, and points the PXE at `syncChainTip: 'proposed'` so tests can - assert on freshly proposed blocks. Both are overridable via `opts`. + block-building / sequencer / sync suites. It raises `aztecProofSubmissionEpochs` to + `NO_REORG_SUBMISSION_EPOCHS` (1024) so unproven blocks are not pruned without a prover, and points the + PXE at `syncChainTip: 'proposed'` so tests can assert on freshly proposed blocks. Both are overridable + via `opts`. The `prover/` suite (real Barretenberg proofs) builds its environment through `FullProverTest`, which extends `SingleNodeTestContext` directly rather than going through a factory. @@ -47,13 +53,13 @@ top-level `it`; CI splits each `it` into its own job. | Path | Contents | |---|---| | `block-building/` | Block assembly mechanics under the production sequencer with pipelining. `block_building` (multi-tx blocks, double-spend rejection, log ordering, regressions, and L1 reorgs), `debug_trace` (blocks proposed through a Forwarder proxy, including a failing-then-succeeding propose call), `multiple_blobs` (a block whose combined side effects span more than one EIP-4844 blob). | -| `sequencer/` | Sequencer configuration, governance signalling, and publisher management on a single node. `gov_proposal.parallel` (a 16-validator committee proposes blocks while casting governance votes, and votes even when block building is disabled), `escape_hatch_vote_only` (governance signals advance while the escape hatch is closed), `reload_keystore` (the keystore is hot-reloaded to add a validator and pick up new coinbases), `slasher_config` (slasher config updated at runtime via the admin API), `multi_eoa` (publisher rotation when an L1 tx is withheld; exercises multi-EOA publisher failover), `publisher_funding_multi` (PublisherManager auto top-up of publisher EOAs when balances drop below threshold), `sequencer_config` (runtime `maxL2BlockGas`/`manaTarget` reconfiguration via a live Bot). | -| `fees/` | Fee mechanics on a single node. `fee_asset_price_oracle` (on-chain fee-asset price-oracle convergence; starts its own Anvil instance with a MockStateView etched at the StateView address). The remaining files run on the `FeesTest` harness (`fees_test.ts`), which extends `SingleNodeTestContext` with the fee/gas domain setup (FPC funding, fee-juice bridging, banana token): `account_init` (fee payment during account-contract initialization), `failures.parallel` (fees still charged when txs revert in setup/app/teardown), `fee_juice_payments` (direct Fee Juice payment with and without initial funds), `fee_settings` (max-fee-per-gas handling, stale-fee-snapshot race, governance fee-config bump), `gas_estimation.parallel` (gas-estimation accuracy and FPC teardown gas prediction), `private_payments.parallel` (private fee payment via the BananaCoin FPC), `public_payments` (public fee payment via the BananaCoin FPC), `sponsored_payments` (sponsored fee payment via SponsoredFPC). | -| `cross-chain/` | L1↔L2 messaging on a single node, on the `CrossChainMessagingTest` harness (`cross_chain_messaging_test.ts`), which extends `SingleNodeTestContext` and owns a `CrossChainTestHarness` plus the L1 inbox/outbox handles (it auto-proves via an `EpochTestSettler` when no prover node runs). `l1_to_l2.parallel` (L1→L2 message readiness, duplicate messages, and inbox drift after a rollup reorg), `l2_to_l1.parallel` (L2→L1 message inclusion across single/multi-message txs, subtree-root balancing, and a reorg-and-remine case), `token_bridge_private.parallel` (private L1→L2 deposit and L2→L1 withdrawal via the TokenBridge), `token_bridge_public.parallel` (public L1→L2 deposit and L2→L1 withdrawal, including mint-on-behalf), `token_bridge_failure_cases.parallel` (rejected withdrawals without approval and mismatched public/private claim attempts). | +| `sequencer/` | Sequencer configuration, governance signalling, and publisher management on a single node. `gov_proposal.parallel` (a 16-validator committee proposes blocks while casting governance votes, and votes even when block building is disabled), `escape_hatch_vote_only` (governance signals advance while the escape hatch is closed), `reload_keystore` (the keystore is hot-reloaded to add a validator and pick up new coinbases), `multi_eoa` (publisher rotation when an L1 tx is withheld; exercises multi-EOA publisher failover), `publisher_funding_multi` (PublisherManager auto top-up of publisher EOAs when balances drop below threshold), `runtime_config` (the former `slasher_config` and `sequencer_config` admin-API checks on one node: slasher inactivity-config getters, plus runtime `maxL2BlockGas`/`manaTarget` reconfiguration enforced via a live Bot). | +| `fees/` | Fee mechanics on a single node. `fee_asset_price_oracle` (on-chain fee-asset price-oracle convergence; starts its own Anvil instance with a MockStateView etched at the StateView address). The remaining files run on the `FeesTest` harness (`fees_test.ts`), which extends `SingleNodeTestContext` with the fee/gas domain setup (FPC funding, fee-juice bridging, banana token): `account_init` (fee payment during account-contract initialization), `failures` (fees still charged when txs revert in setup/app/teardown), `fee_juice_payments` (direct Fee Juice payment with and without initial funds), `fee_settings` (max-fee-per-gas handling, stale-fee-snapshot race, governance fee-config bump), `gas_estimation.parallel` (gas-estimation accuracy and FPC teardown gas prediction), `private_payments.parallel` (private fee payment via the BananaCoin FPC, plus the single-assertion public-FPC and sponsored-FPC payment checks folded onto the same fixture from the former `public_payments`/`sponsored_payments` files). | +| `cross-chain/` | L1↔L2 messaging on a single node, on the `CrossChainMessagingTest` harness (`cross_chain_messaging_test.ts`), which extends `SingleNodeTestContext` and owns a `CrossChainTestHarness` plus the L1 inbox/outbox handles (it auto-proves via an `EpochTestSettler` when no prover node runs). The shared L1→L2 message helpers live in `message_test_helpers.ts` (`createL1ToL2MessageHelpers`, whose injected `markAsProven` lets each suite plug in its own proving policy). `l1_to_l2` (L1→L2 message readiness and duplicate messages, over private and public scope), `l1_to_l2_inbox_drift` (inbox checkpoint drift after a rollup reorg, over private and public scope), `l2_to_l1` (L2→L1 message inclusion across single/multi-message txs and multi-block checkpoints, subtree-root balancing, and a reorg-and-remine case), `token_bridge` (private and public L1→L2 deposits and L2→L1 withdrawals via the TokenBridge — including mint-on-behalf — plus the withdrawal/claim failure cases, merged from the former three `token_bridge_*` files). | | `bot/` | Transaction bot implementations. `bot` (transfer bot, AMM bot, and cross-chain bot; exercises fee-juice portal deposits, L2→L1 messages, and bot contract reuse). | -| `sync/` | World-state sync stress and reorg-replay harness. `synching` builds fixture block data (env-gated, slow) and replays it for sync benchmarks and prune/reorg scenarios; only the outer `it.each` runs in CI. | -| `proving/` | Epoch and proof lifecycle. `world_state_pruning` (consecutive epochs prove and finalized blocks are purged from world state beyond the checkpoint-history window), `empty_blocks` (a proof is submitted even with no txs), `long_proving_time` (a prover delay spanning multiple epochs), `multi_proof` (multiple prover nodes prove one epoch), `optimistic.parallel` (checkpoint-driven proving across the happy path and several mid-epoch / last-slot / during-proving reorg cases), `proof_fails.parallel` (proof not accepted after epoch end; proving aborts when the next epoch ends), `cross_chain_public_message` (an epoch with a public tx that consumes an L1→L2 message in the block it lands, guarding against a sequencer/prover state-root mismatch), `upload_failed_proof` (a failed proving job's state is uploaded and re-run on a fresh instance). | -| `prover/` | Real-proof exercises on the `FullProverTest` harness (real Barretenberg when `FAKE_PROOFS=0`, fake otherwise). `client` (client-side proof generation and `verifyProof` for private and public transfers, no on-chain submission) and `full` (the end-to-end pipeline: client proves, node builds blocks, prover node generates epoch proofs, L1 verifies them). | +| `sync/` | World-state sync stress and snapshot sync. `synching` builds fixture block data (env-gated, slow) and replays it for sync benchmarks and prune/reorg scenarios (only the outer `it.each` runs in CI); `snapshot_sync` exercises the node snapshot upload/download path, syncing fresh nodes from one or multiple snapshot URLs, including fallback from a corrupted snapshot. | +| `proving/` | Epoch and proof lifecycle. `default_node` (basic proving/node coverage that shares one context-default node across three suites: consecutive epochs prove and finalized blocks are purged from world state beyond the checkpoint-history window, a proof is submitted even with no txs, and the node returns initial genesis-block data — the former `world_state_pruning`, `empty_blocks`, and `node_block_api` files), `long_proving_time` (a prover delay spanning multiple epochs), `multi_proof` (multiple prover nodes prove one epoch), `optimistic.parallel` (checkpoint-driven proving across the happy path and several mid-epoch / last-slot / during-proving reorg cases), `proof_fails.parallel` (proof not accepted after epoch end; proving aborts when the next epoch ends), `cross_chain_public_message` (an epoch with a public tx that consumes an L1→L2 message in the block it lands, guarding against a sequencer/prover state-root mismatch), `upload_failed_proof` (a failed proving job's state is uploaded and re-run on a fresh instance). | +| `prover/` | Real-proof exercises on the `FullProverTest` harness (real Barretenberg when `FAKE_PROOFS=0`, fake otherwise), split by where the proving runs so CI can size their containers independently: `client/client` (client-side proof generation and `verifyProof` for private and public transfers, no on-chain submission) and `server/full` (the end-to-end pipeline: client proves, node builds blocks, prover node generates epoch proofs, L1 verifies them). | | `partial-proofs/` | Manually driven partial-proof submission. `single_root` (the prover node's `startProof` path on a single root) and `multi_root` (three partial-proof roots are staged and messages consume against any covering root, exercising the multi-root Outbox semantics). | | `l1-reorgs/` | Behavior under L1 reorgs, split by what reorgs. `blocks.parallel` (prune L2 blocks when a reorg drops a proof, hold when a replacement proof lands in the window, restore blocks when a proof reappears, prune pending-chain blocks, and see new blocks added by a reorg) and `messages.parallel` (L1→L2 messages updated by a reorg, and a missed message inserted by one). `setup.ts` holds the shared `FAST_REORG_TIMING` profile and delayer wiring. | | `recovery/` | Reorg and pending-chain recovery. `manual_rollback` (the `rollbackTo` admin API rolls back to an unfinalized block), `sync_after_reorg` (a fresh node syncs world state past an unpruned reorg window), `prune_when_cannot_build` (a solo sequencer prunes the pending chain via the fallback path when it cannot propose). | From febbec5a1dc52b876e0414821008ac77868692ba Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 8 Jul 2026 06:59:00 -0300 Subject: [PATCH 04/35] test(e2e): instrument and diagnose bot suite setup cost (#24534) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What / why Round-4 e2e speedup, PR 3 of the effort. The `single-node/bot` suite's file-level `beforeAll` costs ~178s on CI with 97% untagged, and the residual proven-chain waits in the `private_payments`/`failures` fees suites (~32s/shard) were also invisible. This PR instruments both so the cost is attributable, and diagnoses the bot residual. Spans are pure passthrough when `TEST_TIMING_FILE` is unset (see `fixtures/timing.ts`), so this is zero behavior change for normal runs and CI. ## Commit - `test(e2e): instrument bot suite setup and fees proven-chain residuals` - Bot suite (`single-node/bot/bot.test.ts`): `setup:wallet` around `EmbeddedWallet.create`, `wallet:create` around `createSchnorrInitializerlessAccount`, and `setup:bot` around the four nested `describe` beforeAll hooks (`Bot.create` / `AmmBot.create` / `CrossChainBot.create` / `BotStore`). - Fees harness (`single-node/fees`): `wait:proven-checkpoint` around `catchUpProvenChain`'s poll loop, and `warp:proven-checkpoint-epoch` around `advanceToNextEpoch` via a new instrumented `FeesTest.advanceToNextEpoch()` wrapper; `waitForEpochProven` and the direct call sites in `failures`/`private_payments` now route through it. - Reuses existing tags where the concept matches (`wallet:create` is already what `createFundedInitializerlessAccounts` fires; `warp:proven-checkpoint-epoch` is already what `advanceToNextEpoch` fires in `block-production/setup.ts`) rather than minting near-duplicates. ## Diagnosis The timing environment (`shared/timing_env.mjs`) folds *every* `beforeAll` in a file — the file-level one plus each nested `describe`'s — into a single suite-scoped `beforeHooksMs`, and every span fired during any beforeAll lands on that same suite line. The shortlist read the whole number as the file-level hook and, seeing only `setup:env:*` tagged, attributed the rest to the one visible untagged file-level call (`createSchnorrInitializerlessAccount`). It is actually the nested hooks. Local read (`TEST_TIMING_SPANS=1`, `PIPELINING_SETUP_OPTS` = 12s L2 slots, production sequencer, fake prover), suite `beforeHooksMs = 153,733 ms`: - `setup:env:*` (file-level `setup()`): ~4.6s — already tagged - `setup:wallet` (`EmbeddedWallet.create`, ephemeral): **137 ms** — the PXE has `autoSync:false`, no genesis walk - `wallet:create` (`createSchnorrInitializerlessAccount`): **69 ms** — initializerless, no deploy tx, as designed - `setup:bot` (4 nested hooks): **149,582 ms = 97.3%** of the file's beforeAll - transaction-bot `Bot.create`: 32.2s (token deploy class+instance, then private+public mint) - bridge-resume `new BotStore(...)`: 0.9 ms - amm-bot `AmmBot.create`: 73.9s (deploys 4 contracts + set_minter + mint + add_liquidity) - cross-chain-bot `CrossChainBot.create`: 43.5s (TestContract deploy + seed 2 L1→L2 msgs + wait ready) **Root cause: structural, not a bug.** The bot suite runs the production Sequencer to exercise CHECKPOINTED/PROPOSED follow modes and L1 fee-juice bridging, so each `Bot.create` deploys real contracts and mints serially at slot cadence (`bot/src/factory.ts`). The two calls the shortlist suspected are ~0.1s each. No fix is shipped here because there is no safe one-liner: speeding this up means batching/parallelizing the `bot/src/factory.ts` deploy/mint paths (production code, PR-6-shaped), genesis-seeding (PR 1/2), or cutting the CI slot time (PR 5) — none belong in the instrumentation PR. Tracked as a round-5 item; the new `setup:bot` span makes that win measurable. Full write-up in `tmp/SPEEDUP_ROUND4_FINDINGS.md`. ## Expected effect No wall-clock change (passthrough spans). The deliverable is visibility: the previously-invisible `setup:bot` / `setup:wallet` / `wallet:create` decomposition of the bot beforeAll, and the `wait:proven-checkpoint` / `warp:proven-checkpoint-epoch` residual in the fees suites. Measured numbers from a full green CI run will be appended below once available. ## Measured impact Both runs are full CI runs. PR run: `ci/x-fast` on head `1f7898cd2b` (CI 1783345614459429, 1,799 test lines). Baseline: `ci/x-full-no-test-cache` on the exact merge-base `a8cfa93df5` (CI 1783341726388943, 1,863 test lines). No wall-clock change claimed (spans are passthrough; suite-level noise floor is ~11s median / ~52s p90) — the deliverable is attribution, measured as span `busyMs`. Bot suite (`suite` line, beforeAll = 181.5s on the PR run vs 182.1s baseline): - Baseline: 7.4s tagged (`setup:env:*` only) → **174.8s / 96% invisible**. - PR run: `setup:bot` **174,203 ms** (4 occurrences, max 86,961 ms = the amm-bot hook), `setup:wallet` **250 ms**, `wallet:create` **144 ms**; residual now **−1.5s ≈ 0** → **100% of the beforeAll is attributed**. - This confirms the diagnosis: `EmbeddedWallet.create` + `createSchnorrInitializerlessAccount` are ~0.4s combined; the "invisible 178s" is the four nested describes' bot-factory setup (deploys + mints at production 12s-slot cadence), folded into the file's suite line by the hook accounting. Fees suites (`private_payments.parallel` × 8 shards + `failures`): - Baseline: avg **32.3s/shard untagged residual** (zero `wait:proven-checkpoint` / `warp:proven-checkpoint-epoch` lines on these suites). - PR run: `wait:proven-checkpoint` fires on all 9 suite hooks, **293,888 ms total** (avg 32.7s/shard — matches the residual exactly), plus one in-test occurrence (32.1s); `warp:proven-checkpoint-epoch` 19 occurrences, 361 ms total (the warp RPC itself is near-free — the cost is entirely the proven-chain poll that follows). Per-shard residual is now ≈ −1s, i.e. fully attributed. Net: **~469s/run of previously invisible setup/wait cost is now tagged and attributable** (174.2s `setup:bot` + 0.4s wallet spans + 294.3s fees proven-chain waits), with no behavior or wall-clock change. Fixes A-1407 (cherry picked from commit 14761799c50c4a2302e68eccf020f0b5ced1e1ee) --- .../src/single-node/bot/bot.test.ts | 25 +++++++++++-------- .../src/single-node/fees/failures.test.ts | 8 +++--- .../src/single-node/fees/fees_test.ts | 17 +++++++++---- .../fees/private_payments.parallel.test.ts | 2 +- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/yarn-project/end-to-end/src/single-node/bot/bot.test.ts b/yarn-project/end-to-end/src/single-node/bot/bot.test.ts index 3a73306a5c8a..75a54e5b92f2 100644 --- a/yarn-project/end-to-end/src/single-node/bot/bot.test.ts +++ b/yarn-project/end-to-end/src/single-node/bot/bot.test.ts @@ -22,6 +22,7 @@ import { EmbeddedWallet } from '@aztec/wallets/embedded'; import { jest } from '@jest/globals'; import { PIPELINED_FEE_PADDING, PIPELINING_SETUP_OPTS } from '../../fixtures/fixtures.js'; +import { testSpan } from '../../fixtures/timing.js'; import { getPrivateKeyFromIndex, setup } from '../../fixtures/utils.js'; import { NO_REORG_SUBMISSION_EPOCHS } from '../setup.js'; @@ -54,8 +55,10 @@ describe('single-node/bot/bot', () => { cheatCodes, config: { l1RpcUrls }, } = setupResult); - wallet = await EmbeddedWallet.create(aztecNode, { ephemeral: true }); - await wallet.createSchnorrInitializerlessAccount(botAccount.secret, botAccount.salt, botAccount.signingKey); + wallet = await testSpan('setup:wallet', () => EmbeddedWallet.create(aztecNode, { ephemeral: true })); + await testSpan('wallet:create', () => + wallet.createSchnorrInitializerlessAccount(botAccount.secret, botAccount.salt, botAccount.signingKey), + ); }); afterAll(() => teardown()); @@ -73,7 +76,9 @@ describe('single-node/bot/bot', () => { botMode: 'transfer', minFeePadding: PIPELINED_FEE_PADDING, }; - bot = await Bot.create(config, wallet, aztecNode, undefined, new BotStore(await openTmpStore('bot'))); + bot = await testSpan('setup:bot', async () => + Bot.create(config, wallet, aztecNode, undefined, new BotStore(await openTmpStore('bot'))), + ); }); // Runs bot.run() once and asserts recipient private and public balances each increase by 1. @@ -131,7 +136,7 @@ describe('single-node/bot/bot', () => { let store: BotStore; beforeAll(async () => { - store = new BotStore(await openTmpStore('bot')); + store = await testSpan('setup:bot', async () => new BotStore(await openTmpStore('bot'))); }); afterAll(async () => { @@ -238,7 +243,9 @@ describe('single-node/bot/bot', () => { followChain: 'CHECKPOINTED', botMode: 'amm', }; - bot = await AmmBot.create(config, wallet, aztecNode, undefined, new BotStore(await openTmpStore('bot'))); + bot = await testSpan('setup:bot', async () => + AmmBot.create(config, wallet, aztecNode, undefined, new BotStore(await openTmpStore('bot'))), + ); }); // Runs the AMM bot once and asserts one of the two private token balances decreased and @@ -302,12 +309,8 @@ describe('single-node/bot/bot', () => { flushSetupTransactions: true, l1ToL2SeedCount: 2, }; - bot = await CrossChainBot.create( - config, - wallet, - aztecNode, - aztecNodeAdmin, - new BotStore(await openTmpStore('bot')), + bot = await testSpan('setup:bot', async () => + CrossChainBot.create(config, wallet, aztecNode, aztecNodeAdmin, new BotStore(await openTmpStore('bot'))), ); }, 600_000); diff --git a/yarn-project/end-to-end/src/single-node/fees/failures.test.ts b/yarn-project/end-to-end/src/single-node/fees/failures.test.ts index 234b84c54f44..44a5124cb0f5 100644 --- a/yarn-project/end-to-end/src/single-node/fees/failures.test.ts +++ b/yarn-project/end-to-end/src/single-node/fees/failures.test.ts @@ -57,7 +57,7 @@ describe('single-node/fees/failures', () => { aztecNode = t.aztecNode; // Prove up until the current state by advancing the epoch and waiting for the prover node. - await t.cheatCodes.rollup.advanceToNextEpoch(); + await t.advanceToNextEpoch(); await t.catchUpProvenChain(); }); @@ -123,7 +123,7 @@ describe('single-node/fees/failures', () => { // @note There is a potential race condition here if other tests send transactions that get into the same // epoch and thereby pays out fees at the same time (when proven). - await t.cheatCodes.rollup.advanceToNextEpoch(); + await t.advanceToNextEpoch(); const provenTimeout = (t.context.config.aztecProofSubmissionEpochs + 1) * t.context.config.aztecEpochDuration * @@ -362,7 +362,7 @@ describe('single-node/fees/failures', () => { ); // Prove the block containing the teardown-reverted tx (revert_code = 2). - await t.cheatCodes.rollup.advanceToNextEpoch(); + await t.advanceToNextEpoch(); const provenTimeout = (t.context.config.aztecProofSubmissionEpochs + 1) * t.context.config.aztecEpochDuration * @@ -389,7 +389,7 @@ describe('single-node/fees/failures', () => { expect(receipt.executionResult).toBe(TxExecutionResult.REVERTED); expect(receipt.transactionFee).toBeGreaterThan(0n); - await t.cheatCodes.rollup.advanceToNextEpoch(); + await t.advanceToNextEpoch(); const provenTimeout = (t.context.config.aztecProofSubmissionEpochs + 1) * t.context.config.aztecEpochDuration * diff --git a/yarn-project/end-to-end/src/single-node/fees/fees_test.ts b/yarn-project/end-to-end/src/single-node/fees/fees_test.ts index 0cc1d3a38a88..cfb353025bcd 100644 --- a/yarn-project/end-to-end/src/single-node/fees/fees_test.ts +++ b/yarn-project/end-to-end/src/single-node/fees/fees_test.ts @@ -131,15 +131,22 @@ export class FeesTest extends SingleNodeTestContext { } async catchUpProvenChain() { - const bn = await this.aztecNode.getBlockNumber(); - while ((await this.aztecNode.getBlockNumber('proven')) < bn) { - await sleep(1000); - } + await testSpan('wait:proven-checkpoint', async () => { + const bn = await this.aztecNode.getBlockNumber(); + while ((await this.aztecNode.getBlockNumber('proven')) < bn) { + await sleep(1000); + } + }); + } + + /** Warps L1 to the next epoch boundary so the current epoch closes and can be proven. */ + async advanceToNextEpoch() { + await testSpan('warp:proven-checkpoint-epoch', () => this.cheatCodes.rollup.advanceToNextEpoch()); } /** Advances to the next epoch and waits for the proven chain to catch up, so all prior fees are paid out. */ async waitForEpochProven() { - await this.cheatCodes.rollup.advanceToNextEpoch(); + await this.advanceToNextEpoch(); await this.catchUpProvenChain(); } diff --git a/yarn-project/end-to-end/src/single-node/fees/private_payments.parallel.test.ts b/yarn-project/end-to-end/src/single-node/fees/private_payments.parallel.test.ts index ad155ec4a467..77b51cfbb7ee 100644 --- a/yarn-project/end-to-end/src/single-node/fees/private_payments.parallel.test.ts +++ b/yarn-project/end-to-end/src/single-node/fees/private_payments.parallel.test.ts @@ -140,7 +140,7 @@ describe('single-node/fees/private_payments', () => { const provenCheckpointBefore = await t.rollupContract.getProvenCheckpointNumber(); const receipt = await localTx.send({ timeout: 300, interval: 10 }); - await t.cheatCodes.rollup.advanceToNextEpoch(); + await t.advanceToNextEpoch(); await waitForProven(aztecNode, receipt, { provenTimeout: 300 }); From d64784be2d5cfeb7efe1ca2f76da9b4da5337781 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 8 Jul 2026 07:00:09 -0300 Subject: [PATCH 05/35] perf(e2e): warp dead waits in multi-node recovery and proving tests (#24566) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 4 of the round-4 e2e speedup effort: extend the merged `warpWithSequencersPaused` primitive (from #24475, proven on `pipeline_prune`) to the remaining production-sequencer-bound dead waits. One commit per converted site so a CI-hostile one reverts in isolation. On the fresh `merge-train/spartan-v5` base, most of the sites the plan targeted have already been warp-optimized (they use `warpToBuildWindowForSlot` / `warpToEpochStart` / `waitUntilNextEpochStarts`, and `pipeline_prune` already uses `warpWithSequencersPaused`). The residual waits in the remaining sites ride real-time consensus/production that a warp cannot safely collapse — warping would skip the checkpoints/attestations/offenses the tests are verifying. After reading each file on the fresh base, only one genuinely-dead wall-clock wait was left to convert; the rest are honest skips (per the round-3 skip rule: an honest skip beats a flaky convert). ## Converted (one commit) - **`multi-node/recovery/proposal_failure_recovery` — missed-L1-publish test.** Between "proposed chain reached slotTwo" and the prune, the test waited in wall-clock (`waitUntilL1Timestamp`) for the L1 clock to roll past slotOne so the archiver prunes the uncheckpointed slotOne/slotTwo blocks. The pipelined slotTwo broadcast has already reached every node and slotThree does not build until slotTwo, so nothing must be produced in that gap — a genuine dead wait, the direct `pipeline_prune` analog. Replaced with `warpWithSequencersPaused(..., { restart: false })`: pause across the warp (warping under a running sequencer would interrupt in-flight builds), keep the sequencers stopped until the prune is confirmed so no proposer builds on the unpruned tip, then restart them for recovery. Expected saving: up to ~1 L2 slot (aztecSlot=36s on this suite's cadence) per run of that test. ## Skipped, with reasons - **`proposal_failure_recovery` — orphan-prune test (second `it`).** The prune wait overlaps P2's concurrent S2 checkpoint rebuild inside the same slot (pipelining builds S2 during S1 while the orphan is pruned). Pausing the sequencers to warp would stop the S2 rebuild the test then asserts on. No pausable dead gap. - **`multi-node/recovery/equivocation_recovery`.** The dominant waits are the heal phase (`waitUntilCheckpointNumber` / `waitForAllNodesToReachCheckpoint` for baseline+2) which requires B/C to actually produce two checkpoints after A is stopped, plus `waitForOffenseOnNodes` which requires real slashing-round detection. Warping skips the very production being verified. - **`multi-node/slashing/slash_veto_demo`.** `waitForSubmittableRound` accrues inactivity offenses to quorum over proven blocks, and the veto/expiry logic rides real slashing-round consensus. This is proof-submission/consensus real time (the prover must actually run) — warp does not help, same as the fees `catchUpProvenChain` finding from round 3. - **`multi-node/slashing/attested_invalid_proposal`** (now under `slashing/`, not `invalid-attestations/` — base drift). Epoch advances already use cheatcode warps (`advanceToEpoch` / `advanceToNextEpoch`); the remaining waits are real-time committee/attestation/offense-detection rounds (bad proposer must build and broadcast, lazy validator must attest, honest node must detect the offense). - **`multi-node/block-production/multi_validator_node`.** Cost is real-time attestation + checkpoint building via an actual deploy tx; the only clock jump (`advanceToEpoch` past the validator-set lag) is already a cheatcode warp, and this test was just flake-fixed (#24543) so it is left untouched. - **`single-node/proving/optimistic.parallel` (`wait:epoch` tails, pool 2).** Already warp-optimized on this base: every epoch-boundary skip uses `warpToEpochStart` / `waitUntilNextEpochStarts` / `warpToBuildWindowForSlot`. The residual `wait:epoch` is the deliberate ~2-slot real-time tail (the epoch's final checkpoint must land and the sequencer must settle) — pausing across it would suppress that checkpoint. The `waitUntilCheckpointNumber(midCheckpoint)` waits require the sequencer to produce two mid-epoch checkpoints, and `waitUntilProvenCheckpointNumber(..., 240)` rides real prover latency. Nothing left to safely convert. - **`wait:tx-mined` sites (pool 3).** These are inclusion waits for in-flight builds, not waits for a future slot, so they do not qualify (as anticipated in the plan). Not converted. ## Local verification - `yarn build` (full TS project): passes. - `yarn format end-to-end` / `yarn lint end-to-end`: clean. - The converted test is a 4-validator mock-gossip pipelining suite, too heavy to run on the dev machine, so it was not executed locally. CI (`ci-no-fail-fast`) is the verifier; per-commit isolation contains any flake. ## Measured impact The conversion landed as scoped: `warp:sequencers-paused` goes from 1 occurrence run-wide (pipeline_prune, pre-existing from #24475) to 2 — the new one on the converted `proposal_failure_recovery` test. - Converted test ("prune and recover when proposer fails to publish to L1"): total 130.8s → 122.5s (−8.3s), body 114.6s → 109.2s (−5.4s). The added `warp:sequencers-paused` span is 36.1s. - The net saving is modest: the pause+warp+drain+deferred-restart cycle itself costs ~36s, only ~8s less than the real-time prune-boundary wait it replaced — not a full slot. The clean attribution is the span count (1→2); the ~8s test-total delta sits near the multi-node noise floor. - Untouched sites are flat within noise: the sibling orphan-prune test 93.7s → 91.5s, and pipeline_prune 260.4s → 258.7s. This is consistent with the PR's own finding that the warp lever is largely exhausted on this base — one convertible site, a single-test gain. Baseline CI run 1783374468714213 (merge-base e1711d3efa, full). PR CI run 1783387686257121 (x-fast). ## Notes for the rest of round 4 - No new shared helper was added — the converted site reuses the existing `warpWithSequencersPaused` on `SingleNodeTestContext` (inherited by `MultiNodeTestContext`). PRs 1a/1b/6/5/7 have nothing new to reuse from here. - Finding relevant to the round's premise: on this drifted base the multi-node recovery/slashing suites' large `wait:checkpoint`/`wait:epoch` pools are dominated by real production (checkpoint building, slashing rounds, prover latency), not dead clock time. The warp lever is largely exhausted there; PR 5 (shrink slot times) is the remaining cadence lever for those suites. Fixes A-1183 (cherry picked from commit cfb6c7f87738c533a8559d36b0f7f5d55c38a2ae) --- ...proposal_failure_recovery.parallel.test.ts | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts b/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts index 35387b9d38b8..9f959f6123a3 100644 --- a/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts +++ b/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts @@ -2,7 +2,6 @@ import type { Archiver } from '@aztec/archiver'; import type { AztecNodeService } from '@aztec/aztec-node'; import { EthAddress } from '@aztec/aztec.js/addresses'; import type { Logger } from '@aztec/aztec.js/log'; -import { waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils'; import { asyncMap } from '@aztec/foundation/async-map'; import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { retryUntil } from '@aztec/foundation/retry'; @@ -188,11 +187,16 @@ describe('multi-node/recovery/proposal_failure_recovery', () => { logger.warn(`Waiting for proposed chain to reach slot ${slotTwo} on all nodes (build during slotOne)`); await test.waitForAllNodesToReachBlockAtSlot(slotTwo, 'proposed', undefined, { timeout: slotAdvanceTimeout }); - // (3) Wait until slotOne has fully ended on L1 — the archiver only prunes once slotAtNextL1Block > slotOne. - // The end-of-slotOne timestamp equals the start-of-slotTwo timestamp. + // (3) Collapse the dead gap where the chain just waits for the L1 clock to roll past slotOne so the + // archiver prunes the uncheckpointed slotOne/slotTwo blocks (it only prunes once slotAtNextL1Block > + // slotOne, and the end-of-slotOne timestamp equals the start-of-slotTwo timestamp). The pipelined slotTwo + // broadcast has already reached every node (step 2) and slotThree does not build until slotTwo, so nothing + // needs to be produced in this window. The sequencers are paused across the warp — warping under a running + // sequencer would interrupt in-flight builds — and kept stopped (restart: false) until the prune is + // confirmed, so no proposer builds against the still-unpruned tip; they are restarted for recovery below. const slotOneEndTimestamp = getTimestampForSlot(slotTwo, test.constants); - logger.warn(`Waiting until L1 timestamp ${slotOneEndTimestamp} (end of slot ${slotOne})`); - await waitUntilL1Timestamp(test.l1Client, slotOneEndTimestamp, undefined, test.L2_SLOT_DURATION_IN_S * 3); + logger.warn(`Warping past the end of slot ${slotOne} (L1 timestamp ${slotOneEndTimestamp}) to trigger the prune`); + await test.warpWithSequencersPaused(nodes, test.context.cheatCodes, slotOneEndTimestamp, { restart: false }); // (4) After slotOne ends without a checkpoint, all nodes should prune. // Verify rollback via the prune event itself: the pruned slot must equal slotOne, and the @@ -216,9 +220,13 @@ describe('multi-node/recovery/proposal_failure_recovery', () => { expect(prunedSlots).toContain(slotOne); } - // (5) Allow the formerly suppressed node to publish again so the chain can recover. + // (5) Allow the formerly suppressed node to publish again, then restart the paused sequencers so the + // chain can build the recovery checkpoint. Restarting only now (after the prune is confirmed) keeps any + // proposer from building on the still-unpruned tip. logger.warn(`Re-enabling checkpoint publishing on node ${proposerOneNodeIndex}`); await nodes[proposerOneNodeIndex].setConfig({ skipPublishingCheckpointsPercent: 0 }); + await test.startSequencers(nodes); + logger.warn('Restarted all sequencers for recovery'); // (6) During slotTwo: the pipelined proposer for slotThree builds and broadcasts → proposed advances again. // The chain must have rewound past slotOne and slotTwo and now build on whatever was From 47379133db57a35258b89f2c555829a782c06bf4 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 8 Jul 2026 07:03:36 -0300 Subject: [PATCH 06/35] perf(e2e): shrink e2e slot times (#24570) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What PR 5 of the round-4 e2e speedup effort: shrink e2e slot times, the highest-leverage remaining lever now that the warp lever is exhausted (PR 4) and the fees setup costs are pure inclusion latency at CI cadence (PR 6). Every cadence-bound cost — setup txs, inclusion waits, epoch walks — scales linearly with what these commits cut. Config-only. Three commits, in **ascending risk order** so the CI-survey/fix phase can revert from the top. A fourth candidate (WIDE_SLOT 72s → 48s) was investigated and **dropped** — see below. All guard math is recomputed from `stdlib/src/timetable/budgets.ts` + `stdlib/src/timetable/proposer_timetable.ts`: maxBlocksPerCheckpoint = floor((S - init - D - 2P - prepCp) / D) must be >= 1 where `S` = aztecSlotDuration, `init` = checkpointProposalInitTime (1s, never clamped), `D` = blockDurationMs/1000, `P` = p2pPropagationTime, `prepCp` = checkpointProposalPrepareTime. Fast-profile clamping (`P -> 0.5`, `prepCp -> 0.5`, `minBlock -> 1`) applies only when `ethereumSlotDuration < 8` (`FAST_PROFILE_ETHEREUM_SLOT_DURATION`). At or above 8s the production budgets apply verbatim. ## Commit 1 (lowest risk) — `perf(e2e): cut default CI L1 block time to 8s` `DEFAULT_L1_BLOCK_TIME`: `process.env.CI ? 12 : 8` → `8`. Local dev already ran at 8, so this only removes a CI-vs-local cadence asymmetry. Default single-node L2 slot goes 24s → 16s. Guard (default single-node: D = DEFAULT_BLOCK_DURATION_MS/1000 = 3, production budgets P=2, prepCp=1, init=1): - before (eth 12): S = 2x12 = 24 → floor((24 - 1 - 3 - 4 - 1)/3) = floor(15/3) = **5** blocks/cp - after (eth 8): S = 2x8 = 16 → floor((16 - 1 - 3 - 4 - 1)/3) = floor(7/3) = **2** blocks/cp Both >= 1. eth=8 is AT the fast-profile boundary, so budgets stay at production values (no clamping). **Blast radius is much smaller than the round-4 plan assumed.** The plan expected this to cut the fees / cross-chain / bot families, but the base has drifted: those suites now run on `PIPELINING_SETUP_OPTS` / `AUTOMINE_E2E_OPTS` (both set `ethereumSlotDuration: 4` explicitly) and no longer read `DEFAULT_L1_BLOCK_TIME`. The genuine default-cadence consumers (eth from `SingleNodeTestContext.getSlotDurations`, no explicit eth) are four single-node proving/recovery suites: `proving/default_node`, `proving/cross_chain_public_message`, `recovery/sync_after_reorg`, `partial-proofs/multi_root`. The only other `process.env.CI` timing coupling (`multi-node/slashing/inactivity_setup.ts`, `process.env.CI ? 8 : 4`) sets its own eth slot and does not read `DEFAULT_L1_BLOCK_TIME`, so it is untouched. ## Commit 2 (medium risk) — `perf(e2e): cut reorg cadence to 24s slots` `REORG_TIMING_BASE`: aztecSlotDuration 36 → 24, blockDurationMs 8000 → 5000 (epoch stays 4 slots). Shared by both reorg profiles, both below the fast-profile boundary: - `FAST_REORG_TIMING` (eth 4): proving/optimistic reorg describes, l1-reorgs/ - `MULTI_VALIDATOR_REORG_TIMING` (eth 6, attestationPropagationTime 0.5): recovery/proposal_failure_recovery, recovery/equivocation_recovery, high-availability/ha_sync, high-availability/ha_checkpoint_handoff Both run fast-profile: P = 0.5, prepCp = 0.5, init = 1. Guard, FAST_REORG_TIMING and MULTI_VALIDATOR_REORG_TIMING (identical budgets): - before: S = 36, D = 8 → floor((36 - 1 - 8 - 1 - 0.5)/8) = floor(25.5/8) = **3** blocks/cp - after: S = 24, D = 5 → floor((24 - 1 - 5 - 1 - 0.5)/5) = floor(16.5/5) = **3** blocks/cp blockDurationMs drops 8000 → 5000 specifically to preserve 3 blocks/checkpoint. Leaving it at 8000 would give floor(13.5/8) = **1**, which would break the l1-reorgs suites' `assertMultipleBlocksPerSlot(2)` (they push TX_COUNT=8 with maxTxsPerBlock=1 to force multi-block checkpoints). All other timing scales off `test.constants.slotDuration` (waits, warps). equivocation_recovery's manual fit calc still holds (3 x 5s = 15 + 0.5 init + 5 final block + 2.5 finalization = 23s <= 24s); its comment is updated. ## Commit 3 (higher risk) — `perf(e2e): cut multi-validator block-production cadence to 24s slots` `MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING`: aztecSlotDurationInL1Slots 3 → 2 (36s → 24s at eth 12), blockDurationMs 6000 → 4000. eth stays 12 (production budgets). Consumers: block-production/simple, first_slot, proof_boundary. Guard (eth 12, production budgets prepCp=1, init=1; P = per-test attestationPropagationTime), S=24, D=4: - simple (P=default 2): floor((24 - 1 - 4 - 4 - 1)/4) = floor(14/4) = **3** (was 4 at 36/6) - proof_boundary (P=default 2): floor((24 - 1 - 4 - 4 - 1)/4) = **3** (was 4) - first_slot (P=0.5): floor((24 - 1 - 4 - 1 - 1)/4) = floor(17/4) = **4** (was 4) All >= 1. simple asserts no block-count (only no-sequencer-failures); proof_boundary asserts proof-vs-boundary timing that scales off `test.constants`; first_slot keeps 4 blocks/checkpoint (its comment stays accurate). So the drop 4 → 3 is harmless for those three. `high_tps` is **pinned to the old 36s/6s cadence at its call site** (`aztecSlotDurationInL1Slots: 3`, `blockDurationMs: 6000` in its setupOpts, overriding the shared profile). Its checkpoint packing is 2 txs x 2.5s = 5s per block, which needs a 6s block sub-slot; a 4s block cannot hold it, and the suite hard-asserts `max-checkpoint-length == 4` and `max-txs-per-block == 2`. Pinning keeps high_tps at its tuned cadence rather than rewriting its whole timing model. The other three suites still take the cut. ## Dropped candidate — WIDE_SLOT 72s → 48s Investigated per the plan's stretch item and **dropped**. `blob_promotion` hard-asserts a checkpoint with >= `PIPELINE_EXPECTED_BLOCKS_PER_CHECKPOINT = 8` blocks. At the profile's D = 5.5s: - current 72/12: floor((72 - 1 - 5.5 - 4 - 1)/5.5) = floor(60.5/5.5) = **11** blocks/cp (fits 8) - 48/12: floor((48 - 1 - 5.5 - 4 - 1)/5.5) = floor(36.5/5.5) = **6** blocks/cp (cannot fit 8) Forcing 8 blocks would require shrinking D to ~4.5s, which lands the guard at exactly 8 with zero margin — under the deliberate `mockGossipSubNetworkLatency: 500ms` the suite runs with, and on top of the profile's documented A-914 constraint (pipelined multiple-blocks-per-slot starves non-proposer nodes with `CheckpointNumberNotSequentialError` when the L2 slot is tightened, worsened by a shorter slot). This is the "provably can't fit" case, and it cannot be verified locally (heavy multi-node pipelining), so it is not shipped. proposed_chain / cross_chain_messages (only need 2 blocks/cp) would survive the guard but share the A-914 risk; not worth the exposure for one profile. ## Local verification - **Commit 1: verified.** Ran `single-node/proving/default_node.test.ts -t 'returns initial block data'` locally (eth=8 = the new CI cadence, which local already uses). The node came up at `ethereumSlotDuration: 8, aztecSlotDuration: 16, blockDurationMs: 3000` and logged `Sequencer timetable initialized with 2 blocks per slot {"maxNumberOfBlocks":2}` — exactly the guard value above. No "Invalid timing configuration", no missed-slot warnings, test passed. Budgets stayed at production values (`attestationPropagationTime: 2, minBlockDuration: 2`), confirming eth=8 does not trip fast-profile clamping. - **Commits 2 and 3: guard math + CI only.** The reorg and multi-validator block-production suites are too heavy for the dev machine (multi-node + real-time waits + proving). The guard math above is the verification; CI validates end-to-end. The formula was confirmed exact against commit 1's live run. ## Revert protocol for the fix phase Commits are ordered by ascending risk; revert individual commits from the top on flake rather than fighting them: 1. **Any block-production suite flake** (simple / first_slot / proof_boundary) → revert commit 3 first. proof_boundary is the most real-time-sensitive (it warps to N-3 then runs the boundary in real time at the tighter 24s cadence); watch it first. 2. **Any reorg / prune / HA flake** (l1-reorgs, proving/optimistic reorg cases, recovery/proposal_failure_recovery, recovery/equivocation_recovery, high-availability/*) → revert commit 2. Watch the l1-reorgs `assertMultipleBlocksPerSlot(2)` assertions and equivocation_recovery's slashing-round timing. 3. **Any of the four default-cadence single-node proving/recovery suites** → revert commit 1 (lowest probability; the cadence is what local already runs). ## Notes for PR 7 (bot suite cadence) The bot suite (`single-node/bot/bot.test.ts`) uses `setup(0, { ...PIPELINING_SETUP_OPTS, ... })`, which sets `ethereumSlotDuration: 4` explicitly. It does not read `DEFAULT_L1_BLOCK_TIME`, so **commit 1 does not affect the bot suite** — its cadence lever remains its own PIPELINING preset, unchanged here. ## Measured impact The cadence cuts land a run-wide −6.4%: −1,600s summed across shards over the 135 suites present in both runs (−2,046s of improvements against +446s of regressions). The wall-clock benefit is smaller, since shards run in parallel across workers. Controls confirm the deltas are real cadence effects, not run-type artifacts. `high_tps` (deliberately pinned at the old cadence) is unchanged (+0.1s), and the WIDE_SLOT / explicit-preset proving suites are flat — pipeline_prune −1.2s, blob_promotion −0.1s, long_proving_time −1.1s, prune_when_cannot_build −0.1s. Only the default-cadence and reorg/block-production timings moved. Largest per-suite wall cuts (default-cadence proving/recovery from commit 1; reorg from commit 2): - optimistic.parallel −399.9s (−23.5%, ×8 shards) - proof_boundary.parallel −346.0s (−25.6%, ×5 shards) - full −202.7s; blocks.parallel −196.3s (−28.8%, ×5 shards) - invalidate_block.parallel −118.5s (×11 shards); sync_after_reorg −108.3s - data_withholding_slash −72.0s; proposal_failure_recovery.parallel −60.1s; cross_chain_public_message −48.2s; equivocation_recovery −46.9s; late_prover_tx_collection −43.3s Regressions, all on high-variance multi-node suites that are not cadence-cut here (run-to-run noise, not attributable to this change): multi_root +74.1s, inactivity_slash +64.4s, attested_invalid_proposal.parallel +32.1s, proposed_chain.parallel +26.1s, bot +22.3s. The metric is whole-suite wall time (sum of test durations + hooks); a cadence change carries no new span to attribute against, so these are run-vs-run wall deltas — the flat pinned/preset controls above are what make the cadence attribution defensible. Baseline CI run 1783417125211175 (merge-base 13a53f11502b, full). PR CI run 1783426164237985 (x-fast). Fixes A-1184 (cherry picked from commit eedf4ebca8aafe937754ab270b7b04726060e954) --- .../block-production/high_tps.test.ts | 5 +++ .../proof_boundary.parallel.test.ts | 4 +- .../block-production/simple.test.ts | 2 +- .../ha_checkpoint_handoff.test.ts | 4 +- .../recovery/equivocation_recovery.test.ts | 8 ++-- ...proposal_failure_recovery.parallel.test.ts | 2 +- .../src/single-node/l1-reorgs/setup.ts | 6 +-- .../proving/optimistic.parallel.test.ts | 6 +-- .../single-node/single_node_test_context.ts | 41 ++++++++++++------- 9 files changed, 48 insertions(+), 30 deletions(-) diff --git a/yarn-project/end-to-end/src/multi-node/block-production/high_tps.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/high_tps.test.ts index 4c4c9611a6bf..37b5a1f36d8f 100644 --- a/yarn-project/end-to-end/src/multi-node/block-production/high_tps.test.ts +++ b/yarn-project/end-to-end/src/multi-node/block-production/high_tps.test.ts @@ -76,6 +76,11 @@ describe('multi-node/block-production/high_tps', () => { ({ test, context, logger, validators, nodes, from } = await setupSimpleBlockProduction({ nodeCount: NODE_COUNT, setupOpts: { + // Pin the old 36s/6s cadence (overriding MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING's 24s/4s): this + // suite's per-block budget is 2 txs x 2.5s = 5s, which needs a 6s block sub-slot (the full T=0..36s + // budget in this file's header is built around it) and does not fit the profile's 4s block. + aztecSlotDurationInL1Slots: 3, + blockDurationMs: 6000, fakeProcessingDelayPerTxMs: TX_DURATION_MS, attestationPropagationTime: 1, minTxsPerBlock: 1, diff --git a/yarn-project/end-to-end/src/multi-node/block-production/proof_boundary.parallel.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/proof_boundary.parallel.test.ts index a48bdff9d1e7..94487bef9f4c 100644 --- a/yarn-project/end-to-end/src/multi-node/block-production/proof_boundary.parallel.test.ts +++ b/yarn-project/end-to-end/src/multi-node/block-production/proof_boundary.parallel.test.ts @@ -28,8 +28,8 @@ type PublishedEvent = Parameters[0]; // Suite: 5 parallel scenarios testing the interaction between the proof submission deadline and // the pipelining boundary slot. MultiNodeTestContext: 3 validator nodes + 1 prover node, -// mockGossipSubNetwork, skipInitialSequencer. Timing: ethSlot=12s, aztecSlot=3×12=36s, -// epoch=default 6, proofSubmissionEpochs=1 (overridden per test via setupTest), blockDurationMs=6s, +// mockGossipSubNetwork, skipInitialSequencer. Timing: ethSlot=12s, aztecSlot=2×12=24s, +// epoch=default 6, proofSubmissionEpochs=1 (overridden per test via setupTest), blockDurationMs=4s, // inboxLag=2 (v5 always enforces the timetable, so the former enforceTimeTable/disableAnvilTestWatcher // overrides are gone). The Delayer is used to steer proof tx timing. describe('multi-node/block-production/proof_boundary', () => { diff --git a/yarn-project/end-to-end/src/multi-node/block-production/simple.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/simple.test.ts index 2f21f3ce519b..3ad7f4bcfe66 100644 --- a/yarn-project/end-to-end/src/multi-node/block-production/simple.test.ts +++ b/yarn-project/end-to-end/src/multi-node/block-production/simple.test.ts @@ -16,7 +16,7 @@ const TX_COUNT_SIMPLE = 8; // Verifies that 3 validator nodes can build blocks without sequencer errors. Lightweight RPC-only // initial node (skipInitialSequencer), mockGossipSubNetwork, no prover. Timing: ethSlot=12s, -// aztecSlot=36s, epoch=default 6, proofSubmissionEpochs=1024, blockDurationMs=6s. Pre-proved txs sent +// aztecSlot=24s, epoch=default 6, proofSubmissionEpochs=1024, blockDurationMs=4s. Pre-proved txs sent // from the hardcoded genesis-funded account (no on-chain account deploy needed). describe('multi-node/block-production/simple', () => { let context: EndToEndContext; diff --git a/yarn-project/end-to-end/src/multi-node/high-availability/ha_checkpoint_handoff.test.ts b/yarn-project/end-to-end/src/multi-node/high-availability/ha_checkpoint_handoff.test.ts index 91fc8e5ac6a5..2a42d2bc0bb6 100644 --- a/yarn-project/end-to-end/src/multi-node/high-availability/ha_checkpoint_handoff.test.ts +++ b/yarn-project/end-to-end/src/multi-node/high-availability/ha_checkpoint_handoff.test.ts @@ -57,8 +57,8 @@ const VALIDATOR_COUNT = 4; * `mockGossipSubNetwork` bus, then 4 validator nodes created via `test.createValidatorNode` in 2 HA pairs. Each pair * shares its two validator keys plus an in-memory `createSharedSlashingProtectionDb` (so only one peer signs per duty) * — explicitly NOT the Postgres-backed docker-compose HA suite, so this is an in-proc `multi-node` test, not infra. - * Production `Sequencer`, no prover node. Timing: ethSlot=6s, aztecSlot=36s, epoch=4, proofSubEpochs=1024, - * blockDurationMs=8s, committeeSize=4, attestationPropagationTime=0.5, inboxLag=2; anvil on interval mining. Nodes build + * Production `Sequencer`, no prover node. Timing: ethSlot=6s, aztecSlot=24s, epoch=4, proofSubEpochs=1024, + * blockDurationMs=5s, committeeSize=4, attestationPropagationTime=0.5, inboxLag=2; anvil on interval mining. Nodes build * empty checkpoints (`buildCheckpointIfEmpty` + `minTxsPerBlock: 0`) so no txs are needed, and each node uses a distinct * coinbase so the secondary assertion can prove which peer produced S2. Time is warped with `cheatCodes.eth.warp`: * `findConsecutiveSamePairSlots` recovers from `ValidatorSelection__EpochNotStable` by warping forward one epoch, and diff --git a/yarn-project/end-to-end/src/multi-node/recovery/equivocation_recovery.test.ts b/yarn-project/end-to-end/src/multi-node/recovery/equivocation_recovery.test.ts index 61fac1f986b9..ab293a3286c5 100644 --- a/yarn-project/end-to-end/src/multi-node/recovery/equivocation_recovery.test.ts +++ b/yarn-project/end-to-end/src/multi-node/recovery/equivocation_recovery.test.ts @@ -57,12 +57,12 @@ describe('multi-node/recovery/equivocation_recovery', () => { // Build 4 validators (V1..V4) using the shared deterministic builder (keys from index 3). const validators = buildMockGossipValidators(NODE_COUNT); - // Timing calculation for 3 blocks per checkpoint with 8s sub-slots: + // Timing calculation for 3 blocks per checkpoint with 5s sub-slots: // - initializationOffset = 0.5s (test mode, ethereumSlotDuration < 8) - // - 3 blocks x 8s = 24s + // - 3 blocks x 5s = 15s // - checkpointFinalization = 0.5s (assemble) + 0 (p2p in test) + 2s (L1 publish) = 2.5s - // - finalBlockDuration = 8s (re-execution) - // - Total: 0.5 + 24 + 8 + 2.5 = 35s => use 36s + // - finalBlockDuration = 5s (re-execution) + // - Total: 0.5 + 15 + 5 + 2.5 = 23s => use 24s const slashingUnit = BigInt(1e14); test = await MultiNodeTestContext.setup({ ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, diff --git a/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts b/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts index 9f959f6123a3..16867863a69f 100644 --- a/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts +++ b/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts @@ -29,7 +29,7 @@ const NODE_COUNT = 4; * blocks, and the next proposer rebuilds a fresh checkpoint that lands on L1. * * Both scenarios share the same 4-validator mock-gossip cluster (one key per node, no prover) on the - * multi-validator reorg cadence (ethSlot=6s, aztecSlot=36s, epoch=4, proofSubmissionEpochs=1024, blockDurationMs=8000, + * multi-validator reorg cadence (ethSlot=6s, aztecSlot=24s, epoch=4, proofSubmissionEpochs=1024, blockDurationMs=5000, * inboxLag=2 — v5 always enforces the timetable). Each test warps L1 to align with its target build slot. */ describe('multi-node/recovery/proposal_failure_recovery', () => { diff --git a/yarn-project/end-to-end/src/single-node/l1-reorgs/setup.ts b/yarn-project/end-to-end/src/single-node/l1-reorgs/setup.ts index c8f0737a6a8f..57a9ea36a398 100644 --- a/yarn-project/end-to-end/src/single-node/l1-reorgs/setup.ts +++ b/yarn-project/end-to-end/src/single-node/l1-reorgs/setup.ts @@ -24,7 +24,7 @@ export const TX_COUNT = 8; /** * The single-node + prover-node fixture shared by the L1-reorg suites (`blocks`, `messages`). Stands * up a {@link SingleNodeTestContext} on the {@link FAST_REORG_TIMING} cadence (ethSlot=4s, - * aztecSlot=36s, block=8s, epoch=4, 32 slots/epoch) with L1 speed-ups disabled so prover/sequencer txs + * aztecSlot=24s, block=5s, epoch=4, 32 slots/epoch) with L1 speed-ups disabled so prover/sequencer txs * can be held back and reorged, registers a {@link TestContract}, and exposes the per-test handles plus * a `sendTransactions` helper that pre-proves and fires `count` lightweight txs to drive multi-block * checkpoints. Reorgs themselves are driven by `EthCheatCodes.reorg`/`reorgWithReplacement` at the call @@ -47,14 +47,14 @@ export class L1ReorgsTest { public async setup(): Promise { this.test = await setupWithProver({ - ...FAST_REORG_TIMING, // ethSlot=4s, aztecSlot=36s, block=8s, epoch=4, 32 slots/epoch (mainnet) + ...FAST_REORG_TIMING, // ethSlot=4s, aztecSlot=24s, block=5s, epoch=4, 32 slots/epoch (mainnet) numberOfAccounts: 1, maxSpeedUpAttempts: 0, // Do not speed up l1 txs, we dont want them to land cancelTxOnTimeout: false, minTxsPerBlock: 0, maxTxsPerBlock: 1, aztecProofSubmissionEpochs: 1, - // Pipelining + multi-blocks-per-slot: 8s blocks fit ~4 blocks per 36s slot, and TX_COUNT=8 + // Pipelining + multi-blocks-per-slot: 5s blocks fit ~3 blocks per 24s slot, and TX_COUNT=8 // ensures multiple checkpoints have multiple blocks }); ({ diff --git a/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts b/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts index 8a719cd3556c..a3983288f7c1 100644 --- a/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts @@ -24,9 +24,9 @@ jest.setTimeout(1000 * 60 * 20); * Setup: a single sequencer/validator node from `setupWithProver` plus the context's fake prover-node (no * `mockGossipSubNetwork`, so no gossip bus), making this a `single-node` test on the production `Sequencer`. Each of the * six `describe` blocks builds a fresh context in its own `beforeEach` and tears it down in the shared `afterEach`. The - * happy-path pair uses defaults (`numberOfAccounts: 1`; ethSlot=8s local/12s CI, aztecSlot=16s/24s, epoch=6, - * proofSubEpochs=1); the five reorg describes use a faster cadence (ethSlot=4s, aztecSlot=36s, epoch=4 — or 8 for the - * with-replacement case so the replacement lands in-epoch — proofSubEpochs=NO_REORG_SUBMISSION_EPOCHS, blockDurationMs=8s, minTxsPerBlock=0, + * happy-path pair uses defaults (`numberOfAccounts: 1`; ethSlot=8s, aztecSlot=16s, epoch=6, + * proofSubEpochs=1); the five reorg describes use a faster cadence (ethSlot=4s, aztecSlot=24s, epoch=4 — or 8 for the + * with-replacement case so the replacement lands in-epoch — proofSubEpochs=NO_REORG_SUBMISSION_EPOCHS, blockDurationMs=5s, minTxsPerBlock=0, * anvilSlotsInAnEpoch=32, maxSpeedUpAttempts=0, cancelTxOnTimeout=false). The `prover-node starts mid-epoch` describe * sets `startProverNode: false` and spins up the prover via `test.createProverNode()` partway through the epoch. * diff --git a/yarn-project/end-to-end/src/single-node/single_node_test_context.ts b/yarn-project/end-to-end/src/single-node/single_node_test_context.ts index 8a119b75d856..83360a4f2d61 100644 --- a/yarn-project/end-to-end/src/single-node/single_node_test_context.ts +++ b/yarn-project/end-to-end/src/single-node/single_node_test_context.ts @@ -60,7 +60,14 @@ import type { TestWallet } from '../test-wallet/test_wallet.js'; export const WORLD_STATE_CHECKPOINT_HISTORY = 2; export const WORLD_STATE_BLOCK_CHECK_INTERVAL = 50; export const ARCHIVER_POLL_INTERVAL = 50; -export const DEFAULT_L1_BLOCK_TIME = process.env.CI ? 12 : 8; +/** + * Default L1 (ethereum) slot duration in seconds for single-node e2e tests. Kept at 8s, the fast-profile + * boundary (`FAST_PROFILE_ETHEREUM_SLOT_DURATION`): at 8s the proposer still uses the production operational + * budgets (fast-profile clamping only kicks in strictly below 8s), so the default single-node L2 slot is + * `2 x 8 = 16s`. CI previously ran at 12s (24s L2 slots); unifying it with the local value removes a + * CI-vs-local cadence asymmetry and cuts every default-cadence single-node suite by a third. + */ +export const DEFAULT_L1_BLOCK_TIME = 8; export type SingleNodeTestOpts = Partial & { numberOfAccounts?: number; @@ -87,15 +94,18 @@ export type TrackedSequencerEvent = { export type BlockProposedEvent = { blockNumber: BlockNumber; slot: SlotNumber; buildSlot: SlotNumber }; /** - * The 36s-slot reorg cadence shared by every reorg/prune/HA test, regardless of single-node vs - * multi-validator topology: a 36s L2 slot, 8s blocks, and a 4-slot epoch. The two concrete reorg + * The 24s-slot reorg cadence shared by every reorg/prune/HA test, regardless of single-node vs + * multi-validator topology: a 24s L2 slot, 5s blocks, and a 4-slot epoch. The 5s block duration is chosen + * so the fast-profile budgets both reorg profiles run under (eth < 8s: p2p 0.5s, prepare 0.5s, init 1s) + * still fit ~3 full block sub-slots per checkpoint — `floor((24 - 1 - 5 - 2*0.5 - 0.5) / 5) = 3` — which + * the l1-reorgs suites' `assertMultipleBlocksPerSlot(2)` assertions require. The two concrete reorg * profiles ({@link FAST_REORG_TIMING}, {@link MULTI_VALIDATOR_REORG_TIMING}) extend this with their topology's L1 * slot duration and any extra knobs. Kept timing-only — `maxSpeedUpAttempts`, `cancelTxOnTimeout`, and * `aztecProofSubmissionEpochs` encode per-test scenario intent and stay explicit at the call site. */ export const REORG_TIMING_BASE = { - aztecSlotDuration: 36, - blockDurationMs: 8000, + aztecSlotDuration: 24, + blockDurationMs: 5000, aztecEpochDuration: 4, } as const; @@ -115,7 +125,7 @@ export const FAST_REORG_TIMING = { } as const; /** - * Timing-only profile naming the 36s/6s reorg-and-prune cadence copied verbatim across the + * Timing-only profile naming the 24s/6s reorg-and-prune cadence copied verbatim across the * multi-validator recovery and high-availability tests (`recovery/proposal_failure_recovery`, * `recovery/equivocation_recovery`, `high-availability/ha_sync`, * `high-availability/ha_checkpoint_handoff`). The multi-validator analogue of @@ -130,17 +140,20 @@ export const MULTI_VALIDATOR_REORG_TIMING = { } as const; /** - * Timing-only profile naming the 36s/12s multi-validator block-production cadence copied across - * `block-production/` (`simple`, `high_tps`, `first_slot`, and `proof_boundary`). Uses - * `aztecSlotDurationInL1Slots: 3` rather than an explicit `aztecSlotDuration: 36` so the L2 slot stays - * coupled to `ethereumSlotDuration` if a test overrides eth. Deliberately omits - * `attestationPropagationTime` (per-scenario: default 2, 0.5, or 1) — set it per test. Spread BEFORE - * per-test overrides. + * Timing-only profile naming the 24s/12s multi-validator block-production cadence copied across + * `block-production/` (`simple`, `first_slot`, and `proof_boundary`). Uses `aztecSlotDurationInL1Slots: 2` + * rather than an explicit `aztecSlotDuration: 24` so the L2 slot stays coupled to `ethereumSlotDuration` + * if a test overrides eth. The 4s block duration keeps enough full block sub-slots per checkpoint under + * the production budgets these eth=12 tests run with (init 1s, prepare 1s, min-block 2s, p2p = + * attestationPropagationTime): `floor((24 - 1 - 4 - 2P - 1) / 4)` = 4 blocks at P<=1, 3 blocks at P=2 + * (the default). Deliberately omits `attestationPropagationTime` (per-scenario: default 2, 0.5, or 1) — + * set it per test. `high_tps` pins the old 36s/6s cadence at its own call site because its 2-txs-x-2.5s + * per-block budget does not fit a 4s block. Spread BEFORE per-test overrides. */ export const MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING = { ethereumSlotDuration: 12, - aztecSlotDurationInL1Slots: 3, - blockDurationMs: 6000, + aztecSlotDurationInL1Slots: 2, + blockDurationMs: 4000, } as const; /** From 1db0e11e816287d76d99f074181bd370c14b178e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 8 Jul 2026 07:27:58 -0300 Subject: [PATCH 07/35] perf(e2e): seed BananaFPC fee juice at genesis instead of bridging (#24564) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What / why PR 2 of the round-4 e2e speedup effort. The fees-family suites pay a `setup:bridge` span (~48s ≈ 4 production slots per process) in `FeesTest.applyFPCSetup`, bridging fee juice from L1 to the BananaFPC via the gas portal (`bridgeFromL1ToL2` = prepare-on-L1 + advance 2 blocks + a claim tx). That funding is pure setup plumbing — the fees tests snapshot the FPC's gas balance and assert deltas, they don't assert on the bridge itself. Fee-juice balances are just public-data leaves at `computeFeePayerBalanceLeafSlot(address)`, which `getGenesisValues` (`world-state/src/testing.ts`) already prefills for genesis-funded accounts — the same mechanism that funds the initial accounts and (already, on this base) the sponsored FPC. This PR seeds the BananaFPC's fee juice at genesis instead of bridging it during setup. Note: the plan's premise that `fundSponsoredFPC: true` bridges was stale — the sponsored FPC has been genesis-funded since #19532 (setup pushes its deterministic address into the genesis-funded list). The remaining `setup:bridge` cost in the fees family is the BananaFPC bridge, which this PR removes. ## Mechanism - The BananaFPC address depends on the BananaCoin address and the FPC admin (the first setup account), and the accounts are generated *inside* `setup()` with random secrets — so the FPC address can't be precomputed by the test before genesis. Fixed deploy salts are added for BananaCoin and BananaFPC so both addresses become deterministic once the admin is known, and a `computeExtraGenesisFundedAddresses(defaultAccounts)` hook on `SetupOptions` runs after the accounts are generated and before genesis values are computed. `FeesTest` uses it to derive the BananaFPC address from the first account and add it to the genesis-funded list. - The returned address flows through the existing initial-accounts path (`addressesToFund` → `getGenesisValues`), so it is funded with the same fee juice as an initial account (`10^22`) and automatically included in the L1 `FeeJuicePortal` `feeJuicePortalInitialBalance` (`fundingNeeded`) accounting — the portal's locked L1 balance stays consistent with total L2 fee-juice supply, exactly as for every other genesis-funded balance. - `applyFPCSetup` deploys the FPC with the fixed salt and asserts the deployed address equals the seeded one, so any drift in the deterministic deploy params surfaces as a clear error instead of a downstream "insufficient fee payer balance". The `bridgeFromL1ToL2` call is removed. - The `setup:bridge` span wrapper is kept in place for the bridging that remains, so its disappearance from the fees setup hooks is the measured proof. ## Scope decisions (site by site) - **`FeesTest.applyFPCSetup` BananaFPC funding** → moved to genesis. Used by `account_init`, `private_payments.parallel`, `failures`, `gas_estimation.parallel`. This is the only funding that moved. - **`FeesTest.mintAndBridgeFeeJuice`** (bridges to an arbitrary recipient) → kept. It fires in `account_init` test bodies that test the bridge/claim flow ("pays natively in the Fee Juice after Alice bridges funds") — real behavior under test. - **`account_init` `FeeJuicePaymentMethodWithClaim` / `prepareTokensOnL1`** → kept. Tests the atomic claim-in-deploy flow. - **Sponsored FPC** → already genesis-funded via `fundSponsoredFPC: true` on this base; no change. - **`cross_chain_messaging_test`** (`fundSponsoredFPC: true`) → already genesis-funded; its own cross-chain bridging harness is the behavior under test, kept. - **`bench/client_flows_benchmark`** bridges to its BananaFPC → out of scope (benchmark harness, not in the fees e2e timing family), kept. - Production/sandbox funding paths (cli, `aztec sandbox`) → untouched. The new hook defaults to unset; only the e2e fixtures opt in. ## Expected effect ~48s × the number of fees processes that ran the FPC bridge in setup (account_init, private_payments, failures, gas_estimation). No C++, no new config, no change to production genesis roots. ## Local verification - `single-node/fees/account_init -t 'pays privately through an FPC'` passes; the span leaderboard for the run shows **zero `setup:bridge`** occurrences (was ~48s), while `deploy:fpc`/`deploy:token` remain. The FPC-paying test asserts the FPC's gas balance decreases by the fee, so the genesis funding is exercised end to end, and the deployed-address assertion held. - `single-node/fees/account_init -t 'after Alice bridges funds'` passes with `setup:bridge` present in the test body, confirming the retained bridge/claim harness still works. ## Measured impact The `setup:bridge` span is eliminated across the run: 17 occurrences totalling 646.4s of bridge setup time in the baseline drop to zero in the PR run. The 13 fees-hook bridges (~48s each) are the bulk of it. - Fees beforeHooks per shard: `private_payments.parallel` −51.1s (×8 shards), `gas_estimation.parallel` −48.1s (×3), `failures` −47.8s, `account_init` −47.8s — each matching the ~48s (4-slot) bridge cost. - `setup:bridge` busyMs removed per suite: account_init 48.1s, failures 48.2s, gas_estimation ×3 = 144.6s, private_payments ×8 = 384.7s (13 fees-hook bridges ≈ 625.5s), plus the ~5s sponsored-FPC bridges on `amm` and `storage_proof`. - Total beforeHooks reduction over the suites present in both runs: −907.4s summed across shards (wall-clock benefit is smaller, since the fees shards run in parallel). - One suite moved the wrong way within noise: `fee_juice_payments` +11.0s. It keeps its real bridging/claim txs (no `setup:bridge` span removed), so this is run-to-run variance, not a regression from this change. Baseline CI run 1783374468714213 (merge-base e1711d3efa, full). PR CI run 1783375878950264 (x-fast — the comparison is per-suite over the suites present in both runs). Fixes A-1405 (cherry picked from commit 1321e9820ce885adc85f602743a1ef997b11ee15) --- yarn-project/end-to-end/src/fixtures/setup.ts | 14 +++++ .../src/single-node/fees/fees_test.ts | 61 +++++++++++++++++-- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/yarn-project/end-to-end/src/fixtures/setup.ts b/yarn-project/end-to-end/src/fixtures/setup.ts index 3dc87c409dd2..6705ed463bf5 100644 --- a/yarn-project/end-to-end/src/fixtures/setup.ts +++ b/yarn-project/end-to-end/src/fixtures/setup.ts @@ -220,6 +220,14 @@ export type SetupOptions = { zkPassportArgs?: ZKPassportArgs; /** Whether to fund the sponsored FPC in genesis (defaults to false). */ fundSponsoredFPC?: boolean; + /** + * Compute extra addresses to fund at genesis from the accounts setup just generated (passed as the + * argument). Runs after the default accounts are created and before genesis values are computed, so a + * test can genesis-fund a contract whose address derives from a default account (e.g. an FPC whose + * admin is the first account) instead of bridging fee juice to it during setup. Each returned address + * is funded with the same fee juice as an initial account and included in the L1 portal `fundingNeeded`. + */ + computeExtraGenesisFundedAddresses?: (defaultAccounts: InitialAccountData[]) => Promise; /** L1 contracts deployment arguments. */ l1ContractsArgs?: Partial; /** Wallet minimum fee padding multiplier */ @@ -458,6 +466,12 @@ async function setupInner( const sponsoredFPCAddress = await getSponsoredFPCAddress(); addressesToFund.push(sponsoredFPCAddress); } + + // Fund any extra addresses whose value depends on the just-generated accounts (e.g. an FPC admin'd + // by a default account), so a test can genesis-fund them instead of bridging fee juice during setup. + if (opts.computeExtraGenesisFundedAddresses) { + addressesToFund.push(...(await opts.computeExtraGenesisFundedAddresses(defaultAccounts))); + } logger.trace('Generated test accounts to fund at genesis'); const genesisTimestamp = BigInt(Math.floor(Date.now() / 1000)); diff --git a/yarn-project/end-to-end/src/single-node/fees/fees_test.ts b/yarn-project/end-to-end/src/single-node/fees/fees_test.ts index cfb353025bcd..14efa5b5c234 100644 --- a/yarn-project/end-to-end/src/single-node/fees/fees_test.ts +++ b/yarn-project/end-to-end/src/single-node/fees/fees_test.ts @@ -1,4 +1,5 @@ import type { AztecAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; import { createLogger } from '@aztec/aztec.js/log'; import type { AztecNode } from '@aztec/aztec.js/node'; import { CheatCodes, getTokenAllowedSetupFunctions } from '@aztec/aztec/testing'; @@ -17,6 +18,7 @@ import { TokenContract as BananaCoin } from '@aztec/noir-contracts.js/Token'; import { CounterContract } from '@aztec/noir-test-contracts.js/Counter'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; import { getCanonicalFeeJuice } from '@aztec/protocol-contracts/fee-juice'; +import { getContractInstanceFromInstantiationParams } from '@aztec/stdlib/contract'; import { Gas, GasSettings } from '@aztec/stdlib/gas'; import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client'; @@ -34,13 +36,41 @@ import { import { TestWallet } from '../../test-wallet/test_wallet.js'; import { SingleNodeTestContext, type SingleNodeTestOpts } from '../single_node_test_context.js'; +// Fixed deploy salts so BananaCoin and its BananaFPC land at deterministic addresses given the FPC +// admin. This lets the BananaFPC's fee-juice balance be seeded at genesis (see `FeesTest.setup`) +// instead of bridged from L1 during setup — the address must be known before genesis is computed. +const BANANA_COIN_SALT = new Fr(0xba4a4a); +const BANANA_FPC_SALT = new Fr(0xfacade); + +const BANANA_COIN_CONSTRUCTOR_ARGS = ['BC', 'BC', 18n] as const; + +/** + * Computes the deterministic BananaCoin and BananaFPC instances for the given admin/deployer, matching + * what {@link FeesTest.applyDeployBananaToken} and {@link FeesTest.applyFPCSetup} deploy with the fixed + * salts above. Used both to seed the BananaFPC's fee juice at genesis and to assert the deployed + * addresses match the seeded one. + */ +async function computeBananaContractAddresses(admin: AztecAddress) { + const bananaCoin = await getContractInstanceFromInstantiationParams(BananaCoin.artifact, { + salt: BANANA_COIN_SALT, + constructorArgs: [admin, ...BANANA_COIN_CONSTRUCTOR_ARGS], + deployer: admin, + }); + const bananaFPC = await getContractInstanceFromInstantiationParams(FPCContract.artifact, { + salt: BANANA_FPC_SALT, + constructorArgs: [bananaCoin.address, admin], + deployer: admin, + }); + return { bananaCoinAddress: bananaCoin.address, bananaFPCAddress: bananaFPC.address }; +} + /** * Test fixture for testing fees. Provides the following setup steps: * InitialAccounts: Initializes 3 Schnorr account contracts. * PublicDeployAccounts: Deploys the accounts publicly. * DeployFeeJuice: Deploys the Fee Juice contract. - * FPCSetup: Deploys BananaCoin and FPC contracts, and bridges gas from L1. - * SponsoredFPCSetup: Deploys Sponsored FPC contract, and bridges gas from L1. + * FPCSetup: Deploys BananaCoin and FPC contracts; the FPC's fee juice is seeded at genesis. + * SponsoredFPCSetup: Registers the Sponsored FPC contract, whose fee juice is seeded at genesis. * FundAlice: Mints private and public bananas to Alice. * SetupSubscription: Deploys a counter contract and a subscription contract, and mints Fee Juice to the subscription contract. * @@ -116,6 +146,11 @@ export class FeesTest extends SingleNodeTestContext { ...this.setupOptions, ...opts, fundSponsoredFPC: true, + // Seed the BananaFPC's fee juice at genesis instead of bridging it from L1 in applyFPCSetup. The + // FPC admin is the first account, so its address is deterministic once the accounts are generated. + computeExtraGenesisFundedAddresses: async defaultAccounts => [ + (await computeBananaContractAddresses(defaultAccounts[0].address)).bananaFPCAddress, + ], l1ContractsArgs: { ...this.setupOptions }, txPublicSetupAllowListExtend: [...(this.setupOptions.txPublicSetupAllowListExtend ?? []), ...tokenAllowList], }); @@ -247,7 +282,10 @@ export class FeesTest extends SingleNodeTestContext { this.logger.info('Applying deploy banana token setup'); const { contract: bananaCoin } = await testSpan('deploy:token', () => - BananaCoin.deploy(this.wallet, this.aliceAddress, 'BC', 'BC', 18n).send({ + BananaCoin.deploy(this.wallet, this.aliceAddress, ...BANANA_COIN_CONSTRUCTOR_ARGS, { + salt: BANANA_COIN_SALT, + deployer: this.aliceAddress, + }).send({ from: this.aliceAddress, }), ); @@ -270,15 +308,26 @@ export class FeesTest extends SingleNodeTestContext { const bananaCoin = this.bananaCoin; const { contract: bananaFPC } = await testSpan('deploy:fpc', () => - FPCContract.deploy(this.wallet, bananaCoin.address, this.fpcAdmin).send({ + FPCContract.deploy(this.wallet, bananaCoin.address, this.fpcAdmin, { + salt: BANANA_FPC_SALT, + deployer: this.aliceAddress, + }).send({ from: this.aliceAddress, }), ); this.logger.info(`BananaPay deployed at ${bananaFPC.address}`); - // bridgeFromL1ToL2 carries its own setup:bridge span. - await this.feeJuiceBridgeTestHarness.bridgeFromL1ToL2(bananaFPC.address, this.aliceAddress); + // The BananaFPC's fee juice is seeded at genesis (see FeesTest.setup) rather than bridged here. + // Assert the deploy landed at the seeded address so a params drift surfaces as a clear error rather + // than a downstream "insufficient fee payer balance". + const { bananaFPCAddress } = await computeBananaContractAddresses(this.aliceAddress); + if (!bananaFPC.address.equals(bananaFPCAddress)) { + throw new Error( + `Deployed BananaFPC address ${bananaFPC.address} does not match the genesis-funded address ` + + `${bananaFPCAddress}; the deterministic deploy params drifted from the genesis funding computation.`, + ); + } this.bananaFPC = bananaFPC; From 915501a712fd9b465178fff96877358ed54d8448 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 8 Jul 2026 07:31:52 -0300 Subject: [PATCH 08/35] perf(bot): parallelize independent bot factory setup steps (#24581) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallelizes the independent setup steps in the bot factory (`@aztec/bot`, `bot/src/factory.ts`). This is production bot code that runs against live networks, so the changes are strictly behavior-preserving: same contracts at the same (salt-derived) addresses, same amounts, same final state. Only the ordering/concurrency of provably-independent setup steps changes. The bot factory is the single largest cost in the `single-node/bot` e2e suite: each `*.create` runs the production sequencer at 12s L2 slots (`PIPELINING_SETUP_OPTS`) and deploys/mints one tx per slot, fully serial. `AmmBot.create` alone was ~74s (7 serial txs). ## Dependency graph per bot type Edges below are "must be mined before", verified against the Noir contract source (`token_contract`, `amm_contract`). ### Transaction bot -- `setup()` (unchanged) `setupAccount` -> register recipient -> `ensureFeeJuiceBalance` -> deploy token -> mint. This chain is fully data-dependent: funding gates the deploy, the deploy gates the mint. The recipient (`createSchnorrAccount`) is register-only (no tx), and the private+public mints are already batched into one tx. Nothing is independent, so `setup()` is left serial. ### AMM bot -- `setupAmm()` Steps: deploy token0 (D0), token1 (D1), LP token (DL), AMM (DA); `set_minter` granting the AMM rights over the LP token (SM); mint token0+token1 to the provider (M, one batched tx); `add_liquidity` (AL). - D0, D1, DL: independent -- distinct contracts, no cross-references. - DA: the AMM constructor only stores the three token addresses (no calls into the tokens). Those addresses are salt-derived, so DA needs only the (pre-derived) addresses, not the token deploys mined. - SM: `Token::set_minter` just writes `minters[amm] = true`; it does not call or validate the AMM contract. It needs the LP token deployed and the (derivable) AMM address only -- not the AMM deployed. The deployer is authorized because the Token constructor sets `minters[admin] = true`. - M: `mint_to_private`/`mint_to_public` require the caller be a minter; the deployer is admin=minter from the constructor, so the token0/token1 mints need only those tokens deployed (no `set_minter`). - AL: `add_liquidity` transfers token0/token1 from the provider (needs M for balances), mints LP tokens (needs SM so the AMM is an LP minter), and targets the AMM (needs DA). Restructured from 7 serial txs into 3 slot-phases: - Phase 1: `Promise.all([D0, D1, DL])` - Phase 2: `Promise.all([DA, SM, M])` - Phase 3: `AL` ### Cross-chain bot -- `setupCrossChain()` Steps: deploy TestContract (an L2 tx), seed N L1->L2 messages (L1 txs), wait for the first message ready. - The seeds reference only the L2 recipient (TestContract) address, which is salt-derived. L1->L2 messages are queued on L1 and do not require the L2 contract to exist yet; they are consumed later in `bot.run()`, after setup completes. So the L2 deploy and the L1 seeding are independent. - The L2 deploy pays via the L2 wallet/PXE account; the seeds use the bot's L1 client -- different accounts, so there is no L1 nonce interaction between them. Restructured to overlap the deploy with the seed loop: `Promise.all([deploy TestContract, seed loop])`, then the message-ready wait. ## What stayed serial, and why - The whole transaction-bot `setup()` (data-dependent chain, nothing independent). - `ensureFeeJuiceBalance` before every deploy (funding must precede spending). - `add_liquidity` after its mints + minter grant + AMM deploy. - The individual L1->L2 seeds inside the loop: they share the bot's single L1 account, so concurrent sends would race on the L1 nonce. The suite already documents this nonce hazard. Only the loop as a whole overlaps the (L2) TestContract deploy. ## Production safety / idempotent re-entry - Every deploy still goes through `registerOrDeployContract`, which checks `getContractMetadata(address).isContractPublished` per contract and only registers (no tx) if already deployed. If one parallel deploy fails and the others succeed, the next `*.create` recovers: the succeeded contracts register-only, the failed one redeploys. Addresses are salt-derived and identical across runs. - `set_minter` is idempotent (writes `true` again). The AMM path's mint and `add_liquidity` always run (no balance guard) -- unchanged from the previous `fundAmm`. - Same-sender concurrent `.send()`s are safe by construction: the PXE serializes simulate/prove through its `SerialQueue` and setup txs pay with random tx nonces. The bot runs the same `EmbeddedWallet` -> PXE path in the e2e suite and in production. - Deliberate, benign failure-path change: because Phase 2 runs DA/SM/M concurrently, a failure of one no longer prevents the others from being sent. Their effects (a deployed-but-unused AMM, a minter grant, an extra mint) are all idempotent-safe, and `add_liquidity` uses fixed amounts, so re-entry converges to the same final state. - `withNoMinTxsPerBlock` (the `flushSetupTransactions` save/zero/restore wrapper around each setup tx) was not safe to re-enter concurrently: interleaved save/zero/restore could read the already-zeroed value and "restore" `minTxsPerBlock` to 0 permanently. It now reference-counts entrants -- the first saves and zeroes, the last restores -- with unit tests covering the overlapping and failure paths (`bot/src/factory.test.ts`). Serial callers see the exact same RPC sequence as before. Final state is identical in every path: same contract addresses, same minter grant, same minted amounts, same liquidity. ## Local evidence Two full local runs of `single-node/bot/bot.test.ts` (production sequencer, 12s L2 slots), all 10 tests passing in both. Hook durations from factory log timestamps, against the round-4 local baseline measured on the same machine at the same cadence (findings from #24534): - `Bot.create` (transaction-bot hook, untouched): 31.8s vs 32.2s baseline -- unchanged, as intended. - `AmmBot.create`: 57.2s vs 73.9s baseline (-16.7s). The three token deploy txs go out within 300ms of each other; the AMM deploy, `set_minter` and the mint batch all go out within the following slot. - `CrossChainBot.create`: 24.0s vs 43.5s baseline (-19.5s). The TestContract deploy overlaps both L1 seeds; the first message is ready almost immediately after the deploy is mined. - Suite-level `beforeHooksMs`: 116.9s vs 153.7s baseline (-36.8s), matching the per-hook deltas. - Unit: `bot/src/factory.test.ts` (4 tests) covering the `withNoMinTxsPerBlock` reentrancy fix. ## Expected effect on CI - `AmmBot.create`: 7 serial txs -> 3 slot-phases (~87s on CI per #24534's spans -> ~55-60s). - `CrossChainBot.create`: deploy overlaps the seed pipeline (~43s -> ~25s). - Transaction bot: unchanged. - Aggregate: roughly -35 to -45s on the bot suite's beforeAll wall time. - Fix-phase proof metric (once #24534's `setup:bot` instrumentation is on this base): the amm-bot and cross-chain `setup:bot` occurrences drop per the numbers above. Measured CI numbers to be appended when a full run lands. ## Measured impact The bot suite (a single shard) drops 216.3s → 118.3s total (**−98.0s, −45%**); beforeHooks 215.9s → 118.0s. - No `setup:bot` spans exist on this base (that instrumentation lives in the unmerged #24534), so the metric is the suite-level hook fold, which covers all three nested bot factory setups (Bot, AmmBot, CrossChainBot). - Noise context from the same run pair: untouched light suites move −2 to −3s per shard (private_initialization −2.9s/shard over 20 shards, deploy_method −2.1s/shard over 14), and the largest counter-move is validators_sentinel +24s (multi-node variance). A −98s single-shard delta is far outside that envelope. - The CI delta exceeds the local −36.8s because the CI baseline pays more missed-slot penalties per serial tx (baseline beforeHooks 215.9s on CI vs 153.7s locally at the same 12s cadence); collapsing 7 serial txs into 3 slot-phases removes proportionally more where slots are more often missed. Baseline CI run 1783393028773172 (merge-base 30966a45, full). PR CI run 1783427250198215 (x-fast). Fixes A-1408 (cherry picked from commit 5d2b6dfbafb09b6c76ba5fdb4a09f327ddde132c) --- yarn-project/bot/src/factory.test.ts | 87 ++++++++++ yarn-project/bot/src/factory.ts | 231 ++++++++++++++------------- 2 files changed, 211 insertions(+), 107 deletions(-) create mode 100644 yarn-project/bot/src/factory.test.ts diff --git a/yarn-project/bot/src/factory.test.ts b/yarn-project/bot/src/factory.test.ts new file mode 100644 index 000000000000..936cfc6ed233 --- /dev/null +++ b/yarn-project/bot/src/factory.test.ts @@ -0,0 +1,87 @@ +import { promiseWithResolvers } from '@aztec/foundation/promise'; +import type { AztecNode, AztecNodeAdmin, AztecNodeAdminConfig } from '@aztec/stdlib/interfaces/client'; +import type { EmbeddedWallet } from '@aztec/wallets/embedded'; + +import { mock } from 'jest-mock-extended'; + +import { type BotConfig, getBotDefaultConfig } from './config.js'; +import { BotFactory } from './factory.js'; +import type { BotStore } from './store/index.js'; + +class TestBotFactory extends BotFactory { + public override withNoMinTxsPerBlock(fn: () => Promise): Promise { + return super.withNoMinTxsPerBlock(fn); + } +} + +describe('BotFactory.withNoMinTxsPerBlock', () => { + let minTxsPerBlock: number | undefined; + let setConfigCalls: (number | undefined)[]; + let factory: TestBotFactory; + + beforeEach(() => { + minTxsPerBlock = 3; + setConfigCalls = []; + + const aztecNodeAdmin = mock(); + aztecNodeAdmin.getConfig.mockImplementation(() => Promise.resolve({ minTxsPerBlock } as AztecNodeAdminConfig)); + aztecNodeAdmin.setConfig.mockImplementation(config => { + setConfigCalls.push(config.minTxsPerBlock); + minTxsPerBlock = config.minTxsPerBlock; + return Promise.resolve(); + }); + + const config: BotConfig = { ...getBotDefaultConfig(), flushSetupTransactions: true }; + const wallet = mock(); + factory = new TestBotFactory(config, wallet, mock(), mock(), aztecNodeAdmin); + }); + + it('zeroes minTxsPerBlock around a call and restores it after', async () => { + await factory.withNoMinTxsPerBlock(() => { + expect(minTxsPerBlock).toBe(0); + return Promise.resolve(); + }); + + expect(setConfigCalls).toEqual([0, 3]); + expect(minTxsPerBlock).toBe(3); + }); + + it('zeroes once and restores once across overlapping calls', async () => { + const gates = [promiseWithResolvers(), promiseWithResolvers(), promiseWithResolvers()]; + const calls = gates.map(gate => factory.withNoMinTxsPerBlock(() => gate.promise)); + + gates[0].resolve(); + await calls[0]; + expect(minTxsPerBlock).toBe(0); + + gates[1].resolve(); + await calls[1]; + expect(minTxsPerBlock).toBe(0); + + gates[2].resolve(); + await calls[2]; + expect(setConfigCalls).toEqual([0, 3]); + expect(minTxsPerBlock).toBe(3); + }); + + it('restores minTxsPerBlock when the wrapped call fails', async () => { + await expect(factory.withNoMinTxsPerBlock(() => Promise.reject(new Error('boom')))).rejects.toThrow('boom'); + + expect(setConfigCalls).toEqual([0, 3]); + expect(minTxsPerBlock).toBe(3); + }); + + it('restores minTxsPerBlock when one of several overlapping calls fails', async () => { + const gate = promiseWithResolvers(); + const failing = factory.withNoMinTxsPerBlock(() => Promise.reject(new Error('boom'))); + const succeeding = factory.withNoMinTxsPerBlock(() => gate.promise); + + await expect(failing).rejects.toThrow('boom'); + expect(minTxsPerBlock).toBe(0); + + gate.resolve(); + await succeeding; + expect(setConfigCalls).toEqual([0, 3]); + expect(minTxsPerBlock).toBe(3); + }); +}); diff --git a/yarn-project/bot/src/factory.ts b/yarn-project/bot/src/factory.ts index d720ccdb83a2..19eac4f95e0b 100644 --- a/yarn-project/bot/src/factory.ts +++ b/yarn-project/bot/src/factory.ts @@ -44,6 +44,11 @@ const FEE_JUICE_TOP_UP_THRESHOLD = 100n * 10n ** 18n; export class BotFactory { private log = createLogger('bot'); + /** Number of in-flight withNoMinTxsPerBlock calls; see that method for why they are counted. */ + private noMinTxsPerBlockDepth = 0; + /** Set by the first withNoMinTxsPerBlock entrant; resolves to the minTxsPerBlock value to restore. */ + private savedMinTxsPerBlock?: Promise<{ minTxsPerBlock?: number }>; + constructor( private readonly config: BotConfig, private readonly wallet: EmbeddedWallet, @@ -86,23 +91,35 @@ export class BotFactory { }> { const defaultAccountAddress = await this.setupAccount(); await this.ensureFeeJuiceBalance(defaultAccountAddress); - const token0 = await this.setupTokenContract(defaultAccountAddress, this.config.tokenSalt, 'BotToken0', 'BOT0'); - const token1 = await this.setupTokenContract(defaultAccountAddress, this.config.tokenSalt, 'BotToken1', 'BOT1'); - const liquidityToken = await this.setupTokenContract( - defaultAccountAddress, - this.config.tokenSalt, - 'BotLPToken', - 'BOTLP', - ); - const amm = await this.setupAmmContract( - defaultAccountAddress, - this.config.tokenSalt, - token0, - token1, - liquidityToken, - ); - await this.fundAmm(defaultAccountAddress, defaultAccountAddress, amm, token0, token1, liquidityToken); + const salt = this.config.tokenSalt; + + // token0, token1 and the LP token are independent contracts with no shared state, so deploy them + // concurrently rather than one slot at a time. + const [token0, token1, liquidityToken] = await Promise.all([ + this.setupTokenContract(defaultAccountAddress, salt, 'BotToken0', 'BOT0'), + this.setupTokenContract(defaultAccountAddress, salt, 'BotToken1', 'BOT1'), + this.setupTokenContract(defaultAccountAddress, salt, 'BotLPToken', 'BOTLP'), + ]); + + const ammDeploy = AMMContract.deploy(this.wallet, token0.address, token1.address, liquidityToken.address, { + salt, + universalDeploy: true, + }); + const ammAddress = (await ammDeploy.getInstance()).address; + + // The AMM constructor only stores the (already-derived) token addresses, and set_minter only records + // the AMM address on the LP token: neither reads the other's on-chain state, so the AMM deploy, the + // LP-minter grant, and the token0/token1 mints are mutually independent and run concurrently. + const [amm] = await Promise.all([ + this.deployAmmContract(defaultAccountAddress, ammDeploy), + this.grantLpTokenMinter(defaultAccountAddress, liquidityToken, ammAddress), + this.mintAmmLiquidity(defaultAccountAddress, token0, token1), + ]); + + // add_liquidity spends the minted token0/token1 balances and mints LP tokens, so it must follow both + // the mints and the minter grant, and target the deployed AMM. + await this.addAmmLiquidity(defaultAccountAddress, defaultAccountAddress, amm, token0, token1, liquidityToken); this.log.info(`AMM initialized and funded`); return { wallet: this.wallet, defaultAccountAddress, amm, token0, token1, node: this.aztecNode }; @@ -140,25 +157,32 @@ export class BotFactory { const rollupContract = new RollupContract(l1Client, l1ContractAddresses.rollupAddress.toString()); const rollupVersion = await rollupContract.getVersion(); - // Deploy TestContract (pays from the standing balance funded above). - const contract = await this.setupTestContract(defaultAccountAddress); + // Derive the TestContract address up front (deterministic from the salt). Seeding L1→L2 messages only + // needs the L2 recipient address — the messages are queued on L1 and don't require the L2 contract to + // exist yet (they're consumed later, after setup completes) — so the deploy (an L2 tx paying from the + // standing balance funded above) and the L1 seeding run concurrently. + const testContractDeploy = TestContract.deploy(this.wallet, { + salt: this.config.tokenSalt, + universalDeploy: true, + }); + const contractAddress = (await testContractDeploy.getInstance()).address; // Recover any pending messages from store (clean up stale ones first) await this.store.cleanupOldPendingMessages(); const pendingMessages = await this.store.getUnconsumedL1ToL2Messages(); - // Seed initial L1→L2 messages if pipeline is empty + // Seed initial L1→L2 messages if pipeline is empty. The seeds are sent one at a time: they share the + // bot's L1 account, so concurrent sends would race on the L1 nonce. const seedCount = Math.max(0, this.config.l1ToL2SeedCount - pendingMessages.length); - for (let i = 0; i < seedCount; i++) { - await seedL1ToL2Message( - l1Client, - EthAddress.fromString(l1ContractAddresses.inboxAddress.toString()), - contract.address, - rollupVersion, - this.store, - this.log, - ); - } + const inboxAddress = EthAddress.fromString(l1ContractAddresses.inboxAddress.toString()); + const [contract] = await Promise.all([ + this.deployTestContract(defaultAccountAddress, testContractDeploy), + (async () => { + for (let i = 0; i < seedCount; i++) { + await seedL1ToL2Message(l1Client, inboxAddress, contractAddress, rollupVersion, this.store, this.log); + } + })(), + ]); // Block until at least one message is ready const allMessages = await this.store.getUnconsumedL1ToL2Messages(); @@ -182,10 +206,8 @@ export class BotFactory { }; } - private async setupTestContract(deployer: AztecAddress): Promise { - const deployOpts: DeployOptions = { from: deployer }; - const deploy = TestContract.deploy(this.wallet, { salt: this.config.tokenSalt, universalDeploy: true }); - const instance = await this.registerOrDeployContract('TestContract', deploy, deployOpts); + private async deployTestContract(deployer: AztecAddress, deploy: DeployMethod): Promise { + const instance = await this.registerOrDeployContract('TestContract', deploy, { from: deployer }); return TestContract.at(instance.address, this.wallet); } @@ -289,34 +311,37 @@ export class BotFactory { return TokenContract.at(instance.address, this.wallet); } - private async setupAmmContract( - deployer: AztecAddress, - salt: Fr, - token0: TokenContract, - token1: TokenContract, - lpToken: TokenContract, - ): Promise { - const deployOpts: DeployOptions = { from: deployer }; - const deploy = AMMContract.deploy(this.wallet, token0.address, token1.address, lpToken.address, { - salt, - universalDeploy: true, - }); - const instance = await this.registerOrDeployContract('AMM', deploy, deployOpts); + private async deployAmmContract(deployer: AztecAddress, deploy: DeployMethod): Promise { + const instance = await this.registerOrDeployContract('AMM', deploy, { from: deployer }); const amm = AMMContract.at(instance.address, this.wallet); - this.log.info(`AMM deployed at ${amm.address}`); - const setMinterInteraction = lpToken.methods.set_minter(amm.address, true); - const { receipt: minterReceipt } = await setMinterInteraction.send({ + return amm; + } + + /** Grants the AMM minting rights over the LP token. set_minter only records the address, so it does not + * require the AMM contract to be deployed first. */ + private async grantLpTokenMinter(deployer: AztecAddress, lpToken: TokenContract, amm: AztecAddress): Promise { + const { receipt } = await lpToken.methods.set_minter(amm, true).send({ from: deployer, wait: { timeout: this.config.txMinedWaitSeconds }, }); - this.log.info(`Set LP token minter to AMM txHash=${minterReceipt.txHash.toString()}`); - this.log.info(`Liquidity token initialized`); + this.log.info(`Set LP token minter to AMM txHash=${receipt.txHash.toString()}`); + } - return amm; + private async mintAmmLiquidity(minter: AztecAddress, token0: TokenContract, token1: TokenContract): Promise { + this.log.info(`Minting ${MINT_BALANCE} tokens of each BotToken0 and BotToken1 for ${minter}`); + const mintBatch = new BatchCall(this.wallet, [ + token0.methods.mint_to_private(minter, MINT_BALANCE), + token1.methods.mint_to_private(minter, MINT_BALANCE), + ]); + const { receipt } = await mintBatch.send({ + from: minter, + wait: { timeout: this.config.txMinedWaitSeconds }, + }); + this.log.info(`Sent mint tx: ${receipt.txHash.toString()}`); } - private async fundAmm( + private async addAmmLiquidity( defaultAccountAddress: AztecAddress, liquidityProvider: AztecAddress, amm: AMMContract, @@ -324,22 +349,6 @@ export class BotFactory { token1: TokenContract, lpToken: TokenContract, ): Promise { - const getPrivateBalances = () => - Promise.all([ - token0.methods - .balance_of_private(liquidityProvider) - .simulate({ from: liquidityProvider }) - .then(r => r.result), - token1.methods - .balance_of_private(liquidityProvider) - .simulate({ from: liquidityProvider }) - .then(r => r.result), - lpToken.methods - .balance_of_private(liquidityProvider) - .simulate({ from: liquidityProvider }) - .then(r => r.result), - ]); - const authwitNonce = Fr.random(); // keep some tokens for swapping @@ -348,12 +357,6 @@ export class BotFactory { const amount1Max = MINT_BALANCE / 2; const amount1Min = MINT_BALANCE / 4; - const [t0Bal, t1Bal, lpBal] = await getPrivateBalances(); - - this.log.info( - `Minting ${MINT_BALANCE} tokens of each BotToken0 and BotToken1. Current private balances of ${liquidityProvider}: token0=${t0Bal}, token1=${t1Bal}, lp=${lpBal}`, - ); - // Add authwitnesses for the transfers in AMM::add_liquidity function const token0Authwit = await this.wallet.createAuthWit(defaultAccountAddress, { caller: amm.address, @@ -378,36 +381,33 @@ export class BotFactory { .getFunctionCall(), }); - const mintBatch = new BatchCall(this.wallet, [ - token0.methods.mint_to_private(liquidityProvider, MINT_BALANCE), - token1.methods.mint_to_private(liquidityProvider, MINT_BALANCE), - ]); - const { receipt: mintReceipt } = await mintBatch.send({ - from: liquidityProvider, - wait: { timeout: this.config.txMinedWaitSeconds }, - }); - - this.log.info(`Sent mint tx: ${mintReceipt.txHash.toString()}`); - - const addLiquidityInteraction = amm.methods.add_liquidity( - amount0Max, - amount1Max, - amount0Min, - amount1Min, - authwitNonce, - ); - const { receipt: addLiquidityReceipt } = await addLiquidityInteraction.send({ - from: liquidityProvider, - authWitnesses: [token0Authwit, token1Authwit], - wait: { timeout: this.config.txMinedWaitSeconds }, - }); + const { receipt } = await amm.methods + .add_liquidity(amount0Max, amount1Max, amount0Min, amount1Min, authwitNonce) + .send({ + from: liquidityProvider, + authWitnesses: [token0Authwit, token1Authwit], + wait: { timeout: this.config.txMinedWaitSeconds }, + }); - this.log.info(`Sent tx to add liquidity to the AMM: ${addLiquidityReceipt.txHash.toString()}`); + this.log.info(`Sent tx to add liquidity to the AMM: ${receipt.txHash.toString()}`); this.log.info(`Liquidity added`); - const [newT0Bal, newT1Bal, newLPBal] = await getPrivateBalances(); + const [t0Bal, t1Bal, lpBal] = await Promise.all([ + token0.methods + .balance_of_private(liquidityProvider) + .simulate({ from: liquidityProvider }) + .then(r => r.result), + token1.methods + .balance_of_private(liquidityProvider) + .simulate({ from: liquidityProvider }) + .then(r => r.result), + lpToken.methods + .balance_of_private(liquidityProvider) + .simulate({ from: liquidityProvider }) + .then(r => r.result), + ]); this.log.info( - `Updated private balances of ${defaultAccountAddress} after minting and funding AMM: token0=${newT0Bal}, token1=${newT1Bal}, lp=${newLPBal}`, + `Updated private balances of ${defaultAccountAddress} after minting and funding AMM: token0=${t0Bal}, token1=${t1Bal}, lp=${lpBal}`, ); } @@ -590,19 +590,36 @@ export class BotFactory { return claim as L2AmountClaim; } - private async withNoMinTxsPerBlock(fn: () => Promise): Promise { + protected async withNoMinTxsPerBlock(fn: () => Promise): Promise { if (!this.aztecNodeAdmin || !this.config.flushSetupTransactions) { this.log.verbose(`No node admin client or flushing not requested (not setting minTxsPerBlock to 0)`); return fn(); } - const { minTxsPerBlock } = await this.aztecNodeAdmin.getConfig(); - this.log.warn(`Setting sequencer minTxsPerBlock to 0 from ${minTxsPerBlock} to flush setup transactions`); - await this.aztecNodeAdmin.setConfig({ minTxsPerBlock: 0 }); + const aztecNodeAdmin = this.aztecNodeAdmin; + // Setup steps run concurrently, so this wrapper can be re-entered while another call is in flight. + // Reference-count the entrants: the first saves the current value and zeroes it, the last restores it. + // A naive save/zero/restore per call could interleave, with a late entrant reading the already-zeroed + // value and "restoring" 0 at the end. + if (this.noMinTxsPerBlockDepth++ === 0) { + this.savedMinTxsPerBlock = (async () => { + const { minTxsPerBlock } = await aztecNodeAdmin.getConfig(); + this.log.warn(`Setting sequencer minTxsPerBlock to 0 from ${minTxsPerBlock} to flush setup transactions`); + await aztecNodeAdmin.setConfig({ minTxsPerBlock: 0 }); + return { minTxsPerBlock }; + })(); + } try { + await this.savedMinTxsPerBlock; return await fn(); } finally { - this.log.warn(`Restoring sequencer minTxsPerBlock to ${minTxsPerBlock}`); - await this.aztecNodeAdmin.setConfig({ minTxsPerBlock }); + if (--this.noMinTxsPerBlockDepth === 0) { + // If saving/zeroing itself failed there is nothing to restore. + const saved = await this.savedMinTxsPerBlock!.catch(() => undefined); + if (saved) { + this.log.warn(`Restoring sequencer minTxsPerBlock to ${saved.minTxsPerBlock}`); + await aztecNodeAdmin.setConfig({ minTxsPerBlock: saved.minTxsPerBlock }); + } + } } } } From a176d96b1d0f5d21c8e4f1ca3d06c9ca7c8af016 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 8 Jul 2026 08:40:21 -0300 Subject: [PATCH 09/35] perf(e2e): seed standard contracts at genesis (#24568) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-4 e2e speedup, PR 1b (adoption half of the AuthRegistry prize). **Stacked on #24567** (`spl/genesis-prefilled-nullifiers`, the genesis-nullifier mechanism). This PR targets that branch; retarget to `merge-train/spartan-v5` once #24567 merges. ## What Make e2e environments start with the standard contracts (AuthRegistry, PublicChecks, HandshakeRegistry) already "published", so `ensureAuthRegistryPublished` and its two siblings skip their two publish txs each. At production cadence `setup:auth-registry` alone was ~16 min/run (2 sequential txs per process, paid per-process across the suite). This collapses that span to sub-second. ## Mechanism (three parts, all default-off outside e2e) - **Genesis nullifiers.** Per standard contract, seed `siloNullifier(ContractClassRegistry, classId)` and `siloNullifier(ContractInstanceRegistry, instanceAddress)` (the real derived address — standard contracts are not magic-address protocol contracts) into the fixtures' `getGenesisValues` call, via PR 1a's new 5th param. These are exactly the nullifiers the publish txs would emit, so the AVM's deployment-nullifier check passes when the contracts are called publicly. A single helper (`getStandardContractGenesisNullifiers`) feeds all four e2e genesis builders (`fixtures/setup.ts`, `e2e_prover_test.ts`, `p2p/p2p_network.ts`, `multi-node/governance/add_rollup.test.ts`) — the latter three recompute a genesis that must reproduce the L1-deployed archive root, so they must seed the identical set. Non-e2e callers (cli, sandbox/local-network) are untouched. - **Archiver preload.** `registerStandardContracts` mirrors `registerProtocolContracts`: it seeds each contract's class (with recomputed `publicBytecodeCommitment`), instance, and public-function signatures into the contract store at block 0, reading the bundled `@aztec/standard-contracts` artifacts. It is idempotent (skips already-registered classes on restart) and gated behind a new archiver config flag `testPreloadStandardContracts` (env `TEST_PRELOAD_STANDARD_CONTRACTS`, default **false**). The flag is set from the e2e node config so every spawned node (validators, prover nodes) picks it up through the normal config path. This adds `@aztec/standard-contracts` as an archiver dependency. - **Guard short-circuit.** The `ensure*Published` helpers are unchanged. Their guards read `wallet.getContractClassMetadata(id).isContractClassPubliclyRegistered` (to `aztecNode.getContractClass`) and `wallet.getContractMetadata(addr).isContractPublished` (to `aztecNode.getContract`) — both are archiver-store reads, not nullifier-tree reads. The store preload alone makes both short-circuit; the genesis nullifiers are what make the contract actually callable in the AVM afterwards. Keeping the helpers doubles as a regression check: if seeding ever breaks, the tx path re-engages and tests still pass, just slow. ## Why the flag is test-only (A-1257 rationale) Preloading unconditionally in production would recreate the A-1257 collision #24254 fixed: a real on-chain publish of a preloaded class would collide with the block-0 preload, because production genesis will **not** carry the matching nullifiers. The flag is only set by the e2e fixtures, which also seed the nullifiers, keeping the store and the nullifier tree consistent. A single source of truth (`getPublishableStandardContracts`) drives both the preloaded set and the seeded-nullifier set so they cannot drift. ## State-write verification Inspected the two publish txs on this base: - `ContractClassRegistry.publish` pushes one nullifier (`classId`, siloed to the class-registry address) and broadcasts a `ContractClassPublished` contract-class log carrying the packed bytecode. No public-data writes. - `ContractInstanceRegistry.publish_for_public_execution` asserts the class-registration nullifier exists, then pushes one nullifier (`address`, siloed to the instance-registry address) and broadcasts a `ContractInstancePublished` private log. No public-data writes. So the full state-write set is two nullifiers + two broadcast logs. The nullifiers are replicated via genesis `prefilledNullifiers`; the archiver normally learns the class/instance from those two logs (`data_store_updater.ts`), and the block-0 preload replaces that path exactly. No `genesisPublicData` is required. PXE-side registration (`wallet.registerContract`) is unaffected and still runs in the helpers. ## Local verification - `automine/accounts/authwit.test.ts` (public-authwit path against the standard AuthRegistry), timing on: passes; `setup:auth-registry` span = **10 ms** (was ~33 s at production cadence). No on-chain AuthRegistry publish (the only `ContractClassRegistry.publish` executions are the test's own AuthWitTest/GenericProxy deploys); the public AuthRegistry contract executes fine against the seeded nullifier. `testPreloadStandardContracts: true` confirmed in the node config dump. - `single-node/fees/account_init.test.ts` (production sequencer + simulated prover node), timing on: 5/5 pass; `setup:auth-registry` span = **10 ms**. Prover node syncs against the seeded genesis (no root divergence), and the flag propagates to it. No duplicate-nullifier errors. - New unit test in `archiver/src/modules/data_store_updater.test.ts`: `registerStandardContracts` preloads each publishable standard contract's class + instance into the store and is idempotent on a second call. ## Expected impact ~14–16 min/run from auth-registry alone, plus more from the two untagged siblings (PublicChecks / HandshakeRegistry) wherever used. ## Measured impact `setup:auth-registry` collapses from 702.2s of setup time run-wide (39 occurrences, 5–35s each) to 482ms total (max 87ms per occurrence) — sub-second run-wide, every shard clean, no seeding or flag-propagation slow path. - The measured beforeHooks reduction is larger than the auth-registry span alone, because the two untagged sibling publishes (PublicChecks, HandshakeRegistry) are removed too. Example: account_init beforeHooks −49.6s vs a 30.9s tagged auth-registry span; the extra ~19s is the two siblings. - Per-suite beforeHooks deltas: account_init −49.6s, fee_settings −49.5s, failures −48.7s, private_payments.parallel −41.8s/shard (×8), gas_estimation.parallel −41.5s/shard (×3), l1_to_l2 −38.1s, l2_to_l1 −37.5s, l1_to_l2_inbox_drift −38.2s, token_bridge −37.7s, fee_juice_payments −37.4s, bot −34.3s; automine/parallel suites shed 5–15s each (their fast-cadence publishes). - Total beforeHooks reduction over the suites present in both runs: −1,649.7s summed across shards (wall-clock benefit is smaller, since shards run in parallel across workers). Baseline CI run 1783389062016888 (#24567 `ci/x-fast`; mechanism-only, so timing-neutral vs merge-base). PR CI run 1783391194852197 (#24568 `ci/x-fast`). ## Notes for downstream - **PR 6 (fees harness):** `applyEnsureAuthRegistryPublished` now short-circuits; the fees setup chain loses the ~32 s/shard auth-registry step, so rebase the overlap/batch shape onto what remains (token deploy, FPC, mints). - **Fix phase / timing capture:** expected span assertion is `setup:auth-registry` around sub-second across the run (10 ms observed locally per process). Any shard still showing tens of seconds means the guards took the slow path (a seeding or flag-propagation bug), not noise. Fixes A-1404 (cherry picked from commit f3200495291ec4c2e328b9659e1e95b00620bda7) --- yarn-project/archiver/package.json | 1 + yarn-project/archiver/src/config.ts | 8 +++++ yarn-project/archiver/src/factory.ts | 34 +++++++++++++++++++ .../src/modules/data_store_updater.test.ts | 29 +++++++++++++++- yarn-project/archiver/tsconfig.json | 3 ++ .../src/fixtures/e2e_prover_test.ts | 2 ++ yarn-project/end-to-end/src/fixtures/setup.ts | 9 +++++ .../fixtures/standard_contracts_genesis.ts | 28 +++++++++++++++ .../multi-node/governance/add_rollup.test.ts | 9 ++++- .../end-to-end/src/p2p/p2p_network.ts | 2 ++ yarn-project/foundation/src/config/env_var.ts | 1 + yarn-project/standard-contracts/src/index.ts | 1 + .../src/publishable_standard_contracts.ts | 22 ++++++++++++ .../stdlib/src/interfaces/archiver.ts | 8 +++++ yarn-project/yarn.lock | 1 + 15 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 yarn-project/end-to-end/src/fixtures/standard_contracts_genesis.ts create mode 100644 yarn-project/standard-contracts/src/publishable_standard_contracts.ts diff --git a/yarn-project/archiver/package.json b/yarn-project/archiver/package.json index 57d099e5eed7..621c76bbb333 100644 --- a/yarn-project/archiver/package.json +++ b/yarn-project/archiver/package.json @@ -75,6 +75,7 @@ "@aztec/l1-artifacts": "portal:../../l1-contracts/l1-artifacts", "@aztec/noir-protocol-circuits-types": "workspace:^", "@aztec/protocol-contracts": "workspace:^", + "@aztec/standard-contracts": "workspace:^", "@aztec/stdlib": "workspace:^", "@aztec/telemetry-client": "workspace:^", "lodash.groupby": "^4.6.0", diff --git a/yarn-project/archiver/src/config.ts b/yarn-project/archiver/src/config.ts index 6c0e996c9d44..c01c048f7fb3 100644 --- a/yarn-project/archiver/src/config.ts +++ b/yarn-project/archiver/src/config.ts @@ -90,6 +90,14 @@ export const archiverConfigMappings: ConfigMappingsType = { description: 'Skip pruning orphan proposed blocks that have no matching proposed checkpoint.', ...booleanConfigHelper(false), }, + testPreloadStandardContracts: { + env: 'TEST_PRELOAD_STANDARD_CONTRACTS', + description: + 'Preload the standard contracts (AuthRegistry, PublicChecks, HandshakeRegistry) into the contract store at ' + + 'block 0. For test environments only, and only safe when genesis seeds the matching registration/deployment ' + + 'nullifiers; otherwise a later on-chain publish would collide with the block-0 preload.', + ...booleanConfigHelper(false), + }, ...chainConfigMappings, ...l1ReaderConfigMappings, viemPollingIntervalMS: { diff --git a/yarn-project/archiver/src/factory.ts b/yarn-project/archiver/src/factory.ts index 2521ba1e0a88..fbdbaf4aca84 100644 --- a/yarn-project/archiver/src/factory.ts +++ b/yarn-project/archiver/src/factory.ts @@ -12,6 +12,7 @@ import { DateProvider } from '@aztec/foundation/timer'; import { createStore } from '@aztec/kv-store/lmdb-v2'; import { protocolContractNames } from '@aztec/protocol-contracts'; import { BundledProtocolContractsProvider } from '@aztec/protocol-contracts/providers/bundle'; +import { getPublishableStandardContracts } from '@aztec/standard-contracts'; import { FunctionType, decodeFunctionSignature } from '@aztec/stdlib/abi'; import type { ArchiverEmitter, BlockHash } from '@aztec/stdlib/block'; import { DEFAULT_BLOCK_DURATION_MS } from '@aztec/stdlib/config'; @@ -68,6 +69,9 @@ export async function createArchiver( ): Promise { const archiverStore = await createArchiverStore(config, initialBlockHash); await registerProtocolContracts(archiverStore); + if (config.testPreloadStandardContracts) { + await registerStandardContracts(archiverStore); + } // Create Ethereum clients const chain = createEthereumChain(config.l1RpcUrls, config.l1ChainId); @@ -227,3 +231,33 @@ export async function registerProtocolContracts(stores: ArchiverDataStores) { await stores.contractInstances.addContractInstances([contract.instance], BlockNumber(blockNumber)); } } + +/** + * Preloads the standard contracts (AuthRegistry, PublicChecks, HandshakeRegistry) into the archiver store at block 0, + * mirroring {@link registerProtocolContracts}. Only invoked for test environments (via `testPreloadStandardContracts`), + * which also seed the matching registration/deployment nullifiers into the genesis nullifier tree so the store and tree + * stay consistent. Idempotent — skips contracts that already exist (e.g. on node restart). + */ +export async function registerStandardContracts(stores: ArchiverDataStores) { + const blockNumber = 0; + for (const contract of await getPublishableStandardContracts()) { + // Skip if already registered (happens on node restart with a persisted store). + if (await stores.contractClasses.getContractClass(contract.contractClass.id)) { + continue; + } + + const publicBytecodeCommitment = await computePublicBytecodeCommitment(contract.contractClass.packedBytecode); + const contractClassPublic: ContractClassPublicWithCommitment = { + ...contract.contractClass, + publicBytecodeCommitment, + }; + + const publicFunctionSignatures = contract.artifact.functions + .filter(fn => fn.functionType === FunctionType.PUBLIC) + .map(fn => decodeFunctionSignature(fn.name, fn.parameters)); + + await stores.functionNames.register(publicFunctionSignatures); + await stores.contractClasses.addContractClasses([contractClassPublic], BlockNumber(blockNumber)); + await stores.contractInstances.addContractInstances([contract.instance], BlockNumber(blockNumber)); + } +} diff --git a/yarn-project/archiver/src/modules/data_store_updater.test.ts b/yarn-project/archiver/src/modules/data_store_updater.test.ts index ff086c1c6a27..8a385ff54876 100644 --- a/yarn-project/archiver/src/modules/data_store_updater.test.ts +++ b/yarn-project/archiver/src/modules/data_store_updater.test.ts @@ -6,6 +6,7 @@ import { ProtocolContractAddress } from '@aztec/protocol-contracts'; import { ContractClassPublishedEvent } from '@aztec/protocol-contracts/class-registry'; import { ContractInstancePublishedEvent } from '@aztec/protocol-contracts/instance-registry'; import { BundledProtocolContractsProvider } from '@aztec/protocol-contracts/providers/bundle'; +import { getPublishableStandardContracts } from '@aztec/standard-contracts'; import { bufferAsFields } from '@aztec/stdlib/abi'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { GENESIS_BLOCK_HEADER_HASH, L2Block } from '@aztec/stdlib/block'; @@ -19,7 +20,7 @@ import { readFileSync } from 'fs'; import { dirname, resolve } from 'path'; import { fileURLToPath } from 'url'; -import { registerProtocolContracts } from '../factory.js'; +import { registerProtocolContracts, registerStandardContracts } from '../factory.js'; import { type ArchiverDataStores, createArchiverDataStores } from '../store/data_stores.js'; import { L2TipsCache } from '../store/l2_tips_cache.js'; import { makeCheckpoint, makePublishedCheckpoint } from '../test/mock_structs.js'; @@ -149,6 +150,32 @@ describe('ArchiverDataStoreUpdater', () => { expect(await store.contractClasses.getContractClass(protocolClassId)).toBeDefined(); }); + it('preloads standard contract classes and instances via registerStandardContracts', async () => { + const standardContracts = await getPublishableStandardContracts(); + expect(standardContracts.length).toBeGreaterThan(0); + + // Not present before the preload. + for (const { contractClass } of standardContracts) { + expect(await store.contractClasses.getContractClass(contractClass.id)).toBeUndefined(); + } + + await registerStandardContracts(store); + + // Both the class and the instance are queryable from the block-0 preload. + for (const { contractClass, address } of standardContracts) { + const retrievedClass = await store.contractClasses.getContractClass(contractClass.id); + expect(retrievedClass?.id.equals(contractClass.id)).toBe(true); + const retrievedInstance = await store.contractInstances.getContractInstance(address, 1n); + expect(retrievedInstance?.address.equals(address)).toBe(true); + } + + // Calling again (e.g. on node restart with a persisted store) is idempotent and must not throw. + await expect(registerStandardContracts(store)).resolves.not.toThrow(); + for (const { contractClass } of standardContracts) { + expect(await store.contractClasses.getContractClass(contractClass.id)).toBeDefined(); + } + }); + it('removes contract class and instance data when blocks are pruned via setCheckpointData', async () => { // First, add a local provisional block with contract data const localBlock = await L2Block.random(BlockNumber(1), { diff --git a/yarn-project/archiver/tsconfig.json b/yarn-project/archiver/tsconfig.json index 655dea45d14d..6ee6c14c7b3e 100644 --- a/yarn-project/archiver/tsconfig.json +++ b/yarn-project/archiver/tsconfig.json @@ -33,6 +33,9 @@ { "path": "../protocol-contracts" }, + { + "path": "../standard-contracts" + }, { "path": "../stdlib" }, diff --git a/yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts b/yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts index 638b2b8fd623..77de58f7796b 100644 --- a/yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts +++ b/yarn-project/end-to-end/src/fixtures/e2e_prover_test.ts @@ -24,6 +24,7 @@ import { TestWallet } from '../test-wallet/test_wallet.js'; import { getACVMConfig } from './get_acvm_config.js'; import { getBBConfig } from './get_bb_config.js'; import { getPrivateKeyFromIndex, getSponsoredFPCAddress, setup, setupPXEAndGetWallet } from './setup.js'; +import { getStandardContractGenesisNullifiers } from './standard_contracts_genesis.js'; type ProvenSetup = { wallet: TestWallet; @@ -229,6 +230,7 @@ export class FullProverTest extends SingleNodeTestContext { undefined, undefined, this.context.genesis!.genesisTimestamp, + await getStandardContractGenesisNullifiers(), ); const proverNodeConfig: Parameters[0] = { diff --git a/yarn-project/end-to-end/src/fixtures/setup.ts b/yarn-project/end-to-end/src/fixtures/setup.ts index 6705ed463bf5..8281c8104cce 100644 --- a/yarn-project/end-to-end/src/fixtures/setup.ts +++ b/yarn-project/end-to-end/src/fixtures/setup.ts @@ -77,6 +77,7 @@ import { MNEMONIC, TEST_MAX_PENDING_TX_POOL_COUNT, TEST_PEER_CHECK_INTERVAL_MS } import { getACVMConfig } from './get_acvm_config.js'; import { getBBConfig } from './get_bb_config.js'; import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js'; +import { getStandardContractGenesisNullifiers } from './standard_contracts_genesis.js'; import { testSpan } from './timing.js'; import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; @@ -474,12 +475,20 @@ async function setupInner( } logger.trace('Generated test accounts to fund at genesis'); + // Preload the standard contracts (AuthRegistry, PublicChecks, HandshakeRegistry) so their `ensure*Published` setup + // helpers short-circuit their publish txs. The archiver flag seeds the bytecode/instance into every spawned node's + // contract store; the genesis nullifiers make the AVM's deployment-nullifier check pass when they are called. Both + // must go together (flag alone would recreate the publish-collision bug), so the flag lives here beside the seeding. + config.testPreloadStandardContracts = true; + const standardContractNullifiers = await getStandardContractGenesisNullifiers(); + const genesisTimestamp = BigInt(Math.floor(Date.now() / 1000)); const { genesisArchiveRoot, genesis, fundingNeeded } = await getGenesisValues( addressesToFund, opts.initialAccountFeeJuice, opts.genesisPublicData, genesisTimestamp, + standardContractNullifiers, ); logger.trace('Computed genesis values'); diff --git a/yarn-project/end-to-end/src/fixtures/standard_contracts_genesis.ts b/yarn-project/end-to-end/src/fixtures/standard_contracts_genesis.ts new file mode 100644 index 000000000000..14c52180234b --- /dev/null +++ b/yarn-project/end-to-end/src/fixtures/standard_contracts_genesis.ts @@ -0,0 +1,28 @@ +import type { Fr } from '@aztec/foundation/curves/bn254'; +import { ProtocolContractAddress } from '@aztec/protocol-contracts'; +import { getPublishableStandardContracts } from '@aztec/standard-contracts'; +import { siloNullifier } from '@aztec/stdlib/hash'; + +/** + * Computes the genesis nullifiers that pre-publish the standard contracts (AuthRegistry, PublicChecks, + * HandshakeRegistry) in e2e environments, mirroring the nullifiers their on-chain publish txs would emit. Per contract: + * - `siloNullifier(ContractClassRegistry, classId)` — the class-registration nullifier that `ContractClassRegistry.publish` pushes. + * - `siloNullifier(ContractInstanceRegistry, instanceAddress)` — the instance-deployment nullifier that + * `ContractInstanceRegistry.publish_for_public_execution` pushes, using the contract's real derived address (standard + * contracts are deployed at artifact-derived addresses, not magic protocol addresses). + * + * Seed these into the genesis nullifier tree (as the 5th `getGenesisValues` arg) alongside the archiver's + * `testPreloadStandardContracts` preload: the store preload makes the `ensure*Published` guards short-circuit, and these + * nullifiers make the AVM's deployment-nullifier check pass when the contracts are called. Every e2e node genesis that + * feeds an L1 `genesisArchiveRoot` must seed the same set, or the world-state root diverges from the deployed rollup. + */ +export async function getStandardContractGenesisNullifiers(): Promise { + const classRegistry = ProtocolContractAddress.ContractClassRegistry; + const instanceRegistry = ProtocolContractAddress.ContractInstanceRegistry; + const nullifiers: Fr[] = []; + for (const { contractClass, address } of await getPublishableStandardContracts()) { + nullifiers.push(await siloNullifier(classRegistry, contractClass.id)); + nullifiers.push(await siloNullifier(instanceRegistry, address.toField())); + } + return nullifiers; +} diff --git a/yarn-project/end-to-end/src/multi-node/governance/add_rollup.test.ts b/yarn-project/end-to-end/src/multi-node/governance/add_rollup.test.ts index b7085a1773ca..55d2e6372c5a 100644 --- a/yarn-project/end-to-end/src/multi-node/governance/add_rollup.test.ts +++ b/yarn-project/end-to-end/src/multi-node/governance/add_rollup.test.ts @@ -34,6 +34,7 @@ import { type Hex, decodeEventLog, encodeFunctionData, getAddress, getContract } import { foundry } from 'viem/chains'; import { sendL1ToL2Message } from '../../fixtures/l1_to_l2_messaging.js'; +import { getStandardContractGenesisNullifiers } from '../../fixtures/standard_contracts_genesis.js'; import { getPrivateKeyFromIndex, getSponsoredFPCAddress } from '../../fixtures/utils.js'; import { TestWallet } from '../../test-wallet/test_wallet.js'; import { @@ -136,7 +137,13 @@ describe('multi-node/governance/add_rollup', () => { genesisArchiveRoot, fundingNeeded, genesis: newGenesis, - } = await getGenesisValues(genesisFundedAddresses, undefined, undefined, context.genesis!.genesisTimestamp + 1n); + } = await getGenesisValues( + genesisFundedAddresses, + undefined, + undefined, + context.genesis!.genesisTimestamp + 1n, + await getStandardContractGenesisNullifiers(), + ); const { rollup: newRollup } = await deployRollupForUpgrade( deployerPrivateKey, diff --git a/yarn-project/end-to-end/src/p2p/p2p_network.ts b/yarn-project/end-to-end/src/p2p/p2p_network.ts index 045ca4529360..fc154d3cb13d 100644 --- a/yarn-project/end-to-end/src/p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/p2p/p2p_network.ts @@ -49,6 +49,7 @@ import { createValidatorConfig, generatePrivateKeys, } from '../fixtures/setup_p2p_test.js'; +import { getStandardContractGenesisNullifiers } from '../fixtures/standard_contracts_genesis.js'; import { getEndToEndTestTelemetryClient } from '../fixtures/with_telemetry_utils.js'; import type { TestWallet } from '../test-wallet/test_wallet.js'; @@ -415,6 +416,7 @@ export class P2PNetworkTest { undefined, undefined, this.context.genesis!.genesisTimestamp, + await getStandardContractGenesisNullifiers(), ); this.genesis = genesis; diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 706ceae059ba..fa3c07ced9ef 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -279,6 +279,7 @@ export type EnvVar = | 'SYNC_SNAPSHOTS_URL' | 'TELEMETRY' | 'TEST_ACCOUNTS' + | 'TEST_PRELOAD_STANDARD_CONTRACTS' | 'SPONSORED_FPC' | 'PREFUND_ADDRESSES' | 'TX_COLLECTION_FAST_NODES_TIMEOUT_BEFORE_REQ_RESP_MS' diff --git a/yarn-project/standard-contracts/src/index.ts b/yarn-project/standard-contracts/src/index.ts index 294dc5b249d6..8bb35a0370f7 100644 --- a/yarn-project/standard-contracts/src/index.ts +++ b/yarn-project/standard-contracts/src/index.ts @@ -2,3 +2,4 @@ export * from './auth-registry/index.js'; export * from './handshake-registry/index.js'; export * from './multi-call-entrypoint/index.js'; export * from './public-checks/index.js'; +export * from './publishable_standard_contracts.js'; diff --git a/yarn-project/standard-contracts/src/publishable_standard_contracts.ts b/yarn-project/standard-contracts/src/publishable_standard_contracts.ts new file mode 100644 index 000000000000..e91c30b0a462 --- /dev/null +++ b/yarn-project/standard-contracts/src/publishable_standard_contracts.ts @@ -0,0 +1,22 @@ +import { getStandardAuthRegistry } from './auth-registry/index.js'; +import { getStandardHandshakeRegistry } from './handshake-registry/index.js'; +import { getStandardPublicChecks } from './public-checks/index.js'; +import type { StandardContract } from './standard_contract.js'; + +/** + * Returns the standard contracts that are published on-chain (class registration + instance + * publication) before their public functions can be called: AuthRegistry, PublicChecks, and + * HandshakeRegistry. These are exactly the contracts guarded by the `ensure*Published` e2e setup + * helpers. + * + * MultiCallEntrypoint is deliberately excluded: it is a client-side entrypoint used to encode + * batched private calls and is never published for public execution. + * + * This is the single source of truth for the set of standard contracts that test environments seed + * at genesis (registration/deployment nullifiers) and preload into the archiver contract store, so + * the two stay consistent — preloading a class whose nullifier is not seeded would recreate the + * publish-collision bug that genesis seeding avoids. + */ +export function getPublishableStandardContracts(): Promise { + return Promise.all([getStandardAuthRegistry(), getStandardPublicChecks(), getStandardHandshakeRegistry()]); +} diff --git a/yarn-project/stdlib/src/interfaces/archiver.ts b/yarn-project/stdlib/src/interfaces/archiver.ts index aec489278b8f..362da31e3a88 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.ts @@ -68,6 +68,13 @@ export type ArchiverSpecificConfig = { /** Skip pruning orphan proposed blocks that have no matching proposed checkpoint. */ skipOrphanProposedBlockPruning?: boolean; + + /** + * Preload the standard contracts (AuthRegistry, PublicChecks, HandshakeRegistry) into the contract store at block 0. + * For test environments only: it must only be set when genesis also seeds the matching registration/deployment + * nullifiers, otherwise a later on-chain publish of a preloaded class would collide with the block-0 preload. + */ + testPreloadStandardContracts?: boolean; }; export const ArchiverSpecificConfigSchema = z.object({ @@ -82,6 +89,7 @@ export const ArchiverSpecificConfigSchema = z.object({ skipPromoteProposedCheckpointDuringL1Sync: z.boolean().optional(), orphanPruneNoProposalTolerance: schemas.Integer.optional(), skipOrphanProposedBlockPruning: z.boolean().optional(), + testPreloadStandardContracts: z.boolean().optional(), }); export type ArchiverApi = Omit< diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 47e8be717696..d9e34c0df90b 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -727,6 +727,7 @@ __metadata: "@aztec/l1-artifacts": "portal:../../l1-contracts/l1-artifacts" "@aztec/noir-protocol-circuits-types": "workspace:^" "@aztec/protocol-contracts": "workspace:^" + "@aztec/standard-contracts": "workspace:^" "@aztec/stdlib": "workspace:^" "@aztec/telemetry-client": "workspace:^" "@jest/globals": "npm:^30.0.0" From 075327eb6a509d19d9bf1b79ecd03d781e6f0b81 Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:24:58 -0400 Subject: [PATCH 10/35] feat(prover): revive cancelled proving jobs from a persisted aborted state (#24578) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v5 version of #24558 (same change, based on `merge-train/spartan-v5`). ## Problem When the prover broker cancelled a proving job it settled the job as a cached `{ status: 'rejected', reason: 'Aborted' }` result — in memory and in the database. Because a job's id is a deterministic hash of its inputs, any later enqueue of the same job returned that cached abort (`Cached proving job … Not enqueuing again`) and the facade failed the job with `Aborted`. So once a proof was aborted it stayed aborted, and a retry — a fresh epoch attempt after a reorg/failure, or a re-orchestration after a restart — got the cached abort instead of re-proving. A planned mid-epoch redeploy hit this and turned into a testnet epoch prune. ## Fix Cancellation is now its own first-class, **revivable** state: - A new `aborted` proving-job status (`ProvingJobStatus` / `ProvingJobSettledResult`). `cancelProvingJob` records it, notifies the current waiter, and **persists** it (`ProvingBrokerDatabase.setProvingJobAborted`). - `enqueueProvingJob` **revives** an aborted job: when the producer re-requests it, the broker clears the aborted state — in memory and, via the new `deleteProvingJobResult`, in the database — and re-enqueues it as if new. So a restart mid-revival keeps the job pending rather than resurrecting the abort. Genuine agent errors and timeouts still settle as terminal `rejected` results. Net effect: cancelling frees the job cleanly and records why, but re-requesting the same proof always brings it back to life — in the same process or after a restart — so an abort can never permanently block an epoch from proving. The stacked PR (prover-node clean-shutdown, v5) stops a clean prover-node shutdown from cancelling its in-flight jobs in the first place. ## Tests `proving_broker.test.ts` (both DB variants): cancel leaves a job `aborted`; re-request revives it and it completes; the aborted state persists across a restart; and a revived job survives a restart mid-revival as pending (not aborted). Note: not run locally in this session — the prover-client suite needs a full noir/wasm bootstrap that wasn't available here — so relied on CI. --- *Created by [claudebox](https://claudebox.work/v2/sessions/6fa5e242ed9ceae7) · group: `slackbot`* --------- Co-authored-by: Phil Windle (cherry picked from commit a4e88ca37c3b264c63aa2d402d5b74370e456bb3) --- .../proving_broker/broker_prover_facade.ts | 2 + .../src/proving_broker/proving_broker.test.ts | 134 +++++++++++++++++- .../src/proving_broker/proving_broker.ts | 74 +++++++--- .../proving_broker/proving_broker_database.ts | 16 +++ .../broker_persisted_database.test.ts | 22 +++ .../proving_broker_database/memory.ts | 10 ++ .../proving_broker_database/persisted.ts | 45 +++++- .../stdlib/src/interfaces/proving-job.ts | 12 ++ 8 files changed, 290 insertions(+), 25 deletions(-) diff --git a/yarn-project/prover-client/src/proving_broker/broker_prover_facade.ts b/yarn-project/prover-client/src/proving_broker/broker_prover_facade.ts index a2a128ce8660..acb80833f38f 100644 --- a/yarn-project/prover-client/src/proving_broker/broker_prover_facade.ts +++ b/yarn-project/prover-client/src/proving_broker/broker_prover_facade.ts @@ -288,6 +288,8 @@ export class BrokerCircuitProverFacade implements ServerCircuitProver { } } else if (result.status === 'rejected') { return { success: false, reason: result.reason }; + } else if (result.status === 'aborted') { + return { success: false, reason: 'Aborted' }; } else { throw new Error(`Unexpected proving job status ${result.status}`); } diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts index 4652413271d4..ca5cb0be1851 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts @@ -201,7 +201,7 @@ describe.each([ await assertJobStatus(id, 'in-queue'); await broker.cancelProvingJob(id); - await assertJobStatus(id, 'rejected'); + await assertJobStatus(id, 'aborted'); }); it('cancels jobs in-progress', async () => { @@ -216,7 +216,135 @@ describe.each([ await broker.getProvingJob(); await assertJobStatus(id, 'in-progress'); await broker.cancelProvingJob(id); - await assertJobStatus(id, 'rejected'); + await assertJobStatus(id, 'aborted'); + }); + + it('revives an aborted job when its producer re-requests it', async () => { + const provingJob: ProvingJob = { + id: makeRandomProvingJobId(), + type: ProvingRequestType.PARITY_BASE, + epochNumber: EpochNumber(1), + inputsUri: makeInputsUri(), + }; + + await broker.enqueueProvingJob(provingJob); + await broker.getProvingJob(); + await assertJobStatus(provingJob.id, 'in-progress'); + + await broker.cancelProvingJob(provingJob.id); + await assertJobStatus(provingJob.id, 'aborted'); + + // Re-requesting the same job (a retry without a restart) revives it rather than returning the + // cached abort. The job stays cached through the revive, so the start-of-call status is + // 'in-queue', and it can then be completed. + await expect(broker.enqueueProvingJob(provingJob)).resolves.toEqual({ status: 'in-queue' }); + await assertJobStatus(provingJob.id, 'in-queue'); + const returnedJob = await broker.getProvingJob({ allowList: [ProvingRequestType.PARITY_BASE] }); + expect(returnedJob?.job).toEqual(provingJob); + + const retryValue = makeOutputsUri(); + await broker.reportProvingJobSuccess(provingJob.id, retryValue); + await assertJobStatus(provingJob.id, 'fulfilled'); + }); + + it('persists the aborted state across a restart and revives on re-request', async () => { + const provingJob: ProvingJob = { + id: makeRandomProvingJobId(), + type: ProvingRequestType.PARITY_BASE, + epochNumber: EpochNumber(1), + inputsUri: makeInputsUri(), + }; + + await broker.enqueueProvingJob(provingJob); + await broker.cancelProvingJob(provingJob.id); + await assertJobStatus(provingJob.id, 'aborted'); + + // A deploy restarts both the prover node and the broker. The aborted state is persisted, so it + // survives the restart rather than being lost or restored as a permanent rejection. + await broker.stop(); + broker = new ProvingBroker(database, { + proverBrokerJobTimeoutMs: jobTimeoutMs, + proverBrokerPollIntervalMs: brokerIntervalMs, + proverBrokerJobMaxRetries: maxRetries, + proverBrokerMaxEpochsToKeepResultsFor: 1, + proverBrokerDebugReplayEnabled: false, + }); + await broker.start(); + await assertJobStatus(provingJob.id, 'aborted'); + + // The prover re-requests the job after the restart, which revives it (returning the cached + // 'in-queue' status), and it can be completed. + await expect(broker.enqueueProvingJob(provingJob)).resolves.toEqual({ status: 'in-queue' }); + await assertJobStatus(provingJob.id, 'in-queue'); + const returnedJob = await broker.getProvingJob({ allowList: [ProvingRequestType.PARITY_BASE] }); + expect(returnedJob?.job).toEqual(provingJob); + + const value = makeOutputsUri(); + await broker.reportProvingJobSuccess(provingJob.id, value); + await assertJobStatus(provingJob.id, 'fulfilled'); + }); + + it('persists the revived (non-aborted) state, so a restart mid-revival stays revived', async () => { + const provingJob: ProvingJob = { + id: makeRandomProvingJobId(), + type: ProvingRequestType.PARITY_BASE, + epochNumber: EpochNumber(1), + inputsUri: makeInputsUri(), + }; + + await broker.enqueueProvingJob(provingJob); + await broker.cancelProvingJob(provingJob.id); + await assertJobStatus(provingJob.id, 'aborted'); + + // Revive the job but do not complete it before the broker restarts. + await broker.enqueueProvingJob(provingJob); + await assertJobStatus(provingJob.id, 'in-queue'); + + await broker.stop(); + broker = new ProvingBroker(database, { + proverBrokerJobTimeoutMs: jobTimeoutMs, + proverBrokerPollIntervalMs: brokerIntervalMs, + proverBrokerJobMaxRetries: maxRetries, + proverBrokerMaxEpochsToKeepResultsFor: 1, + proverBrokerDebugReplayEnabled: false, + }); + await broker.start(); + + // Reviving cleared the persisted aborted state, so the job comes back pending rather than + // aborted and keeps being proven without needing another re-request. + await assertJobStatus(provingJob.id, 'in-queue'); + }); + + it('revives once when the producer re-requests an aborted job concurrently', async () => { + const provingJob: ProvingJob = { + id: makeRandomProvingJobId(), + type: ProvingRequestType.PARITY_BASE, + epochNumber: EpochNumber(1), + inputsUri: makeInputsUri(), + }; + + await broker.enqueueProvingJob(provingJob); + await broker.cancelProvingJob(provingJob.id); + await assertJobStatus(provingJob.id, 'aborted'); + + // Two overlapping re-requests race through the revive. jobsCache stays populated as the enqueue + // lock throughout, so exactly one of them re-enqueues the job and both observe it as in-queue + // rather than the stale abort. The revived job is then delivered once and completes normally. + const [first, second] = await Promise.all([ + broker.enqueueProvingJob(provingJob), + broker.enqueueProvingJob(provingJob), + ]); + expect(first).toEqual({ status: 'in-queue' }); + expect(second).toEqual({ status: 'in-queue' }); + await assertJobStatus(provingJob.id, 'in-queue'); + + const returnedJob = await broker.getProvingJob({ allowList: [ProvingRequestType.PARITY_BASE] }); + expect(returnedJob?.job).toEqual(provingJob); + await assertJobStatus(provingJob.id, 'in-progress'); + await expect(broker.getProvingJob({ allowList: [ProvingRequestType.PARITY_BASE] })).resolves.toBeUndefined(); + + await broker.reportProvingJobSuccess(provingJob.id, makeOutputsUri()); + await assertJobStatus(provingJob.id, 'fulfilled'); }); it('returns job result if successful', async () => { @@ -523,7 +651,7 @@ describe.each([ await broker.getProvingJob(); await assertJobStatus(id, 'in-progress'); await broker.cancelProvingJob(id); - await assertJobStatus(id, 'rejected'); + await assertJobStatus(id, 'aborted'); const id2 = makeRandomProvingJobId(); await broker.enqueueProvingJob({ diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.ts index 27364938d5e1..d1ab4342cf8c 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.ts @@ -272,15 +272,40 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr async #enqueueProvingJob(job: ProvingJob): Promise { // We return the job status at the start of this call - const jobStatus = this.#getProvingJobStatus(job.id); + let jobStatus = this.#getProvingJobStatus(job.id); if (this.jobsCache.has(job.id)) { const existing = this.jobsCache.get(job.id); assert.deepStrictEqual(job, existing, 'Duplicate proving job ID'); - this.logger.warn(`Cached proving job id=${job.id} epochNumber=${job.epochNumber}. Not enqueuing again`, { - provingJobId: job.id, - }); - this.instrumentation.incCachedJobs(job.type); - return jobStatus; + + if (this.resultsCache.get(job.id)?.status === 'aborted') { + // The producer is re-requesting a job it previously cancelled: revive it rather than + // returning the cached abort, clearing the aborted state in memory and in the database so the + // revival survives a restart. + // + // Concurrency model: `jobsCache` is the enqueue lock. Every path that puts a job on the queue + // populates `jobsCache` *synchronously, before its first await* (see the "New proving job" + // block below), so a second concurrent enqueue of the same id observes the entry at the top + // of this method and takes a cached, no-op branch instead of enqueuing a duplicate. The revive + // must keep holding that lock: we tear down the settled state and re-set `jobsCache` in a + // single synchronous span (no await in between), and only then await the database. Because a + // concurrent re-request can only interleave at that await — by which point `jobsCache` is + // populated again and the aborted result is gone — it falls into the cached branch and no-ops, + // so the job is enqueued exactly once. (`cleanUpProvingJobState` also drops the promise, which + // was resolved with the abort, so `enqueueJobInternal` below mints a fresh one for the retry.) + this.logger.info(`Reviving aborted proving job id=${job.id} epochNumber=${job.epochNumber}`, { + provingJobId: job.id, + }); + this.cleanUpProvingJobState([job.id]); + this.jobsCache.set(job.id, job); + await this.database.deleteProvingJobResult(job.id); + jobStatus = this.#getProvingJobStatus(job.id); + } else { + this.logger.warn(`Cached proving job id=${job.id} epochNumber=${job.epochNumber}. Not enqueuing again`, { + provingJobId: job.id, + }); + this.instrumentation.incCachedJobs(job.type); + return jobStatus; + } } if (this.isJobStale(job)) { @@ -306,15 +331,35 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr } async #cancelProvingJob(id: ProvingJobId): Promise { - if (!this.jobsCache.has(id)) { + const job = this.jobsCache.get(id); + if (!job) { this.logger.warn(`Can't cancel a job that doesn't exist id=${id}`, { provingJobId: id }); return; } - // notify listeners of the cancellation - if (!this.resultsCache.has(id)) { - this.logger.info(`Cancelling job id=${id}`, { provingJobId: id }); - await this.#reportProvingJobError(id, 'Aborted', false, undefined, true); + // Leave jobs that have already settled (completed or failed) alone: those results are terminal. + if (this.resultsCache.has(id)) { + return; + } + + this.logger.info(`Cancelling job id=${id}`, { provingJobId: id }); + this.inProgress.delete(id); + + // Record the cancellation as its own settled state and persist it, so it survives a restart and + // notifies the current waiter. Unlike a completion or failure this is not terminal: re-enqueuing + // the same job id revives it (see #enqueueProvingJob), so the abort never permanently blocks the + // proof. + const result: ProvingJobSettledResult = { status: 'aborted' }; + this.resultsCache.set(id, result); + this.promises.get(id)?.resolve(result); + this.completedJobNotifications.push(id); + this.instrumentation.incAbortedJobs(job.type); + + try { + await this.database.setProvingJobAborted(id); + } catch (saveErr) { + this.logger.error(`Failed to save proving job aborted status id=${id}`, saveErr, { provingJobId: id }); + throw saveErr; } } @@ -401,7 +446,6 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr err: string, retry = false, filter?: ProvingJobFilter, - aborted = false, ): Promise { const info = this.inProgress.get(id); const item = this.jobsCache.get(id); @@ -462,11 +506,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr this.promises.get(id)!.resolve(result); this.completedJobNotifications.push(id); - if (aborted) { - this.instrumentation.incAbortedJobs(item.type); - } else { - this.instrumentation.incRejectedJobs(item.type); - } + this.instrumentation.incRejectedJobs(item.type); if (info) { const duration = this.msTimeSource() - info.startedAt; this.instrumentation.recordJobDuration(item.type, duration); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_database.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_database.ts index a2e0fa070077..c91112df9eb2 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_database.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_database.ts @@ -38,6 +38,22 @@ export interface ProvingBrokerDatabase { */ setProvingJobError(id: ProvingJobId, err: string): Promise; + /** + * Records that a proof request was cancelled. Unlike a result or error this is not terminal: + * re-enqueuing the same job id revives it, so the aborted state can survive a restart without + * permanently blocking the proof. + * @param id - The ID of the cancelled proof request + */ + setProvingJobAborted(id: ProvingJobId): Promise; + + /** + * Clears any stored result for a proof request, returning it to the pending state while keeping + * the job itself. Used when reviving an aborted job so the revival is persisted and a restart + * cannot resurrect the stale aborted state. + * @param id - The ID of the proof request whose result should be cleared + */ + deleteProvingJobResult(id: ProvingJobId): Promise; + /** * Closes the database */ diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_database/broker_persisted_database.test.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_database/broker_persisted_database.test.ts index 2f921e75090a..3cd951fcc6db 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_database/broker_persisted_database.test.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_database/broker_persisted_database.test.ts @@ -132,6 +132,28 @@ describe('ProvingBrokerPersistedDatabase', () => { } }); + it('keeps a queued result-delete ordered after the writes it follows', async () => { + const id = makeRandomProvingJobId(EpochNumber(42)); + const job: ProvingJob = { + id, + epochNumber: EpochNumber(42), + type: ProvingRequestType.PARITY_BASE, + inputsUri: makeInputsUri(), + }; + await db.addProvingJob(job); + + // Abort the job and then delete its result without awaiting the abort first — mimicking a revive + // that re-requests a job whose aborted write may not have flushed yet. Because the delete rides + // the same write queue, it stays ordered after the abort and the result ends up cleared. If it + // were applied out of order the abort would resurrect on reload. + const abortWrite = db.setProvingJobAborted(id); + const deleteWrite = db.deleteProvingJobResult(id); + await Promise.all([abortWrite, deleteWrite]); + + const allJobs = await toArray(db.allProvingJobs()); + expect(allJobs).toEqual([[job, undefined]]); + }); + it('can add items over multiple epochs', async () => { const numJobs = 5; const startEpoch = 12; diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_database/memory.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_database/memory.ts index 317798180cac..0456023b16b1 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_database/memory.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_database/memory.ts @@ -36,6 +36,16 @@ export class InMemoryBrokerDatabase implements ProvingBrokerDatabase { return Promise.resolve(); } + setProvingJobAborted(id: ProvingJobId): Promise { + this.results.set(id, { status: 'aborted' }); + return Promise.resolve(); + } + + deleteProvingJobResult(id: ProvingJobId): Promise { + this.results.delete(id); + return Promise.resolve(); + } + deleteProvingJobs(ids: ProvingJobId[]): Promise { for (const id of ids) { this.jobs.delete(id); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts index 021a1e64c84c..3dd2686583ed 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts @@ -41,7 +41,11 @@ class SingleEpochDatabase { return this.store.estimateSize(); } - async batchWrite(jobs: ProvingJob[], results: Array<[ProvingJobId, ProvingJobSettledResult]>) { + async batchWrite( + jobs: ProvingJob[], + results: Array<[ProvingJobId, ProvingJobSettledResult]>, + deletedResults: ProvingJobId[] = [], + ) { await this.store.transactionAsync(async () => { for (const job of jobs) { await this.jobs.set(job.id, jsonStringify(job)); @@ -49,6 +53,11 @@ class SingleEpochDatabase { for (const [id, result] of results) { await this.jobResults.set(id, jsonStringify(result)); } + // Deletes are applied after sets so that, if a set and a delete for the same id land in one + // batch (e.g. an abort immediately followed by a revive's result-delete), the delete wins. + for (const id of deletedResults) { + await this.jobResults.delete(id); + } }); } @@ -66,6 +75,11 @@ class SingleEpochDatabase { await this.jobResults.set(id, jsonStringify(result)); } + async setProvingJobAborted(id: ProvingJobId): Promise { + const result: ProvingJobSettledResult = { status: 'aborted' }; + await this.jobResults.set(id, jsonStringify(result)); + } + async setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise { const result: ProvingJobSettledResult = { status: 'fulfilled', value }; await this.jobResults.set(id, jsonStringify(result)); @@ -80,10 +94,19 @@ class SingleEpochDatabase { } } +/** + * An item queued for a batched write. A {@link ProvingJob} adds/updates a job; a `[id, result]` + * tuple sets a job's settled result; a `[id, undefined]` tuple deletes a job's result. Deletes ride + * the same queue as writes so they stay FIFO-ordered against them: a revive's result-delete must + * land after the abort that preceded it, otherwise a still-queued abort could flush later and + * resurrect the cancelled state on disk. + */ +type BrokerWrite = ProvingJob | [ProvingJobId, ProvingJobSettledResult | undefined]; + export class KVBrokerDatabase implements ProvingBrokerDatabase { private metrics: LmdbMetrics; - private batchQueue: BatchQueue; + private batchQueue: BatchQueue; public readonly tracer: Tracer; @@ -112,12 +135,16 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase { } // exposed for testing - public async commitWrites(items: Array, epochNumber: number) { + public async commitWrites(items: Array, epochNumber: number) { const jobsToAdd = items.filter((item): item is ProvingJob => 'id' in item); - const resultsToAdd = items.filter((item): item is [ProvingJobId, ProvingJobSettledResult] => Array.isArray(item)); + const resultOps = items.filter((item): item is [ProvingJobId, ProvingJobSettledResult | undefined] => + Array.isArray(item), + ); + const resultsToSet = resultOps.filter((op): op is [ProvingJobId, ProvingJobSettledResult] => op[1] !== undefined); + const resultsToDelete = resultOps.filter(op => op[1] === undefined).map(([id]) => id); const db = await this.getEpochDatabase(EpochNumber(epochNumber)); - await db.batchWrite(jobsToAdd, resultsToAdd); + await db.batchWrite(jobsToAdd, resultsToSet, resultsToDelete); } private async estimateSize() { @@ -207,6 +234,14 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase { return this.batchQueue.put([id, { status: 'rejected', reason }], getEpochFromProvingJobId(id)); } + setProvingJobAborted(id: ProvingJobId): Promise { + return this.batchQueue.put([id, { status: 'aborted' }], getEpochFromProvingJobId(id)); + } + + deleteProvingJobResult(id: ProvingJobId): Promise { + return this.batchQueue.put([id, undefined], getEpochFromProvingJobId(id)); + } + setProvingJobResult(id: ProvingJobId, value: ProofUri): Promise { return this.batchQueue.put([id, { status: 'fulfilled', value }], getEpochFromProvingJobId(id)); } diff --git a/yarn-project/stdlib/src/interfaces/proving-job.ts b/yarn-project/stdlib/src/interfaces/proving-job.ts index f9bdc3a06657..a84e82b9af2b 100644 --- a/yarn-project/stdlib/src/interfaces/proving-job.ts +++ b/yarn-project/stdlib/src/interfaces/proving-job.ts @@ -441,9 +441,20 @@ export const ProvingJobRejectedResult = z.object({ }); export type ProvingJobRejectedResult = z.infer; +/** + * The result of a job that was cancelled by its producer. Unlike a fulfilled or rejected result it + * is not truly terminal: re-enqueuing the same job id revives it, so an abort never permanently + * blocks a proof from being produced. + */ +export const ProvingJobAbortedResult = z.object({ + status: z.literal('aborted'), +}); +export type ProvingJobAbortedResult = z.infer; + export const ProvingJobSettledResult = z.discriminatedUnion('status', [ ProvingJobFulfilledResult, ProvingJobRejectedResult, + ProvingJobAbortedResult, ]); export type ProvingJobSettledResult = z.infer; @@ -453,5 +464,6 @@ export const ProvingJobStatus = z.discriminatedUnion('status', [ z.object({ status: z.literal('not-found') }), ProvingJobFulfilledResult, ProvingJobRejectedResult, + ProvingJobAbortedResult, ]); export type ProvingJobStatus = z.infer; From 0cd9ab1387cf8fc11fc4376767436641aff219f8 Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Wed, 8 Jul 2026 09:46:15 -0400 Subject: [PATCH 11/35] chore: add writing-e2e-tests skill (#24597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `yarn-project/.claude/skills/writing-e2e-tests/SKILL.md`: a skill that gives an agent everything it needs to place, structure, and write a robust e2e test with little guidance. Written against the v5-next e2e layout (category directories with per-category READMEs and context classes). ## What it covers - **Step 0 decision ladder**: unit test → new expectation in an existing test → new `it` in an existing suite → new file on an existing context/harness → new standalone test. - **Where to place the test**: recap of the category directories (`automine/` via `AutomineTestContext`, `single-node/` via `setupWithProver`/`setupBlockProducer`, `multi-node/` via `MultiNodeTestContext` on the mock-gossip bus, `p2p/` via `P2PNetworkTest` on real libp2p, plus `composed/`, `infra/`, `spartan/`, `bench/`), the multi-node-vs-p2p decision rule, pointers to the per-category READMEs as the authoritative reference, file naming (`describe` matches path, header comment), automatic CI registration through `bootstrap.sh` globs, `.parallel.test.ts` semantics, and jest/bash timeout sync. - **Setup reuse**: the three layers (category contexts → domain harnesses → root `SetupOptions`), the standard suite file shape, one-environment-per-file, preset spread order, guarded teardown. - **Readability**: intent-only test bodies, the named-waiter surface (`fixtures/wait_helpers.ts`, context waiters, `ChainMonitor`), shared helpers and co-located `*_test_helpers.ts`, simulators with `afterEach` checks, shared error constants. - **Speed**: themes distilled from the recent `test(e2e)`/`perf(e2e)`/`chore(e2e)` speedup PRs and their tracking issues — avoid unneeded setup, genesis seeding over setup txs, `BatchCall` batching over `Promise.all` overlap (with the PXE-serialization caveat), warping dead waits with `markProvenAndWarp`/`warpWithSequencersPaused` ("an honest wait beats a flaky warp"), named timing profiles over ad-hoc cadences, and measuring with the `testSpan` instrumentation + `track-e2e-times` skill before optimizing. - **Flakiness**: twelve golden rules synthesized from six months of deflake PRs and the accumulated flaky-test gotchas (poll-don't-sleep via named waiters, receipt-anchored assertions, mined ≠ checkpointed ≠ proven, `syncChainTip` tag awareness, pipelining timing margins + `findSlotsWithProposers`, invariant-not-exact-value assertions, L1 account/nonce and port hygiene, freeze-L1-across-restarts, gossip-mesh readiness via `runGossipScenario`, fee padding, determinism, honest timeouts), plus `deflaker.sh` validation and `.test_patterns.yml` as a last resort. - Final pre-ship checklist. All helper/API names cited were verified to exist on `merge-train/spartan-v5` (`AutomineTestContext.setup`, `setupWithProver`/`setupBlockProducer`, `MOCK_GOSSIP_MULTI_VALIDATOR_OPTS`, `wait_helpers.ts` waiters, `warpWithSequencersPaused`, `markProvenAndWarp`, `testSpan`/`TEST_TIMING_FILE`, `runGossipScenario`, `waitForP2PMeshConnectivity`, `findSlotsWithProposers`, `getPaddedMaxFeesPerGas`, etc.). Genesis-prefill techniques that only exist in still-open speedup PRs are described as themes to look for rather than as concrete APIs. (cherry picked from commit 4c2b6d8a6d1c56d436ad47eedff18d23f98487cf) --- .../.claude/skills/writing-e2e-tests/SKILL.md | 327 ++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 yarn-project/.claude/skills/writing-e2e-tests/SKILL.md diff --git a/yarn-project/.claude/skills/writing-e2e-tests/SKILL.md b/yarn-project/.claude/skills/writing-e2e-tests/SKILL.md new file mode 100644 index 000000000000..c62e1c156758 --- /dev/null +++ b/yarn-project/.claude/skills/writing-e2e-tests/SKILL.md @@ -0,0 +1,327 @@ +--- +name: writing-e2e-tests +description: How to write end-to-end tests in yarn-project/end-to-end. Use when adding e2e coverage for a feature, creating a new e2e test or suite, or deciding where an e2e test should live. Covers the test categories (automine, single-node, multi-node, p2p, composed), setup reuse, readability conventions, speed techniques, and flakiness prevention. +--- + +# Writing E2E Tests + +E2E tests live in `yarn-project/end-to-end/src`. They spin up a real stack — anvil, an Aztec node +(archiver, world state, sequencer, p2p), a PXE-backed `TestWallet`, and optionally prover and +validator nodes — so they are the most expensive tests in the repo. Every decision below follows +from that: reuse setup, pick the cheapest category that exercises the feature, and make the test +robust against timing jitter because CI machines are slow and noisy. + +For debugging a failing e2e test, use the `debug-e2e` skill; for profiling where suite time goes, +`track-e2e-times`. For unit tests, use `unit-test-implementation` — and prefer a unit test whenever +the feature doesn't genuinely need the full stack. + +## Step 0: do you need a new test at all? + +Work down this ladder and stop at the first step that fits. Each step down costs CI minutes forever. + +1. **A unit test in the owning package.** If the behavior is observable without a live chain, it's + not an e2e test. +2. **A new expectation in an existing test.** If an existing test already drives the code path + (e.g. it sends the tx type you care about), add an `expect` there instead of paying another + setup. Grep for the contract method or subsystem you're touching. +3. **A new `it` in an existing suite.** Suites share one setup in `beforeAll`; a new `it` costs + seconds, a new file costs minutes. +4. **A new file on an existing category context or domain harness** (e.g. `AutomineTestContext`, + `setupWithProver`, `TokenContractTest`, `FeesTest`, `MultiNodeTestContext`). +5. **A brand-new standalone test.** Last resort — justified when the feature needs a setup shape + no existing suite has. + +## Where to place the test + +The top level of `src/` groups tests **by node topology**; the second level names the primary +behavior under test. Each category directory has a `README.md` describing its base class, setup +factories, helper surface, and subfolders — **read the README of the category you pick before +writing**; it is the authoritative, up-to-date reference and this skill only summarizes it. + +### Categories + +Pick the **cheapest category whose machinery your feature actually needs**. Cost and flake risk +increase down the table. + +| Category | Context / entrypoint | Use for | +|---|---|---| +| `automine/` | `AutomineTestContext.setup({ numberOfAccounts })` (`automine_test_context.ts`) | Contract or protocol behavior that doesn't depend on real block-building or consensus: tokens, accounts, authwits, notes/events/effects, deploys, simulation. Deterministic and fast: the `AutomineSequencer` builds one block per submitted tx, publishes synchronously, no committee/prover/validator. | +| `single-node/` | `setupWithProver(opts)` or `setupBlockProducer(opts)` (`single-node/setup.ts`, over `SingleNodeTestContext`) | One production sequencer, no committee: block building, sequencer config/governance signalling, fees, cross-chain messaging, world-state sync, the proving/epoch lifecycle, partial proofs, L1 reorgs, recovery. `setupWithProver` adds a fake in-process prover; `setupBlockProducer` has no prover (and points the PXE at the `proposed` tip). Real Barretenberg proofs live in `single-node/prover/` on `FullProverTest`. | +| `multi-node/` | `MultiNodeTestContext` (extends `SingleNodeTestContext`) + presets in `multi_node_test_context.ts` | N validators on an **in-memory `MockGossipSubNetwork` bus** (no real libp2p): committee block production, attestations, invalid-attestation handling, HA pairs, slashing/offense detection, governance upgrades. Presets: `buildMockGossipValidators(n)`, `MOCK_GOSSIP_MULTI_VALIDATOR_OPTS`, `SLASHER_ENABLED_MULTI_VALIDATOR_OPTS`, `setupHaPairs`. | +| `p2p/` | `P2PNetworkTest` (`p2p/p2p_network.ts`) + `runGossipScenario` (`p2p/shared.ts`) | **Real libp2p only**: peer discovery/rediscovery, gossip mesh formation, req/resp, preferred-peer topologies, peer auth. Slowest and most flake-prone; nodes bind fixed ports, so two p2p files can never run concurrently locally. | +| `composed/` | docker-compose against a running network (`scripts/run_test.sh compose`) | The packaged sandbox/network as users see it: persistence, cheat codes, tutorials, uniswap, HA, web3signer. Also `guides/` for docs examples. | +| `infra/`, `spartan/`, `bench/` | see their READMEs | Deployment/ops smoke tests, k8s network tests, and benchmarks (see the `adding-benchmarks` skill) — not homes for feature coverage. | + +The decision that trips people up most: **multi-node vs p2p**. If the subject — proposals, +attestations, checkpointing, pruning/recovery, offense detection — is faithfully reproduced by the +mock-gossip bus, it belongs in `multi-node/`, which is far cheaper. Only reach for `p2p/` when the +behavior genuinely cannot be reproduced without real networking. + +Also cheaper than jumping categories: `setup()` options can bend a category upward — +`startProverNode: true`, `skipInitialSequencer: true`, and `mockGossipSubNetwork: true` give you +extra nodes without real libp2p. + +### File placement and CI registration + +- Second-level folders name the behavior under test (`token/`, `proving/`, `slashing/`), not the + shared setup. A new folder is created only when it earns its keep: a shared harness, a coherent + domain of several files. Otherwise the file lives flat in the category. +- Each file has a **single top-level `describe` named to match its path** + (`describe('automine/token/transfer', ...)`), and starts with a short header comment describing + the coverage and the setup shape (see `automine/token/transfer.test.ts` for the pattern). +- A co-located `setup.ts` in the subfolder holds shared timing profiles/option wiring (e.g. + `single-node/l1-reorgs/setup.ts`, `multi-node/slashing/setup.ts`); domain harnesses are + co-located `*_test.ts` files (not `.test.ts`, so jest doesn't run them). +- CI picks up new files **automatically**: `end-to-end/bootstrap.sh` `test_cmds` globs each + category. Each file runs as its own isolated job with a default `TIMEOUT=20m`; if your suite + legitimately needs more, add a per-test override in the `case` block there — and keep it in sync + with the file's `jest.setTimeout`. +- `*.parallel.test.ts` marks a file with more than one top-level `it`: CI extracts each `it` title + and runs it as a **separate job** (`jest -t ''`). Every `it` must pass in isolation — no + cross-test state — and titles must be unique and stable (they become job/container names). +- `*.notest.ts` parks a test without running it (prefer fixing or deleting). +- Jest gives each test/hook 300s (`--testTimeout=300000` in `test:e2e`). Set an explicit + `jest.setTimeout(...)` at the top of the `describe` when setup or waits legitimately exceed it — + and only then (see Flakiness below). + +## Setup reuse + +**Search for an existing setup before building one.** The layers, outermost first: + +1. **Category context classes** (table above) own the environment: anvil + L1 deploy, node + spawning, the `ChainMonitor`, waiters, and teardown. Don't call the root `setup()` + (`fixtures/setup.ts`) directly from a new test — go through the category's context/factory, and + pass options through it. +2. **Domain harnesses** extend a context with domain state and opt-in setup phases: + `automine/token/token_contract_test.ts` (`TokenContractTest`: `applyBaseSnapshots()`, + `applyMint()`), `automine/token/blacklist_token_contract_test.ts`, + `single-node/fees/fees_test.ts` (`FeesTest`: `applyBaseSetup()`, `applyFPCSetup()`, + `applyFundAliceWithBananas()`, ...), `single-node/cross-chain/cross_chain_messaging_test.ts`, + `single-node/prover/` (`FullProverTest`), `multi-node/slashing/inactivity_setup.ts`. +3. **Root `setup()` options** (`SetupOptions` in `fixtures/setup.ts`) cover most needs without new + code: genesis-funded accounts (`initialFundedAccounts`, `numberOfInitialFundedAccounts`), + `fundSponsoredFPC`, `startProverNode`, `skipInitialSequencer`, validators + (`initialValidators`), custom genesis (`genesisPublicData`), timing (`aztecSlotDuration`, + `ethereumSlotDuration`, `aztecEpochDuration`), `mockGossipSubNetwork`, and any + `AztecNodeConfig` field. Read the type before adding a new option. + +The standard shape of a suite test file: + +```typescript +describe('automine/token/transfer', () => { + const t = new TokenContractTest('transfer'); + let { asset, adminAddress, wallet, otherAddress, tokenSim } = t; + + beforeAll(async () => { + t.applyBaseSnapshots(); + await t.setup(); + await t.applyMint(); + ({ asset, adminAddress, wallet, otherAddress, tokenSim } = t); + }); + + afterAll(() => t.teardown()); + afterEach(async () => { + await t.tokenSim.check(); // model-based invariant check after every test + }); + + it('transfers between accounts', async () => { /* ... */ }); +}); +``` + +Rules of thumb: + +- **One environment per file, set up in `beforeAll`** — never per test. If tests can't share + state, make them not need to (fresh contract instance per test is fine; fresh network per test + is not), or split the file. +- Only apply the setup phases you need — every `apply*` costs txs (and therefore blocks). +- New shared state for several tests → an `apply*` method on the harness (or a new harness + extending the context), so other files can reuse it. +- `afterAll(() => teardown())`, and if a local `teardown` variable is set inside `beforeAll`, + guard it: `afterAll(() => teardown?.())` — if setup throws, an unguarded call masks the real + error with `TypeError: teardown is not a function`. +- When combining a preset with overrides, **spread the preset first** so your overrides win: + `{ ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, aztecEpochDuration: 4 }`. Spreading the preset last + silently reverts your options. + +## Readability + +The test body should read as **intent**: what is executed, what is asserted. Push mechanics into +helpers, preferably shared ones. + +- **Prefer the named waiters over hand-rolled polling.** Node/wallet-level waits live in + `fixtures/wait_helpers.ts` (`waitForBlockNumber`, `waitForProvenBlock`, `waitForNodeCheckpoint`, + `waitForTxs`, `waitForTxStatus`, `waitForPendingTxCount`, `waitForSequencerState`, ...); context + waiters live on `SingleNodeTestContext`/`MultiNodeTestContext` (`waitUntilEpochStarts`, + `waitUntilProvenCheckpointNumber`, `waitForNodeToSync`, `waitForSequencerEvent`, + `waitForAllNodes*`, `findSlotsWithProposers`); L1-side waits on `ChainMonitor` + (`waitUntilCheckpoint`, `waitUntilL2Slot`, `waitUntilL1Timestamp`). A raw + `retryUntil`/`.on`/`sleep` in a test body is a smell — wrap it or find the existing helper. +- **Reuse the shared helpers** before writing inline plumbing: `fixtures/token_utils.ts` + (`deployToken`, `mintTokensToPrivate`, `mintNotes`, `expectTokenBalance`), + `shared/submit-transactions.ts`, `shared/cross_chain_test_harness.ts`, + `fixtures/l1_to_l2_messaging.ts`, `expectMapping` from `fixtures/setup.ts`. Suite-local + assertion helpers go in a co-located file (e.g. `automine/token/token_test_helpers.ts`) — never + duplicated across test bodies. +- **Simulators for stateful suites**: `TokenSimulator` (`src/simulators/`) mirrors expected + balances in memory; an `afterEach` calls `tokenSim.check()` so every test gets full-state + verification without per-test assertion boilerplate. Follow this pattern for new stateful + suites. +- **Expected errors are shared constants**, not inline strings: `U128_UNDERFLOW_ERROR`, + `DUPLICATE_NULLIFIER_ERROR`, `NO_L1_TO_L2_MSG_ERROR`, etc. in `fixtures/fixtures.ts`. Add new + protocol-level error patterns there. +- Destructure the harness once in `beforeAll` so test bodies use plain names (see the example + above). +- Log with the context logger (`t.logger.info('...')`) at phase boundaries of long tests — it is + what makes CI logs debuggable — but don't narrate every line. +- A comment in a test explains a non-obvious *why* (e.g. "proxy makes msg_sender differ from the + note owner to trigger authwit validation"), never *what* the next line does. + +## Speed + +We favor robustness over speed, but e2e minutes are the bottleneck of every CI run. Techniques +that have actually paid off in the ongoing e2e speedup effort: + +1. **Don't create setup you don't need.** The single biggest cost is network + account setup. + Joining an existing suite costs ~0; every avoided account deploy saves a proof + a block. Use + genesis-funded accounts over deploys, the hardcoded schnorr account + (`fixtures/schnorr_hardcoded_account_contract.ts`) when the test doesn't care about identity, + `AutomineTestContext.registerContract(...)` / `TestContract` for contracts usable without an + on-chain deploy, and only the `apply*` phases you need. +2. **Seed state at genesis instead of executing setup txs.** `setup()` options like + `initialFundedAccounts`, `fundSponsoredFPC: true`, and `genesisPublicData` bake state into the + genesis trees for free. Prefer these over bridging/minting/deploying in `beforeAll`. (The + current speedup round extends this: standard-contract registration and FPC funding seeded via + prefilled genesis nullifiers/public data — check `SetupOptions` for `prefilled*`/preload + options and reuse them when available.) +3. **Batch same-sender setup txs into one `BatchCall`** — one proof and one block instead of N. + See `mintNotes` in `fixtures/token_utils.ts`. Note limits apply (e.g. only one contract-class + log per tx, so two contract deploys can't batch). +4. **Overlap independent setup txs with `Promise.all`** — but know the ceiling: the PXE + serializes simulation/proving on one queue, so concurrent sends often land in consecutive + blocks anyway. Batching (one tx) beats overlapping (N txs) when the sender is the same. +5. **Warp over dead waits.** When the test waits for a timestamp/epoch boundary and *nothing + needs to be produced* during the wait, jump: `cheatCodes.warpL2TimeAtLeastTo/By` (L1+L2 + together), `cheatCodes.eth.warp(ts, { resetBlockInterval: true })` (L1 only, for big jumps), + `markProvenAndWarp` on `AutomineTestContext` (marks checkpoints proven first so a long warp + doesn't trip the pruning window), `warpWithSequencersPaused` on `SingleNodeTestContext` (pauses + sequencers across the warp so in-flight jobs don't cascade). **An honest wait beats a flaky + warp**: if sequencers/provers must actually do something during the window (attest, prove, + slash), warping skips the behavior under test — don't convert those. +6. **Don't tighten slot durations ad hoc.** Slot/epoch durations interact with the sequencer + timetable; too-tight cadences are the top historical flake source (see Flakiness). Use the + named timing profiles (category presets, the co-located `setup.ts` profiles) instead of + inventing per-test numbers, and leave slack — CI event loops stall for hundreds of ms + routinely. +7. **Measure before optimizing.** Setup and wait helpers are span-instrumented via `testSpan` + (`fixtures/timing.ts`, enabled by `TEST_TIMING_FILE`); use the `track-e2e-times` skill to get a + ranked breakdown of where the time goes. Wrap new expensive shared helpers in `testSpan` so + they show up. Cite before/after span numbers when proposing a speed change. + +## Flakiness + +A test that fails 1-in-50 runs costs more than it's worth. These are the recurring root causes +from six months of deflake PRs — write the test right the first time. + +### The golden rules + +1. **Never `sleep()` to wait for a state change.** Poll the condition with the named waiters + (Readability above), or `retryUntil(fn, name, timeoutSec, intervalSec)` from + `@aztec/foundation/retry` when no named helper fits. A raw sleep is only acceptable to yield + the event loop, never to "give X time to finish". +2. **Assert against the tx receipt, not the chain tip.** Use `receipt.blockNumber`; never + `getBlock('latest')` right after sending — an empty block/checkpoint may have landed in + between. For `contract.methods.foo().send()`, destructure `{ receipt }`. +3. **Mined ≠ checkpointed ≠ proven.** A tx wait proves a node saw the tx mined — not that the + archiver indexed it, that it survived a reorg, or that it was proven. + - Asserting on archiver/world-state after a tx: wait on the subsystem's own durable marker, + e.g. `waitForNodeCheckpoint(node, target)` or `waitForBlockNumber(node, n, { tag: + 'checkpointed' })`. + - If the test can experience pruning/reorgs (proving, recovery, multi-node — anything stopping + nodes), anchor the PXE to the durable tip: `syncChainTip: 'checkpointed'` in pxe opts, and + use `send({ wait: { waitForStatus: TxStatus.CHECKPOINTED } })` for setup txs that later + assertions depend on. (`setupBlockProducer` deliberately uses `'proposed'` so tests can + assert on fresh blocks — know which one your suite needs.) Classic flake signature: + `Block not found in the node. This might indicate a reorg has occurred`, or a receipt + wait hanging forever. + - **Proven/finalized never advances by itself in most setups**: the `AnvilTestWatcher`'s + auto-prove is dormant once anvil is in interval mining. If the test needs the proven tip to + move, use `markProvenAndWarp` / `cheatCodes.rollup.markAsProven()` or run a prover + (`setupWithProver`, `startProverNode: true`). Otherwise a `while (proven < n)` loop hangs to + wall clock. +4. **Leave timing margin.** Under proposer pipelining the sequencer builds slot N during slot + N-1, so effects land a slot later than naive math suggests, and config injected via + `node.setConfig()` is snapshotted when a job is *constructed* — one slot early. Anchor slot + arithmetic to a fresh boundary (`monitor.waitUntilNextL2Slot()`) before reading `currentSlot`; + target slots with `+3/+4` margin rather than `+2`; when targeting a specific proposer, use + `findSlotsWithProposers` rather than hard-coded slot pairs (the prior pipelined slot must not + share the proposer); after a mid-test `setConfig` of block-gating options, wait for the + sequencer to pick it up before sending dependent txs. Historical top-flake: timetable too + tight — the presets' slot durations exist because validators must simulate, attest, and + publish within them on a loaded CI machine; don't undercut them in a new test. +5. **Don't force `minTxsPerBlock >= 1` under a wall-clock sequencer** unless tx-gated block + production is the behavior under test. It stalls scheduled empty checkpoints and drops txs. +6. **Assert the invariant, not an incidental exact value.** Exact block numbers, exact slots, + exact committee members, and `toBeGreaterThan` off-by-ones are the most common deflake diffs. + If the exact value matters, *derive* it from the receipt/committed header/actual committee; + if the system decides it (which slot a fault lands on, when a prune executes), **discover it + by polling, then assert on the discovered value** — don't hardcode the assumption. For + ramp-up/settle phases, assert a tolerance budget, not zero. When filtering sequencer events, + exclude known-benign failures explicitly (see the existing filters in `watchSequencerEvents` + call sites) rather than asserting no events at all. +7. **Serialize against shared resources.** + - L1 accounts: never reuse mnemonic index 0 (the sequencer's publisher) for a test actor; + take a dedicated unused index via `getPrivateKeyFromIndex(i)` (see + `L1_DIRECT_WRITE_ACCOUNT_INDEX` in `fixtures/fixtures.ts`). Nonce races present as + `nonce too low` / stuck publishers. + - Await the receipt of a prerequisite L1 tx before sending a dependent one. + - Parallel local runs need distinct `ANVIL_PORT`s (the fixture honors the env var); p2p tests + bind fixed UDP/TCP ports, so never run two p2p files at the same time. + - Data directories: use the context's management (`P2PNetworkTest.dataDirFor(label)`), don't + `mkdtemp`/`rmSync` in test files. +8. **Attach listeners before causing the event**, and freeze time across restarts. Listeners + registered after initial sync miss events that fire during sync — if the test stops/recreates + nodes while an L1 deadline approaches, pause anvil mining across the gap and resume + deterministically (set next timestamp + mine) so the transition happens while someone is + listening. Add a fail-fast assertion that the deadline hasn't passed yet. +9. **P2P: connectivity ≠ gossip readiness.** Wait for the gossip mesh + (`waitForP2PMeshConnectivity`; raise `minMeshPeerCount` when a proposal must reach the whole + committee within a slot) before sending txs, or they publish to zero peers and silently + expire. Prefer the `runGossipScenario` skeleton over hand-rolling the bootstrap→nodes→mesh + sequence. Don't over-specify topology (requiring a full clique flakes on one missing edge); + assert the property the test needs. Gossip is not a durable record — late attestations get + rejected by acceptance windows before downstream consumers see them. +10. **Fees evolve between snapshot and inclusion.** Use `getPaddedMaxFeesPerGas` / + `walletMinFeePadding` rather than exact predicted fees, and derive expected committed fees + from the block header, not from a later `getCurrentBaseFees()` call. +11. **Keep the test deterministic.** Mock `Math.random` when the code under test makes random + choices; never mix fake clocks with real sleeps — drive all timing through the fake clock. +12. **Timeouts express expected duration, not hope.** Raise `jest.setTimeout` only when the flow + legitimately takes that long (proving, multiple epochs) and say why; a bumped timeout that + hides a hang just moves the failure to the 20m CI kill. Keep the bootstrap.sh `TIMEOUT` and + `jest.setTimeout` in sync. + +### Before you ship it + +- Run the test repeatedly: `scripts/deflaker.sh yarn workspace @aztec/end-to-end test:e2e ` + (100 runs, stops at first failure). At minimum run it 3-5 times locally, including once under + load. +- Run with verbose logs once and read them: `LOG_LEVEL='info; debug:sequencer,archiver,publisher' + yarn workspace @aztec/end-to-end test:e2e src//.test.ts -t 'test name'`. +- If a known-unfixable external flake remains, the last resort is an entry in `.test_patterns.yml` + (repo root) with a **tightly-scoped `error_regex`** and an owner — it alerts instead of failing + CI. This is for tracked product fragility, not a substitute for fixing the test. + +## Checklist + +- [ ] Couldn't be a unit test, an added expectation, or a new `it` in an existing suite +- [ ] Cheapest category that exercises the feature (automine → single-node → multi-node → p2p); + category README read +- [ ] One environment per file in `beforeAll`, via the category context/factory; preset spread + first; guarded teardown +- [ ] Single top-level `describe` named to match the path; header comment; `.parallel` suffix iff + multiple independent top-level `it`s +- [ ] Test body reads as intent; named waiters and shared helpers; shared error constants +- [ ] No sleeps; receipt-anchored assertions; correct tip tag (`proposed` vs `checkpointed`) +- [ ] No exact-value assertions on system-decided values; timing margin per rule 4 +- [ ] `jest.setTimeout` justified and in sync with bootstrap.sh `TIMEOUT` if overridden +- [ ] Deflaker/local repeat runs pass; verbose-log run reviewed From 8b2dab2e0f10e956749d8f45e217e27137b74e39 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 8 Jul 2026 11:22:53 -0300 Subject: [PATCH 12/35] perf(e2e): overlap and batch setup txs in e2e harnesses (#24569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stacked on #24564; retarget to `merge-train/spartan-v5` after it merges. (Unifies former #24584 into this PR — the two touched the same mint path in `fees_test.ts`, so they now land together.) Restructures the serial setup-tx chains in the fees and cross-chain e2e harnesses so independent setup transactions stop paying one production slot each: independent txs run concurrently, and same-sender calls are batched into a single tx (one proof, one tx, one slot). Part of the round-4/5 e2e speedup effort. ## Commits - **perf(e2e): overlap fees harness setup txs** — after `FeesTest.setup` deploys BananaCoin, the BananaFPC deploy, the SponsoredFPC registration (a PXE registration, no L2 tx), and Alice's banana mints each depend only on the token, so the fees suites (`account_init`, `gas_estimation`, `private_payments`) now run them under a `Promise.all`. The cross-chain harness (`deployAndInitializeTokenAndBridgeContracts`) deploys the L2 token and its bridge concurrently: the bridge takes the token address as a constructor arg, but that address is deterministic before the deploy tx mines (the deploy is now constructed with an explicit `deployer`, same address derivation as the previous send-time lock), and the bridge constructor only stores it without calling into the token. - **perf(e2e): parallelize dual token mints** + **perf(e2e): batch same-sender setup txs into single txs** — `applyFundAliceWithBananas` sent Alice's private and public banana mints back-to-back (measured 100% serial: `tx:mint` busyMs == totalMs on the base). The first commit made them concurrent; the second replaces the concurrency with a single `BatchCall([mint_to_private, mint_to_public])` tx, because measurement showed concurrent same-sender txs still pay *consecutive* slots: the PXE serializes `simulateTx`/`proveTx` on a single queue, so the second tx is only proven after the first and misses the first's slot. Overlapping cannot remove whole slots — only batching can. Both mints are Alice sends on `BananaCoin` touching disjoint balances; the private-balance before/after assertion is preserved, and the public mint's effect is checked by the downstream tests that spend Alice's public bananas. Precedent for the pattern already lives in this tree (`mintNotes` in `fixtures/token_utils.ts`). ## Dependency analysis - FPC deploy needs the token address (constructor arg) → runs after the token deploy, concurrent with the mint batch. - Mints need the token → concurrent with the FPC deploy, and the private/public mints are mutually independent (disjoint balances), so they share one tx. - SponsoredFPC setup is a wallet registration only → free to join the `Promise.all`. - TokenBridge's initializer stores `{token, portal}` in its own storage and never calls the token, so the bridge deploy does not need the token to be mined first. - #24564 invariants preserved: `BANANA_COIN_SALT`/`BANANA_FPC_SALT` + `deployer: alice` are untouched, and the genesis-funded-address assertion in `applyFPCSetup` still runs (and passed locally). ## Same-sender concurrency Concurrent `.send()` calls from the same sender through the same TestWallet are safe: the PXE serializes `simulateTx`/`proveTx` through a `SerialQueue` (`#putInJobQueue`), so simulations never interleave and only the block-inclusion waits overlap. The setup txs pay fees from Alice's public fee-juice balance (`PREEXISTING_FEE_JUICE`) with random tx nonces, so there are no nullifier or note conflicts between the concurrent txs. Verified locally with full `Promise.all` of the send+wait calls; no NO_WAIT fallback or cross-sender split was needed. ## Surveyed and skipped - Batching token class+instance publication: not applicable on this base — `DeployMethod.request()` already merges class publication, instance publication, and the constructor call into a single execution payload sent as one tx. Measured locally, `deploy:token` is a single ~12s (1 slot) occurrence; the ~24s occurrences in CI span data are inclusion latency at CI cadence, not a second tx. - `applyFPCSetup`: a single `deploy:fpc` tx — no same-sender pair inside the phase. Folding the FPC deploy into the mint batch (one class publication + app calls is within the class-log limit) would remove the remaining consecutive slot, but it merges two harness phases that four test files compose differently via `Promise.all`, and deploy-inside-BatchCall is not a proven pattern here — a restructure, not a mechanical merge. Left as a possible follow-up. - `applySponsoredFPCSetup`: no on-chain tx — it only calls `wallet.registerContract` locally; the sponsored FPC is genesis-funded and its class preloaded. Nothing to batch. - Two different contract deploys cannot share a tx (`MAX_CONTRACT_CLASS_LOGS_PER_TX = 1`), which rules out any two-deploy batch (e.g. cross-chain token+bridge) — those stay concurrent rather than batched. ## Local verification (12s slots; CI runs at its own cadence) Overlap commits (fees + cross-chain): - Base (`account_init`, spans on): 5/5 passed; beforeAll 83.5s = `setup:auth-registry` 32.0s + `deploy:token` 12.0s + `tx:mint` 23.9s (busyMs == totalMs, fully serial) + `deploy:fpc` 11.8s. - After overlap: 5/5 passed; beforeAll 72.2s (−11.3s); `deploy:fpc` fully overlapped with the mint window. - `gas_estimation`: 3/3 passed; `private_payments`: 8/8 passed; `token_bridge` (cross-chain overlap): 7/7 passed — `deploy:token` 12.1s and `deploy:bridge` 23.7s ran concurrently (union ≈ 23.7s vs ~48s serial on the CI baseline); `l2_to_l1`: 6/6 passed (regression check on the untouched path). Batch commit (`account_init`, 5/5 green on all runs): | run | tx:mint count | tx:mint busyMs | deploy:fpc busyMs | beforeHooksMs | |-----|---------------|----------------|-------------------|---------------| | before batch | 2 | 23618 | 23713 | 71607 | | run 1 (after) | 1 | 11834 | 24065 | 72372 | | run 2 (after) | 1 | 11807 | 23743 | 71678 | The mint phase drops from two consecutive slots (~23.6s) to one (~11.8s), one proof and one tx fewer per shard. No intermittent failures across any run. ## Measured impact Measured on CI in two steps (each commit group had its own PR run before unification). **Overlap** (PR run 1783394912998666 vs #24564's run 1783375878950264): −162s summed across shards. Per-suite beforeHooks: private_payments.parallel −16.0s (×8), gas_estimation.parallel −15.5s (×3), token_bridge −12.0s, fee_juice_payments −12.0s, account_init −11.5s. The overlap is real but partial: `tx:mint` busyMs/totalMs = 0.68 run-wide — concurrent txs land in consecutive slots because the PXE serializes proving (the finding that motivated the batch commit). Cross-chain `token_bridge` collapses its deploy phase from serial 35.9s to concurrent ~23.4s (union ≈ max, as designed). **Batch** (PR run 1783432285938536 vs 1783394912998666): `tx:mint` 42 → 29 occurrences (one merged per fees shard); busyMs 523s → 431s (−92s of tx/proving work; busyMs == totalMs in the PR run — no overlapping mint txs left to union). Wall-clock lands where the mint is the setup critical path: `failures` −12.0s, `fee_juice_payments` −11.5s (≈ one 12s slot each). Suites where the mint overlaps `applyFPCSetup` are flat (`account_init` −0.4s, `private_payments.parallel` +2s, `gas_estimation.parallel` +3s): `deploy:fpc` is unchanged (13 occurrences, ~2 slots each) and remains their critical path, so removing a slot from the mint branch of the `Promise.all` doesn't move the union. Run-wide sum over common suites: −8s; noise envelope on untouched suites ±20–37s. Net vs #24564's baseline: roughly −185s summed across shards, plus 13 fewer txs/proofs per run (less proving contention through the sequencer). The remaining ~12s/shard lever on the overlapped fees suites is folding the FPC deploy into the same batch, noted above as a follow-up. Fixes A-1406 Fixes A-1409 (cherry picked from commit 1d280af721df1039a38b43b6f65ec34a061c41da) --- .../src/shared/cross_chain_test_harness.ts | 24 +++++++++---------- .../src/single-node/fees/account_init.test.ts | 5 ++-- .../src/single-node/fees/fees_test.ts | 21 ++++++++++++---- .../fees/gas_estimation.parallel.test.ts | 5 ++-- .../fees/private_payments.parallel.test.ts | 9 ++++--- 5 files changed, 38 insertions(+), 26 deletions(-) diff --git a/yarn-project/end-to-end/src/shared/cross_chain_test_harness.ts b/yarn-project/end-to-end/src/shared/cross_chain_test_harness.ts index b2cfe4ac419c..acd4cf12ba1e 100644 --- a/yarn-project/end-to-end/src/shared/cross_chain_test_harness.ts +++ b/yarn-project/end-to-end/src/shared/cross_chain_test_harness.ts @@ -76,19 +76,17 @@ export async function deployAndInitializeTokenAndBridgeContracts( client: l1Client, }); - // deploy l2 token - const { contract: token } = await testSpan('deploy:token', () => - TokenContract.deploy(wallet, owner, 'TokenName', 'TokenSymbol', 18).send({ - from: owner, - }), - ); - - // deploy l2 token bridge and attach to the portal - const { contract: bridge } = await testSpan('deploy:bridge', () => - TokenBridgeContract.deploy(wallet, token.address, tokenPortalAddress).send({ - from: owner, - }), - ); + // Deploy the L2 token and its bridge concurrently so they share a slot: the bridge takes the token + // address as a constructor arg, but that address is known deterministically before the token deploy + // mines, and the bridge's constructor only stores it without calling into the token. + const tokenDeploy = TokenContract.deploy(wallet, owner, 'TokenName', 'TokenSymbol', 18, { deployer: owner }); + const tokenAddress = await tokenDeploy.getAddress(); + const [{ contract: token }, { contract: bridge }] = await Promise.all([ + testSpan('deploy:token', () => tokenDeploy.send({ from: owner })), + testSpan('deploy:bridge', () => + TokenBridgeContract.deploy(wallet, tokenAddress, tokenPortalAddress).send({ from: owner }), + ), + ]); if ((await token.methods.get_admin().simulate({ from: owner })).result !== owner.toBigInt()) { throw new Error(`Token admin is not ${owner}`); diff --git a/yarn-project/end-to-end/src/single-node/fees/account_init.test.ts b/yarn-project/end-to-end/src/single-node/fees/account_init.test.ts index b8aee5c64243..12da781fbdf1 100644 --- a/yarn-project/end-to-end/src/single-node/fees/account_init.test.ts +++ b/yarn-project/end-to-end/src/single-node/fees/account_init.test.ts @@ -33,8 +33,9 @@ describe('single-node/fees/account_init', () => { beforeAll(async () => { await t.setup({ ...PIPELINING_SETUP_OPTS }); - await t.applyFundAliceWithBananas(); - await t.applyFPCSetup(); + // Alice's banana mints and the BananaFPC deploy each depend only on the BananaCoin deployed + // during setup, so they run concurrently and share slots. + await Promise.all([t.applyFundAliceWithBananas(), t.applyFPCSetup()]); ({ aliceAddress, wallet, bananaCoin, bananaFPC, logger, aztecNode } = t); }); diff --git a/yarn-project/end-to-end/src/single-node/fees/fees_test.ts b/yarn-project/end-to-end/src/single-node/fees/fees_test.ts index 14efa5b5c234..f03a17092ef5 100644 --- a/yarn-project/end-to-end/src/single-node/fees/fees_test.ts +++ b/yarn-project/end-to-end/src/single-node/fees/fees_test.ts @@ -1,4 +1,5 @@ import type { AztecAddress } from '@aztec/aztec.js/addresses'; +import { BatchCall } from '@aztec/aztec.js/contracts'; import { Fr } from '@aztec/aztec.js/fields'; import { createLogger } from '@aztec/aztec.js/log'; import type { AztecNode } from '@aztec/aztec.js/node'; @@ -408,12 +409,24 @@ export class FeesTest extends SingleNodeTestContext { public async applyFundAliceWithBananas() { this.logger.info('Applying fund Alice with bananas setup'); - await this.mintPrivateBananas(this.ALICE_INITIAL_BANANAS, this.aliceAddress); + // Both mints are alice sends on BananaCoin touching disjoint balances, so they ride a single + // BatchCall tx (one proof, one slot) rather than two concurrent txs. The PXE serializes proving, + // so two concurrent sends would still be proven back-to-back and land in consecutive slots. + const { result: privateBalanceBefore } = await this.bananaCoin.methods + .balance_of_private(this.aliceAddress) + .simulate({ from: this.aliceAddress }); + await testSpan('tx:mint', () => - this.bananaCoin.methods.mint_to_public(this.aliceAddress, this.ALICE_INITIAL_BANANAS).send({ - from: this.aliceAddress, - }), + new BatchCall(this.wallet, [ + this.bananaCoin.methods.mint_to_private(this.aliceAddress, this.ALICE_INITIAL_BANANAS), + this.bananaCoin.methods.mint_to_public(this.aliceAddress, this.ALICE_INITIAL_BANANAS), + ]).send({ from: this.aliceAddress }), ); + + const { result: privateBalanceAfter } = await this.bananaCoin.methods + .balance_of_private(this.aliceAddress) + .simulate({ from: this.aliceAddress }); + expect(privateBalanceAfter).toEqual(privateBalanceBefore + this.ALICE_INITIAL_BANANAS); } public async applyFundAliceWithPrivateBananas() { diff --git a/yarn-project/end-to-end/src/single-node/fees/gas_estimation.parallel.test.ts b/yarn-project/end-to-end/src/single-node/fees/gas_estimation.parallel.test.ts index 395bb286069d..c4d3d67f04e8 100644 --- a/yarn-project/end-to-end/src/single-node/fees/gas_estimation.parallel.test.ts +++ b/yarn-project/end-to-end/src/single-node/fees/gas_estimation.parallel.test.ts @@ -46,8 +46,9 @@ describe('single-node/fees/gas_estimation', () => { beforeAll(async () => { await t.setup({ ...PIPELINING_SETUP_OPTS }); - await t.applyFPCSetup(); - await t.applyFundAliceWithBananas(); + // Alice's banana mints and the BananaFPC deploy each depend only on the BananaCoin deployed + // during setup, so they run concurrently and share slots. + await Promise.all([t.applyFPCSetup(), t.applyFundAliceWithBananas()]); ({ wallet, aliceAddress, bobAddress, bananaCoin, bananaFPC, gasSettings, logger, aztecNode } = t); }); diff --git a/yarn-project/end-to-end/src/single-node/fees/private_payments.parallel.test.ts b/yarn-project/end-to-end/src/single-node/fees/private_payments.parallel.test.ts index 77b51cfbb7ee..d0ef1c912059 100644 --- a/yarn-project/end-to-end/src/single-node/fees/private_payments.parallel.test.ts +++ b/yarn-project/end-to-end/src/single-node/fees/private_payments.parallel.test.ts @@ -46,11 +46,10 @@ describe('single-node/fees/private_payments', () => { // cycle: the prover-node submits a proof as soon as the epoch is complete, so ~8x shorter // epochs ≈ ~8x faster proof cadence per cycle. Setup itself stays slot-bound. await t.setup({ ...PIPELINING_SETUP_OPTS, aztecProofSubmissionEpochs: 640, aztecEpochDuration: 4 }); - await t.applyFPCSetup(); - // Register the SponsoredFPC (funded at genesis via FeesTest's fundSponsoredFPC) so the folded - // sponsored-payment it can use it; this is a PXE registration, not an L2 tx. - await t.applySponsoredFPCSetup(); - await t.applyFundAliceWithBananas(); + // The BananaFPC deploy, the SponsoredFPC registration (funded at genesis via FeesTest's + // fundSponsoredFPC; a PXE registration, not an L2 tx), and Alice's banana mints each depend only + // on the BananaCoin deployed during setup, so they run concurrently and share slots. + await Promise.all([t.applyFPCSetup(), t.applySponsoredFPCSetup(), t.applyFundAliceWithBananas()]); ({ wallet, aliceAddress, From e0db4086ce1981a8e4c81890845c79e1ccb3c1a9 Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:01:00 -0400 Subject: [PATCH 13/35] docs: revert threat model for node (#24465) (#24610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes `yarn-project/THREAT_MODEL.md`, added in [#24465](https://github.com/AztecProtocol/aztec-packages/pull/24465). Per discussion in #team-alpha: detailed threat models and invariants may make it easier for attackers to find bugs, so the team wants this content pulled from the public repo for now. It will be re-added to LabsBox/ClaudeBox (private/internal) — see [claudebox#1357](https://github.com/AztecProtocol/claudebox/pull/1357) — and can come back here once the team is finding fewer bugs. Only the threat model doc itself is removed here — the incidental doc cleanups bundled into the same squashed commit (stale BLOCK protocol references in `yarn-project/p2p/README.md` and `yarn-project/p2p/src/services/reqresp/README.md`, and the missing `DUPLICATE_ATTESTATION` entry in `yarn-project/slasher/README.md`) are kept as-is. (cherry picked from commit 4d026929bc2f97638332f1bd19595de0c92f2828) --- yarn-project/THREAT_MODEL.md | 341 ----------------------------------- 1 file changed, 341 deletions(-) delete mode 100644 yarn-project/THREAT_MODEL.md diff --git a/yarn-project/THREAT_MODEL.md b/yarn-project/THREAT_MODEL.md deleted file mode 100644 index 1dee4b96cba8..000000000000 --- a/yarn-project/THREAT_MODEL.md +++ /dev/null @@ -1,341 +0,0 @@ -# Aztec Network Threat Model - -This document describes the threat model of the Aztec L2 network: how transactions flow from users into the proven -chain, what each participant can and cannot do, and the properties the implementation must uphold. It is intended as a -guideline for the security of the node implementation (`yarn-project`) and its interaction with the L1 rollup -contracts (`l1-contracts`). - -**In scope**: transaction dissemination and the mempool, the p2p layer, block and checkpoint production, committee -attestation, L1 checkpoint submission and sync, epoch proving, slashing, and the escape hatch. - -**Out of scope**: client-side private execution and proving (PXE, wallets), hardening of a node's public RPC interface, -the cryptographic soundness of the proving system itself (treated as an assumption below), and L1 governance internals. - -## 1. System overview - -### Actors - -| Actor | Role | -| --- | --- | -| User | Executes private functions locally, produces a client-side proof, submits the tx to a node via RPC. | -| Node | Syncs L2 state from L1 and p2p, maintains a mempool, serves RPC. Every other server-side actor runs one. | -| Proposer | The validator elected for a slot. Builds blocks, collects attestations, submits the checkpoint to L1. | -| Committee | Per-epoch sample of validators. Re-executes proposals and attests to checkpoints; its attestations gate proof acceptance (training wheels for the proving system, A2) and back data availability (A8). | -| Prover | Generates the epoch validity proof and submits it to L1. Permissionless. | -| Escape hatch proposer | Bonded candidate, randomly selected, may propose without a committee during periodic windows. | -| Vetoer | Designated L1 role that can block slash payloads during the execution delay. | -| L1 contracts | Rollup (checkpoints, proofs, pruning, invalidation), Inbox/Outbox (cross-chain messages), slashing and governance periphery. | - -### Time and validator selection - -Time is divided into **slots** (fixed windows, e.g. 72 s): each slot has exactly one elected proposer and produces at -most one checkpoint, built as several blocks in fixed sub-slots. Slots group into **epochs**: each epoch has one -committee, and epochs are the unit of proving and pruning. - -The committee for epoch N is sampled (Fisher–Yates, without replacement) from the registered validator set, seeded from -Ethereum's RANDAO. Both inputs are taken from the past: the validator set is snapshotted `lagInEpochsForValidatorSet` -epochs before N and the seed `lagInEpochsForRandao` epochs before N, with set lag ≥ seed lag — by the time the -randomness is known, the population it samples from can no longer be changed. L1 stores a commitment to each epoch's -committee to prevent substitution (`ValidatorSelectionLib`); nodes mirror the computation locally -([epoch-cache README](epoch-cache/README.md)). The proposer for a slot is -`keccak(epoch, slot, seed) % committeeSize` — deterministic and computable by anyone. - -Selection bias is an audit surface in its own right: seed grinding via L1 `prevrandao`, and timing games against the -validator-set snapshot. The lag scheme above and the escape hatch's snapshot-before-seed ordering are the existing -defenses. Edge case: an empty committee (target size 0) means anyone may propose, and nodes accept checkpoints without -attestation validation. - -### Transaction lifecycle - -1. **Submission.** A user sends a tx (with its client-side proof) to a node via JSON-RPC `sendTx`. The node fully - validates it — proof verification plus protocol rules (double-spend, fees, gas limits, expiration, metadata) — - before admitting it to the mempool ([server.ts](aztec-node/src/aztec-node/server.ts), - [tx validators](p2p/src/msg_validators/tx_validator/)). -2. **Propagation.** The node gossips the tx on the p2p `tx` topic. Every receiving node runs the same validation - pipeline *before* the message is re-propagated: gossipsub only forwards messages the local node accepted. Peers that - originate invalid data are penalized; validation outcomes that could be another node's fault are dropped without - penalty (see §6). -3. **Mempool.** The pool ([tx_pool_v2](p2p/src/mem_pools/tx_pool_v2/)) admits txs subject to nullifier-conflict, - fee-payer-balance, and priority rules, and evicts txs that become ineligible (nullifiers mined by a block, expired - timestamps, insufficient fee-payer balance, invalid anchor block after a reorg, lowest priority when full). -4. **Block building.** The proposer for a slot builds several blocks back-to-back (sub-slots), pulling txs from its - mempool and executing public calls against a fork of world state. Each block is signed and broadcast as a - `BlockProposal`; the last block ships inside the `CheckpointProposal` that closes the slot. Production runs - pipelined: blocks for slot N are built during slot N−1 ([sequencer-client README](sequencer-client/README.md)). -5. **Attestation.** Committee members re-execute every block in the proposal and, if the result matches, sign a - `CheckpointAttestation` over the checkpoint header and archive root. Attestations are checkpoint-only; individual - blocks are never attested ([validator-client README](validator-client/README.md)). -6. **L1 submission.** Once the proposer holds a quorum of attestations — ⌊2n/3⌋+1 of the committee — it submits the - checkpoint to the rollup contract in a single Multicall3 tx (invalidations first, then propose, then - governance/slashing votes). At propose time L1 validates the header, blob commitments, and the proposer signature, - but **not** the attestations (posted as calldata) and **not** tx validity (`ProposeLib`). -7. **Sync.** Nodes track two chains. The **proposed chain** comes from p2p: proposals are re-executed locally and - pushed into the archiver as provisional blocks. The **checkpointed (pending) chain** comes from L1 - `CheckpointProposed` events: each node verifies the posted attestations from calldata (committee membership and - quorum, per *delayed attestation verification*) before fetching and decoding blobs; checkpointed blocks are **not** - re-executed ([archiver README](archiver/README.md), [validation.ts](archiver/src/modules/validation.ts)). -8. **Proving.** Prover nodes prove epochs optimistically, starting sub-tree work as checkpoints land on L1. The rollup - accepts an epoch proof only if the proof verifies **and** the last checkpoint in the range carries valid committee - attestations (`EpochProofLib`). The proven tip then advances. If no proof - lands within the proof submission window, all unproven checkpoints are pruned — the pending chain reorgs back to the - proven tip; nodes unwind preemptively and the mempool resurrects the affected txs. -9. **Finality.** Proven state enables L2→L1 message consumption via the Outbox, and fees/rewards are distributed from - the committee-attested checkpoint headers. Once the L1 block containing the verified proof is itself finalized on - L1, the state can no longer be reorged out. - -### Chain states - -| Chain | Source | Trust | -| --- | --- | --- | -| Proposed | p2p proposals | Re-executed locally; trustless but reorgs freely within/across slots. | -| Checkpointed (pending) | L1 events + blobs | Attestation-gated; contents trusted from the committee (not re-executed). Reorgs on prune or invalidation. | -| Proven | L1 verified proof | Trust reduces to circuit soundness + L1 verifier. Still subject to L1 reorgs. | -| Finalized | Proof's L1 block finalized | The L1 block containing the verified proof is finalized; cannot be reorged out of L1. | - -### Where the details live - -| Topic | Document | -| --- | --- | -| Proposer flow, pipelining, timetable, L1 publisher | [sequencer-client README](sequencer-client/README.md) | -| Proposal validation, attestation creation, building limits | [validator-client README](validator-client/README.md) | -| Committee/proposer selection, RANDAO seed, epoch caching | [epoch-cache README](epoch-cache/README.md) | -| Gossip topics, message validation, peer scoring, req/resp | [p2p README](p2p/README.md) and sub-READMEs | -| Mempool state machine and eviction | [tx_pool_v2 README](p2p/src/mem_pools/tx_pool_v2/README.md) | -| L1 sync, reorgs, invalid checkpoints, pruning | [archiver README](archiver/README.md) | -| Epoch proving pipeline | [prover-node README](prover-node/README.md) | -| Slashing architecture and offenses | [slasher README](slasher/README.md) | -| Operator-facing slashing guide (amounts, veto, ejection) | [slashing configuration guide](../docs/docs-operate/operators/sequencer-management/slashing-configuration.md) | - -## 2. Trust assumptions - -- **A1 — Committee quorum.** Safety of the *pending* chain assumes fewer than ⌊2n/3⌋+1 members of any epoch committee - are malicious; liveness assumes at least ⌊2n/3⌋+1 are honest and online (`computeQuorum` in - [epoch-helpers](stdlib/src/epoch-helpers/), mirrored in `InvalidateLib`). -- **A2 — Proven-chain soundness.** It must be impossible to prove an invalid state transition: the *proven* chain - assumes the protocol circuits are sound and the L1 verifier is correct, without relying on committee honesty. The - committee is nevertheless a second, independent gate: proof submission requires committee attestations on the proven - range (V4), so it acts as training wheels for the proving system — exploiting a soundness bug also requires a - colluding quorum (or an escape-hatch bond, §7). Invalid state reaches the proven chain only if *both* layers fail. -- **A3 — Completeness.** Every state transition the network considers valid must be provable. An unprovable-but-attested - checkpoint forces a prune, so a completeness bug converts into a liveness attack. -- **A4 — Prover liveness.** At least one prover is willing and able to prove each epoch; otherwise the chain reorgs at - the proof submission deadline. -- **A5 — L1 liveness.** Ethereum remains live and censorship-resistant enough for time-windowed actions to land: - checkpoint proposal, invalidation, proof submission, slashing votes and execution. -- **A6 — Blind checkpoint trust.** Nodes follow attested checkpoints without re-executing them. A committee quorum can - therefore feed nodes invalid pending state (bounded by A2 + pruning). This assumption may be revisited. -- **A7 — Uniform validation config.** Slashing relies on all honest validators making identical deterministic validity - decisions; operators must not run divergent block/checkpoint validation limits. -- **A8 — Data availability.** Checkpoint effects are published as L1 blobs; tx preimages needed for proving are served - over p2p. Committees are slashed if the data behind their attested checkpoints is not made available. - -## 3. Threat actors and worst-case impact - -| Actor | Can do (accepted worst case) | Cannot do (must hold) | Mitigations | -| --- | --- | --- | --- | -| Malicious user | Submit txs that later become ineligible (e.g. nullifier races); attempt mempool flooding | Saturate mempools with unincludable txs; get an invalid tx included; force an unprovable state transition | Full validation incl. proof verification at every hop; eviction rules; priority fees + price-bump replacement; per-tx gas floors | -| Malicious peer (non-validator) | Send garbage, replay, or spam over gossip/req-resp | Get invalid data re-broadcast by honest nodes; cause honest nodes to be penalized by their peers; eclipse a node cheaply | Validate-before-forward; peer scoring, rate limits, bans (§6); AUTH handshake on nodes that only accept validator peers | -| Malicious proposer | Censor txs; waste its own slot and the next one (pipelining); post an unattested/badly-attested checkpoint to L1; equivocate | Convince honest nodes of an invalid state transition; corrupt state beyond slot N+1; brick other nodes' sync | Re-execution before attestation/adoption; pipeline depth capped at 2; delayed attestation verification + permissionless invalidation; slashing (§5) | -| Committee minority (< quorum, not proposer) | Withhold their own attestations; equivocate (slashable) | Prevent a slot from being attested; get honest members slashed | Quorum only needs ⌊2n/3⌋+1 of n; equivocation slashing | -| Committee quorum (≥ ⌊2n/3⌋+1 malicious) | Post invalid-but-attested checkpoints that nodes follow blindly; withhold tx data; halt the pending chain until the epoch prune | Get invalid state proven or exited on L1 (A2); avoid slashing for data withholding / attested-invalid | Unprovable state → prune at proof-window expiry; archiver preemptive unwind; slashing incl. `DATA_WITHHOLDING` | -| Malicious prover | Nothing by proving (proofs are verified); grief by *not* proving | Prove an invalid transition (A2); steal fees/rewards via forged proof calldata | Proof verification; attestation gate at proof submission (A2); fees bound to attested headers (see §4.4); A4 for liveness | -| Escape hatch proposer | Same as a malicious committee during its hatch window: post arbitrary unattested checkpoints, halt pending chain until prune | Exceed malicious-committee damage; escape its bond | Bond + punishment for failing to propose-and-prove; hatch windows are bounded (`ACTIVE_DURATION` out of every `FREQUENCY` epochs) | -| Malicious slashing majority | Vote through an unfair slash (requires >50% of a round's proposers) | Execute it silently or instantly | Execution delay + vetoer; offense votes are public on L1 | - -## 4. Security properties - -Properties are numbered for reference from audit findings. Each states the requirement, the enforcing mechanism, and -notable caveats. - -### 4.1 P2P and mempool - -- **P1 — Validate before forward.** No node re-broadcasts a gossip message it has not fully validated (including tx - proof verification). Enforced by running validation inside the gossipsub message-validation hook; only `Accept` - results propagate. -- **P2 — No penalty for relayed faults.** A peer must not be penalized for forwarding data that was plausibly valid - from its point of view. Encoded three ways: (a) tx validation splits outcomes into `REJECT` + penalty for - sender-attributable faults (malformed data, invalid proof) vs `IGNORE`/low-severity for state-divergence-explainable - ones (recent double-spend, timestamp expiry); (b) proposal gossip validators only run shallow, syntactic checks - (signature, slot window, proposer identity, tx-hash consistency) — deep re-execution failures are attributed to the - *proposer* via slashing and never penalize the relaying peer; (c) equivocating or oversized proposals are - deliberately `Accept`ed and re-broadcast as slashing evidence so the network can witness the offense, again without - penalizing relayers. A malicious node must not be able to craft data that honest nodes accept, re-broadcast, and are - then penalized for. -- **P3 — Mempool inclusion-eligibility.** Every tx in the pending pool must be includable in a future block; txs - invalidated by chain progress are evicted (mined nullifiers, expiry, fee-payer balance, reorged anchor blocks). - Accepted residual: executing a tx inside a block can invalidate sibling txs mid-slot (nullifier collision); at least - one tx of any colliding set is includable, and the builder drops the rest during execution. -- **P4 — Flood resistance.** Pool size is capped with priority-fee eviction; replacement requires a configurable price - bump; gossip messages have per-topic size caps; req/resp protocols have per-peer and global rate limits (§6). -- **P5 — Sybil/eclipse resistance.** Peer discovery (discv5), connection gating, auth handshakes on nodes that only - accept validator peers (`p2pAllowOnlyValidators`, off by default; no such nodes are deployed today), failed-auth - lockouts, and IP colocation penalties raise the cost of surrounding a node. This is a defense-in-depth area, not an - absolute guarantee. - -### 4.2 Block proposal and attestation - -- **B1 — Proposer exclusivity.** Only the slot's elected proposer can land a checkpoint: L1 verifies the proposer - signature over the payload digest at propose time (`ValidatorSelectionLib.verifyProposer`); validators independently - check proposal provenance before processing. During hatch windows, only the designated escape hatch proposer may - propose. -- **B2 — Re-execution gate.** Honest validators attest only after re-executing all blocks in the checkpoint and - matching the resulting state (archive root, header hash). Honest full nodes adopt proposed blocks into their view - only via the same re-execution path. A proposal that fails validation is rejected and triggers a slash vote - (`BROADCASTED_INVALID_BLOCK_PROPOSAL` / `BROADCASTED_INVALID_CHECKPOINT_PROPOSAL`). -- **B3 — Equivocation detection.** Two proposals for the same position with different content, or two attestations by - the same signer for the same slot, are slashable (`DUPLICATE_PROPOSAL`, `DUPLICATE_ATTESTATION`). The first duplicate - proposal is deliberately propagated so other validators can witness the offense. Checkpoint-vs-L1 equivocation is - caught by comparing signed p2p proposals against L1-confirmed checkpoints. -- **B4 — Bounded proposer blast radius.** Pipelining lets a proposer build on an in-flight parent, so a failed or - malicious slot can waste the *next* proposer's work, but no more: pipeline depth is capped at 2 checkpoints beyond the - confirmed tip, and a parent that fails to land cleanly causes the child's work to be discarded and an invalidation to - be enqueued ([checkpoint_proposal_job.ts](sequencer-client/src/sequencer/checkpoint_proposal_job.ts)). -- **B5 — Attestation quorum.** A checkpoint is valid only with ⌊2n/3⌋+1 committee signatures over the consensus payload - (EIP-712, slot-bound). Enforced off-chain by every syncing node, and on-chain at proof submission and in - `invalidateInsufficientAttestations`. - -### 4.3 Checkpointed chain and delayed attestation verification - -L1 does **not** verify committee attestations at propose time (gas optimization). The security burden moves to: - -- **C1 — Nodes never follow bad-attestation checkpoints.** Every node validates attestations (signatures, committee - membership at the correct index, quorum) from L1 **calldata** before fetching or decoding blobs, so a checkpoint with - invalid attestations is rejected without touching potentially malformed blob data - ([validation.ts](archiver/src/modules/validation.ts)). -- **C2 — Sync never bricks.** The archiver skips invalid checkpoints, advances its syncpoint past them, and keeps - processing. The `inHash` check cannot be weaponized: L1 enforces `header.inHash == inbox.consume(...)` at propose - time, so a local mismatch indicates a node bug, not attacker-controlled input (`ProposeLib`). -- **C3 — Bad-attestation checkpoints are removable and unprovable.** Anyone can call `invalidateBadAttestation` / - `invalidateInsufficientAttestations` to purge them (no rebate; expected callers: next proposer, then committee, then - any validator, with timed fallbacks in the sequencer). They cannot enter the proven chain: `submitEpochRootProof` - re-verifies the last checkpoint's attestations (`InvalidateLib`, `EpochProofLib`). -- **C4 — Posting bad attestations is punished.** `PROPOSED_INSUFFICIENT_ATTESTATIONS`, - `PROPOSED_INCORRECT_ATTESTATIONS`, and `PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS` target the - publishing proposer (only the one who publishes to L1 — a pipelined builder that discarded its work is not at fault). -- **C5 — Malicious-quorum damage is time-bounded.** Invalid-but-attested state persists at most until the epoch's proof - submission window expires, at which point the rollup prunes on the next propose and nodes preemptively unwind - (`canPruneAtTime`). Honest checkpoints built on top are collateral damage of the prune (accepted). - -### 4.4 Proving - -- **V1 — Soundness (A2).** No prover input may yield an accepted proof of an invalid transition. Circuit-level; out of - scope here but the anchor for everything above. -- **V2 — Completeness (A3).** Anything the network accepts as valid must be provable. Consequence for node config: - restrictions enforced only in validator software (e.g. block/tx caps) must never make circuit-valid state unprovable - or unsyncable — see S1. -- **V3 — Prune on missing proof.** Epochs unproven past the window are removed; the chain falls back to the proven tip. - This is the designed recovery from both prover outages and malicious-quorum garbage. -- **V4 — Unsound-verifier blast radius.** Defense in depth for a hypothetical L1 verifier bug: (a) epoch-proof fee - data is not free calldata — fees and rewards derive from checkpoint headers that L1 re-hashes against the - committee-attested header hashes, so a forged proof cannot redirect or inflate fees; (b) proof submission - requires valid committee attestations on the range's last checkpoint, so a lone prover cannot promote arbitrary state - — it needs a colluding quorum (or a hatch window, see §7). This is the committee's training-wheels role from A2. - Residual exposure: Outbox withdrawals from a maliciously-proven state; tracked as a known gap in §7. - -### 4.5 L1 sync completeness - -- **S1 — Sync everything provable.** Nodes must be able to sync from L1 any checkpoint that provers can prove, even if - local validator policy would have refused to attest to it. Validator-side caps (`VALIDATOR_MAX_*`) apply only to p2p - proposal validation; the archiver's L1 path enforces only attestation validity, `inHash` consistency, and structural - blob decoding. Rationale: escape-hatch and malicious-quorum checkpoints bypass validator policy but can still reach - the proven chain, and a node that refuses to sync them forks itself off. -- **S2 — L1 reorg resilience.** Message and checkpoint sync detect L1 reorgs (rolling hashes, archive root comparison), - unwind to the common ancestor, and re-fetch — including the subtle case of checkpoints added *behind* the syncpoint. - See [archiver README](archiver/README.md) § Edge Cases. - -### 4.6 Slashing fairness - -- **F1 — Honest validators are never slashable.** Every offense requires either the offender's own signature - (proposals, attestations — EIP-712 slot-bound, so replay across slots is impossible) or sustained, locally-observable - inactivity. An honest validator signs only what it built or successfully re-executed, so no message from a malicious - peer can route it into slashable behavior. HA deployments coordinate signing via a shared store to prevent - self-equivocation ([validator-ha-signer](validator-ha-signer/)). -- **F2 — Individual accountability.** Offenses target individuals: an honest committee member is not slashed because - the majority misbehaved. Note `DATA_WITHHOLDING` targets *all attesters* of the checkpoint — making the data - available is part of the attester's duty, and any single honest attester publishing the txs clears the whole - committee. -- **F3 — Governance backstop.** Slashes need >50% of a round's proposers to vote, wait out an execution delay, and can - be vetoed. Protects against correlated false positives from software bugs (and from A7 violations). - -## 5. Slashing - -Consensus-based: proposers vote (2 bits per validator: 0–3 slash units) during round N on offenses from round N−2; the -L1 `SlashingProposer` tallies and executes after the delay unless vetoed. Full mechanics in -[slasher README](slasher/README.md); offense rationale in AZIP-7. - -| Offense | Target | Trigger | -| --- | --- | --- | -| `DATA_WITHHOLDING` | All attesters of the checkpoint | Checkpoint txs not available on p2p within the tolerance window | -| `INACTIVITY` | Individual validator | Missed proposals/attestations beyond threshold for consecutive committee epochs (Sentinel) | -| `BROADCASTED_INVALID_BLOCK_PROPOSAL` | Proposer | Block proposal fails validation/re-execution | -| `BROADCASTED_INVALID_CHECKPOINT_PROPOSAL` | Proposer | Checkpoint proposal invalid: truncates its own later block, header mismatch on recomputation, malformed fee modifier | -| `PROPOSED_INSUFFICIENT_ATTESTATIONS` | Publishing proposer | L1 checkpoint with < ⌊2n/3⌋+1 signatures | -| `PROPOSED_INCORRECT_ATTESTATIONS` | Publishing proposer | L1 checkpoint with non-committee or invalid signatures | -| `PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS` | Publishing proposer | Built on an invalid-attestation checkpoint | -| `DUPLICATE_PROPOSAL` | Proposer | Conflicting proposals for the same position (p2p or p2p-vs-L1) | -| `DUPLICATE_ATTESTATION` | Attester | Conflicting attestations for the same slot | -| `ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL` | Attesters | Attested in a slot where the node detected (via re-execution) a slashable invalid proposal | - -Amounts map to three L1-configured tiers. On the v5 testnet (AZIP-16 preset) any offense effectively ejects the -validator (slash drops stake below the local ejection threshold, forcing full exit); mainnet uses smaller amounts. -Grace period after genesis; own validators are auto-protected from a node's own votes; `SLASH_VALIDATORS_NEVER/ALWAYS` -allow operator overrides. - -Audit angles: offense detection soundness under A7 (config divergence), vote encoding and tally correctness on L1, -veto/delay windows vs validator exit timing, and whether any honest behavior can satisfy an offense predicate. - -## 6. P2P peer penalization - -Peer standing combines two layers. The **application-level score** -([peer_scoring.ts](p2p/src/services/peer-manager/peer_scoring.ts)) -accumulates penalties only (there is no reward path), decays by ×0.9 per minute, and drives the peer manager: score -below −50 → GOODBYE + disconnect on the next heartbeat (~30s); below −100 → 24h ban. While banned, the score is frozen -(no decaying out early) and fresh inbound connections are refused; a mere "disconnect"-scored peer may reconnect. The -**gossipsub score** independently combines the app score (weight 10, `-Infinity` for unauthenticated peers when the -node only accepts validator peers via `p2pAllowOnlyValidators`), per-topic delivery scoring (rewards up to +33/topic on predictable-rate topics; the `tx` -topic has no delivery scoring), an invalid-message penalty (P4, weight −20, ~4-slot decay, incremented by every gossip -`REJECT`), and an IP-colocation penalty (−5). Tuning math: [gossipsub README](p2p/src/services/gossipsub/README.md). - -Application-level severities (penalty points against the −50/−100 thresholds; boundaries are strict, so e.g. two -`LowTolerance` strikes reach exactly −100 and a third is needed to ban): - -| Severity | Points | Example conditions | -| --- | --- | --- | -| `LowToleranceError` | 50 | Invalid tx proof, undeserializable message, double-spend of a long-settled nullifier, malformed req/resp payload | -| `MidToleranceError` | 10 | Most tx validation failures (metadata, gas, phases, size), inconsistent req/resp batch responses | -| `HighToleranceError` | 2 | Conditions plausibly caused by state lag or transient faults: recent double-spend, timestamp expiry, per-peer rate-limit breach, connection resets/timeouts | - -Req/resp (PING, STATUS, AUTH, GOODBYE, TX, BLOCK_TXS): per-peer and global rate limits per protocol (handshake -protocols 5/s per peer, 10/s global; TX and BLOCK_TXS 10/s per peer, 200/s global). The per-peer check runs before the -global one, so a single peer cannot exhaust the shared budget and have others blamed; exceeding a per-peer limit costs -a `HighToleranceError`, exceeding the global limit returns `RATE_LIMIT_EXCEEDED` with no penalty. Malformed -requests/responses penalize the sender (`LowToleranceError`); transport errors and timeouts are treated as transient -(`HighToleranceError`); self-inflicted aborts are never penalized. - -Handshake failures are a separate subsystem that bypasses scoring: failed AUTH attempts (nodes running with -`p2pAllowOnlyValidators`) get -exponential dial backoff (5 min doubling to 160 min) and, past `p2pMaxFailedAuthAttemptsAllowed` (default 3), inbound -denial until 1h passes without failures. Trusted/private/preferred peers are exempt from manager-initiated -disconnection and capacity eviction, but **not** from scoring or gossipsub graylisting. - -The reject-vs-ignore mapping per message type (which failures penalize the sender vs get silently dropped) is the -enforcement of P2 and is catalogued per validator in [message validator READMEs](p2p/src/msg_validators/). - -## 7. Known gaps - -1. **Descendants of invalid checkpoints.** If a malicious quorum attests to a *descendant* of an invalid-attestation - checkpoint, nodes currently follow it (they assume an honest majority) instead of ignoring it until proven. Flagged - in [archiver README](archiver/README.md); the corresponding slash (`PROPOSED_DESCENDANT_…`) exists, but node-side - chain-selection does not. -2. **Escape hatch × unsound verifier.** `submitEpochRootProof` skips attestation verification for hatch epochs (there - is no committee), so during a hatch window the attestation gate of V4(b) is absent: a malicious hatch proposer plus - an L1 verifier bug could prove invalid state. Fee theft is still blocked (V4(a)); Outbox withdrawals are the - remaining exposure. Compensating controls today: bond, random selection among bonded candidates, bounded window. -3. **Validation config divergence (A7).** Divergent `VALIDATOR_MAX_*` limits across honest validators can produce - divergent invalidity verdicts and therefore divergent slash votes against honest proposers. Mitigated by the >50% - vote quorum and the vetoer; no protocol-level enforcement of config uniformity exists. -4. **Committee trust is a standing decision (A6).** Following attested checkpoints without re-execution is deliberate - and may be revisited (e.g. stateless validation of checkpointed data); until then, C1–C5 are the whole story for - pending-chain integrity. -5. **Blob retention.** Nodes syncing later than the L1 blob retention window depend on out-of-protocol blob archives to - reconstruct the checkpointed chain; availability of that path is operational, not protocol-enforced. From 4f980cd7028cb7756f4e5e57bac7920824e35037 Mon Sep 17 00:00:00 2001 From: AztecBot <49558828+AztecBot@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:25:14 +0000 Subject: [PATCH 14/35] docs: document initializerless accounts (v5-next backport of #24512) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backport of #24512 ("docs: document initializerless accounts") into `v5-next`, requested by Alejo in Slack. ## What this adds (same as the original PR) - **New concept page** `foundational-topics/accounts/deployment.md` ("Account Deployment"): the standard initialize-and-deploy flow vs the initializerless flow, how the signing key is committed into the address via `immutables_hash`, and the trade-offs (no deploy cost/delay; signing key fixed forever; per-PXE local setup; `getDeployMethod()` throws). - **New "Create an initializerless account" section** in `aztec-js/how_to_create_account.md` with the `createSchnorrInitializerlessAccount` snippet and caveats. - **Cross-links** from `fees.md`, `wallets.md`, `contract_creation.md`, and `aztec-nr/framework-description/immutables.md`. ## How the backport differs from the original - The original PR mirrored all changes into `developer_versioned_docs/version-v5.0.0-rc.2/`. That snapshot does not exist on `v5-next` (the only snapshot here is `version-v4.3.0`, which documents the v4 line and must not describe a v5-only feature), so this backport touches **only the current docs** under `docs/docs-developers/`. Git's directory-rename detection tried to remap the mirrored copies into the v4.3.0 snapshot during the cherry-pick; those were dropped deliberately. - Everything else is the original commit cherry-picked verbatim; the only diff-vs-original differences are context lines where `v5-next` prose already diverged from `next`. ## Notes - Unlike `next`, `createSchnorrInitializerlessAccount` **does** exist in this branch's `yarn-project` (verified), so the fenced snippet could be converted to an `#include_code` example here as a follow-up. Kept as a fenced block for a faithful backport. - Verified all new internal links resolve to files present on `v5-next`, and `initializerless` is already in `docs-words.txt` on this branch. Refs #24512. --- *Created by [claudebox](https://claudebox.work/v2/sessions/045227c21dcc966b) · group: `slackbot`* (cherry picked from commit 753460447a972906bfbc7eda6d68818cc2f26010) --- .../docs/aztec-js/how_to_create_account.md | 24 +++++++- .../framework-description/immutables.md | 2 + .../accounts/deployment.md | 61 +++++++++++++++++++ .../foundational-topics/contract_creation.md | 2 +- .../docs/foundational-topics/fees.md | 2 +- .../docs/foundational-topics/wallets.md | 2 + 6 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 docs/docs-developers/docs/foundational-topics/accounts/deployment.md diff --git a/docs/docs-developers/docs/aztec-js/how_to_create_account.md b/docs/docs-developers/docs/aztec-js/how_to_create_account.md index 5f9fcfe0a43e..4b69c16aff01 100644 --- a/docs/docs-developers/docs/aztec-js/how_to_create_account.md +++ b/docs/docs-developers/docs/aztec-js/how_to_create_account.md @@ -30,9 +30,29 @@ The secret is used to derive the account's encryption keys, and the salt ensures Save the `secret` and `salt` values securely. You need both to recover access to your account. If you lose them, you will permanently lose access to the account and any assets it holds. ::: +## Create an initializerless account + +Alternatively, create an [initializerless account](../foundational-topics/accounts/deployment.md), which needs no deployment transaction at all: + +```typescript +const secret = Fr.random(); +const salt = Fr.random(); +const account = await wallet.createSchnorrInitializerlessAccount(secret, salt); +console.log("Account address:", account.address.toString()); +``` + +An initializerless account commits its signing public key into the address itself (through the instance's `immutables_hash`), so there is no onchain state to initialize. Creating the account registers it locally in the PXE, and it is ready to use immediately: skip the deployment section below entirely. Fees are only needed for the account's first real transaction, paid with any of the usual [payment methods](./how_to_pay_fees.md). + +Two things to keep in mind: + +- The signing key cannot be changed later. A new key means a new address. +- Calling `getDeployMethod()` on an initializerless account throws, since there is nothing to deploy. + +See [account deployment](../foundational-topics/accounts/deployment.md) for how this works and how to choose between the two account types. + ## Deploy the account -New accounts must be deployed before they can send transactions. Deployment requires paying fees. +Accounts created with `createSchnorrAccount` must be deployed before they can send transactions (initializerless accounts skip this step). Deployment requires paying fees. ### Using the Sponsored FPC @@ -70,5 +90,5 @@ Confirm the account was deployed successfully. Substitute the account variable f - [Deploy contracts](./how_to_deploy_contract.md) with your new account - [Send transactions](./how_to_send_transaction.md) from an account -- Learn about [account abstraction](../foundational-topics/accounts/index.md) +- Learn about [account abstraction](../foundational-topics/accounts/index.md) and [account deployment](../foundational-topics/accounts/deployment.md) - Implement [authentication witnesses](./how_to_use_authwit.md) diff --git a/docs/docs-developers/docs/aztec-nr/framework-description/immutables.md b/docs/docs-developers/docs/aztec-nr/framework-description/immutables.md index 0e93cfda9fbe..629d66a9c3c0 100644 --- a/docs/docs-developers/docs/aztec-nr/framework-description/immutables.md +++ b/docs/docs-developers/docs/aztec-nr/framework-description/immutables.md @@ -31,3 +31,5 @@ Initialization cost is completely eliminated (no constructor transaction). The p ## Getting started For installation instructions, usage examples, and a reference implementation of an initializerless Schnorr account contract, see the [aztec-immutables-macro README](https://github.com/defi-wonderland/aztec-immutables-macro/tree/dev). + +The protocol ships a built-in account contract using the same pattern: see [account deployment](../../foundational-topics/accounts/deployment.md) for how initializerless accounts work and [creating accounts](../../aztec-js/how_to_create_account.md#create-an-initializerless-account) for using them from Aztec.js. diff --git a/docs/docs-developers/docs/foundational-topics/accounts/deployment.md b/docs/docs-developers/docs/foundational-topics/accounts/deployment.md new file mode 100644 index 000000000000..d189520dfa81 --- /dev/null +++ b/docs/docs-developers/docs/foundational-topics/accounts/deployment.md @@ -0,0 +1,61 @@ +--- +title: Account Deployment +tags: [accounts] +description: How Aztec accounts come into existence, from the standard initializer plus deployment flow to initializerless accounts that need no onchain transaction at all. +references: + [ + "noir-projects/noir-contracts/contracts/account/schnorr_initializerless_account_contract/src/main.nr", + "noir-projects/noir-contracts/contracts/account/schnorr_account_contract/src/main.nr", + ] +--- + +## How accounts come into existence + +Every account in Aztec is a [contract instance](../contract_creation.md), and its address is computed deterministically from its instantiation parameters. There are two ways to make an account usable: + +1. **The standard flow**: create the account locally, then send a deployment transaction that runs its initializer. +2. **The initializerless flow**: create the account locally, and that is it. No deployment transaction is ever sent. + +This page explains both flows and when to choose each. + +## The standard flow: initialize and deploy + +A regular account contract (for example the default Schnorr account) stores its signing public key in private state. Writing that state requires running the contract's constructor, a [private initializer function](../../aztec-nr/framework-description/functions/attributes.md#initializer-functions-initializer), which in turn requires sending a transaction to the network and paying [fees](../fees.md) for it. + +The address commits to that pending initialization: the constructor call's selector and arguments are hashed into the instance's `initialization_hash`, which is folded into the address derivation. This is why the address can be computed, shared, and even receive funds before the deployment transaction is sent, and why the account only becomes able to send transactions after it. + +## Initializerless accounts + +An initializerless account removes the deployment transaction entirely. Instead of storing the signing key in private state through an initializer, the key is committed directly into the address: the instance's `immutables_hash` field is set to the hash of the signing public key, and `immutables_hash` participates in address derivation just like `initialization_hash` does. + +Since the address itself is the commitment to the signing key, there is no onchain state to create: + +- The account contract has no initializer. Its `constructor` is an unconstrained utility function that runs only in the PXE: it checks that the provided public key hashes to the instance's `immutables_hash` and stores the key in the PXE's local store. +- When the account later authorizes a transaction, its entrypoint loads the key from the local store and proves in the private kernel that the key's hash matches the `immutables_hash` committed in the address. Tampering with the local key is not possible without changing the address. + +Creating an initializerless account is therefore a purely local operation: the wallet computes the address, registers the contract instance in the PXE, and runs the utility constructor through a simulation. No transaction is sent, no fee is paid, and the account can start transacting as soon as it has a way to pay for its first real transaction (for example a [Sponsored FPC](../fees.md#payment-methods) or a Fee Juice balance at its address). + +The genesis-funded test accounts in the local network are initializerless Schnorr accounts: their addresses receive Fee Juice at genesis, and wallets materialize them locally without any deployment. + +## Trade-offs + +Initializerless accounts are a good default when the account's authorization logic only depends on parameters that never change: + +- **No deployment cost or delay.** The account is usable the moment it is created, which makes onboarding flows and receive-only addresses cheap. +- **The signing key is fixed forever.** The key is baked into the address, so there is no way to rotate it. A new key means a new address. The default Schnorr account offers no key rotation either, but account contracts that keep keys in state can be designed to support it. +- **Local setup is still required per PXE.** A fresh PXE does not know about the account until the wallet registers the instance and runs the utility constructor again. This is a purely offline step with no fee, but it must happen in every new environment before the account can be used. +- **There is no deployment method.** Calling `getDeployMethod()` on an initializerless account throws, since there is nothing to deploy. + +Use the standard flow when the account contract needs an initializer: for example when it takes arbitrary constructor arguments or stores its authorization material in private notes. + +## Using initializerless accounts + +- In Aztec.js, call `createSchnorrInitializerlessAccount(secret, salt)` on your wallet. See [creating accounts](../../aztec-js/how_to_create_account.md#create-an-initializerless-account). +- In the CLI, pass the account type: `aztec-wallet create-account -t schnorr_initializerless`. The command registers the account locally and returns immediately, with no deployment transaction. +- In Aztec.nr, the same address-immutables pattern can be applied to your own contracts. See [immutables](../../aztec-nr/framework-description/immutables.md). + +## Next steps + +- [Create an account in Aztec.js](../../aztec-js/how_to_create_account.md) +- [Understand fees and payment methods](../fees.md) +- [Contract creation and address derivation](../contract_creation.md) diff --git a/docs/docs-developers/docs/foundational-topics/contract_creation.md b/docs/docs-developers/docs/foundational-topics/contract_creation.md index 899ece79d9cd..a8bfa6801bdf 100644 --- a/docs/docs-developers/docs/foundational-topics/contract_creation.md +++ b/docs/docs-developers/docs/foundational-topics/contract_creation.md @@ -55,7 +55,7 @@ A contract instance includes: - `deployer`: Optional address of the contract deployer. Zero for universal deployment - `original_contract_class_id`: Identifier of the contract class the instance was deployed with. Updating the instance to a new class via the ContractInstanceRegistry does not change this value, since it is part of the address preimage - `initialization_hash`: Hash of the selector and arguments to the constructor -- `immutables_hash`: Hash of the contract's compile-time immutable state +- `immutables_hash`: Hash of the contract's compile-time immutable state. [Initializerless accounts](./accounts/deployment.md) use it to commit the signing key into the address - `public_keys`: Public keys participating in address derivation (nullifier, incoming viewing, outgoing viewing, tagging, message-signing, and fallback keys). Only the incoming viewing key is held as an elliptic curve point; the other five are held as their `hash_public_key` digests. ### Instance Address diff --git a/docs/docs-developers/docs/foundational-topics/fees.md b/docs/docs-developers/docs/foundational-topics/fees.md index 8ff84586cdbc..adfbcaf88c81 100644 --- a/docs/docs-developers/docs/foundational-topics/fees.md +++ b/docs/docs-developers/docs/foundational-topics/fees.md @@ -103,7 +103,7 @@ In `aztec.js`, the `L1FeeJuicePortalManager` class handles the L1 side (token ap ### Payment methods -An account with Fee Juice can pay for its transactions directly. A new account can even pay for its own deployment transaction, provided Fee Juice was bridged to its address before deployment. +An account with Fee Juice can pay for its transactions directly. A new account can even pay for its own deployment transaction, provided Fee Juice was bridged to its address before deployment. [Initializerless accounts](./accounts/deployment.md) skip the deployment transaction entirely, so they only need fees once they send their first real transaction. Alternatively, accounts can use [fee-paying contracts (FPCs)](../aztec-js/how_to_pay_fees.md#use-fee-payment-contracts) to pay for transactions. An FPC holds its own Fee Juice balance to pay the protocol, and can accept other tokens from users in exchange. diff --git a/docs/docs-developers/docs/foundational-topics/wallets.md b/docs/docs-developers/docs/foundational-topics/wallets.md index 895570670b5c..c299f7405c2b 100644 --- a/docs/docs-developers/docs/foundational-topics/wallets.md +++ b/docs/docs-developers/docs/foundational-topics/wallets.md @@ -21,6 +21,8 @@ A wallet must support at least one specific account contract implementation, whi Note that users must be able to receive funds in Aztec before deploying their account. A wallet should let a user generate a [deterministic complete address](./accounts/keys.md#address-derivation) without having to interact with the network, so they can share it with others to receive funds. This requires that the wallet pins a specific contract implementation, its initialization arguments, a deployment salt, and the user's keys. These values yield a deterministic address, so when the account contract is actually deployed, it is available at the precalculated address. Once the account contract is deployed, the user can start sending transactions using it as the transaction origin. +Some account contracts avoid deployment altogether: [initializerless accounts](./accounts/deployment.md) commit their signing key into the address itself, so the wallet only needs to register them locally before they can start transacting. + ## Transaction lifecycle Every transaction in Aztec is broadcast to the network as a zero-knowledge proof of correct execution, in order to preserve privacy. This means that transaction proofs are generated on the wallet and not on a remote node. This is one of the biggest differences with regard to EVM chain wallets. From c0d24418e617d89bac2db5f8bde14dcd0afcebbf Mon Sep 17 00:00:00 2001 From: Martin Verzilli Date: Wed, 8 Jul 2026 14:23:43 +0200 Subject: [PATCH 15/35] fix: tweak depositToAztec gas config (#24607) While trying to deploy the latest nightly standard contracts to testnet I experienced gas estimation issues. According to Claude these issues are due to variability in Merkle tree insertions: there's around 40% of gas cost spread between the happiest case (no tree "climbing" needed) and the worst case (10 level climb). If eth_estimateGas occurs during the happiest case, and actual TX execution during the worst case, the default 20% buffer we use in Aztec.js turns out not to be sufficient. Closes F-794 (cherry picked from commit 2734da1e6b3036b2d1a64c6b8a9553a7a52e6017) --- .../aztec.js/src/ethereum/portal_manager.ts | 99 ++++++++++++++----- 1 file changed, 73 insertions(+), 26 deletions(-) diff --git a/yarn-project/aztec.js/src/ethereum/portal_manager.ts b/yarn-project/aztec.js/src/ethereum/portal_manager.ts index 5b913a6d2b4e..5f2113902cf0 100644 --- a/yarn-project/aztec.js/src/ethereum/portal_manager.ts +++ b/yarn-project/aztec.js/src/ethereum/portal_manager.ts @@ -1,3 +1,9 @@ +import { + type L1TxConfig, + type L1TxUtils, + createL1TxUtils, + getL1TxUtilsConfigEnvVars, +} from '@aztec/ethereum/l1-tx-utils'; import type { ExtendedViemWalletClient, ViemContract } from '@aztec/ethereum/types'; import { extractEvent } from '@aztec/ethereum/utils'; import type { EpochNumber } from '@aztec/foundation/branded-types'; @@ -16,7 +22,7 @@ import { computeL2ToL1MessageHash, computeSecretHash } from '@aztec/stdlib/hash' import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import { getL2ToL1MessageLeafId } from '@aztec/stdlib/messaging'; -import { type Hex, getContract, toFunctionSelector } from 'viem'; +import { type Hex, encodeFunctionData, getContract, toFunctionSelector } from 'viem'; /** L1 to L2 message info to claim it on L2. */ export type L2Claim = { @@ -51,10 +57,26 @@ export async function generateClaimSecret(logger?: Logger): Promise<[Fr, Fr]> { return [secret, secretHash]; } +// `Inbox.sendL2Message` (reached by every `depositToAztec*` call) inserts into a height-10 +// (`L1_TO_L2_MSG_SUBTREE_HEIGHT`) incremental frontier tree. The per-insert cost swings by up to ~10 hash levels +// (which translates to ~40k gas) depending on where the global message index lands when the tx is mined: inserts that +// complete a subtree cascade through cold SLOADs + an SSTORE vs cheap inserts that touch one slot. A point-in-time +// `eth_estimateGas` taken at a cheap index therefore under-sizes a deposit that mines at a subtree boundary. That ~40k +// swing exceeds the default 20% buffer on a ~100k deposit. That is why we raise the gas-limit buffer to 2x here. +// Note: a larger operator override via L1_GAS_LIMIT_BUFFER_PERCENTAGE still wins. +const INBOX_DEPOSIT_GAS_LIMIT_BUFFER_PERCENTAGE = 100; + +/** Per-call gas config for `depositToAztec*`: the Inbox-deposit buffer floor, or a larger operator override. */ +function inboxDepositGasConfig(): L1TxConfig { + const configuredBuffer = getL1TxUtilsConfigEnvVars().gasLimitBufferPercentage ?? 0; + return { gasLimitBufferPercentage: Math.max(configuredBuffer, INBOX_DEPOSIT_GAS_LIMIT_BUFFER_PERCENTAGE) }; +} + /** Helper for managing an ERC20 on L1. */ export class L1TokenManager { private contract: ViemContract; private handler: ViemContract | undefined; + private readonly l1TxUtils: L1TxUtils; public constructor( /** Address of the ERC20 contract. */ @@ -76,6 +98,7 @@ export class L1TokenManager { client: this.extendedClient, }); } + this.l1TxUtils = createL1TxUtils(this.extendedClient, { logger }, getL1TxUtilsConfigEnvVars()); } /** Returns the amount of tokens available to mint via the handler. @@ -108,8 +131,10 @@ export class L1TokenManager { const mintAmount = await this.getMintAmount(); this.logger.info(`Minting ${mintAmount} tokens for ${stringifyEthAddress(address, addressName)}`); // NOTE: the handler mints a fixed amount. - await this.extendedClient.waitForTransactionReceipt({ - hash: await this.handler.write.mint([address]), + await this.l1TxUtils.sendAndMonitorTransaction({ + to: this.handler.address, + abi: FeeAssetHandlerAbi, + data: encodeFunctionData({ abi: FeeAssetHandlerAbi, functionName: 'mint', args: [address] }), }); } @@ -121,8 +146,10 @@ export class L1TokenManager { */ public async approve(amount: bigint, address: Hex, addressName = '') { this.logger.info(`Approving ${amount} tokens for ${stringifyEthAddress(address, addressName)}`); - await this.extendedClient.waitForTransactionReceipt({ - hash: await this.contract.write.approve([address, amount]), + await this.l1TxUtils.sendAndMonitorTransaction({ + to: this.contract.address, + abi: TestERC20Abi, + data: encodeFunctionData({ abi: TestERC20Abi, functionName: 'approve', args: [address, amount] }), }); } } @@ -131,6 +158,7 @@ export class L1TokenManager { export class L1FeeJuicePortalManager { private readonly tokenManager: L1TokenManager; private readonly contract: ViemContract; + private readonly l1TxUtils: L1TxUtils; constructor( portalAddress: EthAddress, @@ -145,6 +173,7 @@ export class L1FeeJuicePortalManager { abi: FeeJuicePortalAbi, client: extendedClient, }); + this.l1TxUtils = createL1TxUtils(extendedClient, { logger }, getL1TxUtilsConfigEnvVars()); } /** Returns the associated token manager for the L1 ERC20. */ @@ -176,9 +205,14 @@ export class L1FeeJuicePortalManager { await this.contract.simulate.depositToAztecPublic(args); - const txReceipt = await this.extendedClient.waitForTransactionReceipt({ - hash: await this.contract.write.depositToAztecPublic(args), - }); + const { receipt: txReceipt } = await this.l1TxUtils.sendAndMonitorTransaction( + { + to: this.contract.address, + abi: FeeJuicePortalAbi, + data: encodeFunctionData({ abi: FeeJuicePortalAbi, functionName: 'depositToAztecPublic', args }), + }, + inboxDepositGasConfig(), + ); this.logger.info('Deposited to Aztec public successfully', { txReceipt }); @@ -249,6 +283,7 @@ export class L1FeeJuicePortalManager { export class L1ToL2TokenPortalManager { protected readonly portal: ViemContract; protected readonly tokenManager: L1TokenManager; + protected readonly l1TxUtils: L1TxUtils; constructor( portalAddress: EthAddress, @@ -263,6 +298,7 @@ export class L1ToL2TokenPortalManager { abi: TokenPortalAbi, client: extendedClient, }); + this.l1TxUtils = createL1TxUtils(extendedClient, { logger }, getL1TxUtilsConfigEnvVars()); } /** Returns the token manager for the underlying L1 token. */ @@ -280,15 +316,17 @@ export class L1ToL2TokenPortalManager { const [claimSecret, claimSecretHash] = await this.bridgeSetup(amount, mint); this.logger.info('Sending L1 tokens to L2 to be claimed publicly'); - const { request } = await this.portal.simulate.depositToAztecPublic([ - to.toString(), - amount, - claimSecretHash.toString(), - ]); - - const txReceipt = await this.extendedClient.waitForTransactionReceipt({ - hash: await this.extendedClient.writeContract(request), - }); + const args = [to.toString(), amount, claimSecretHash.toString()] as const; + await this.portal.simulate.depositToAztecPublic(args); + + const { receipt: txReceipt } = await this.l1TxUtils.sendAndMonitorTransaction( + { + to: this.portal.address, + abi: TokenPortalAbi, + data: encodeFunctionData({ abi: TokenPortalAbi, functionName: 'depositToAztecPublic', args }), + }, + inboxDepositGasConfig(), + ); const log = extractEvent( txReceipt.logs, @@ -334,11 +372,17 @@ export class L1ToL2TokenPortalManager { const [claimSecret, claimSecretHash] = await this.bridgeSetup(amount, mint); this.logger.info('Sending L1 tokens to L2 to be claimed privately'); - const { request } = await this.portal.simulate.depositToAztecPrivate([amount, claimSecretHash.toString()]); - - const txReceipt = await this.extendedClient.waitForTransactionReceipt({ - hash: await this.extendedClient.writeContract(request), - }); + const args = [amount, claimSecretHash.toString()] as const; + await this.portal.simulate.depositToAztecPrivate(args); + + const { receipt: txReceipt } = await this.l1TxUtils.sendAndMonitorTransaction( + { + to: this.portal.address, + abi: TokenPortalAbi, + data: encodeFunctionData({ abi: TokenPortalAbi, functionName: 'depositToAztecPrivate', args }), + }, + inboxDepositGasConfig(), + ); const log = extractEvent( txReceipt.logs, @@ -437,7 +481,7 @@ export class L1TokenPortalManager extends L1ToL2TokenPortalManager { } // Call function on L1 contract to consume the message - const { request: withdrawRequest } = await this.portal.simulate.withdraw([ + const withdrawArgs = [ recipient.toString(), amount, false, @@ -445,10 +489,13 @@ export class L1TokenPortalManager extends L1ToL2TokenPortalManager { BigInt(numCheckpointsInEpoch), messageIndex, siblingPath.toBufferArray().map((buf: Buffer): Hex => `0x${buf.toString('hex')}`), - ]); + ] as const; + await this.portal.simulate.withdraw(withdrawArgs); - await this.extendedClient.waitForTransactionReceipt({ - hash: await this.extendedClient.writeContract(withdrawRequest), + await this.l1TxUtils.sendAndMonitorTransaction({ + to: this.portal.address, + abi: TokenPortalAbi, + data: encodeFunctionData({ abi: TokenPortalAbi, functionName: 'withdraw', args: withdrawArgs }), }); const isConsumedAfter = await this.outbox.read.hasMessageBeenConsumedAtEpoch([BigInt(epochNumber), messageLeafId]); From 87b3e0d9f0577c48bc3d936e7a74a8715ca7c1bf Mon Sep 17 00:00:00 2001 From: Nicolas Chamo Date: Wed, 8 Jul 2026 14:13:32 -0300 Subject: [PATCH 16/35] feat(e2e): interactive handshake e2e (#24590) (cherry picked from commit 23e8e1ea00af85bfaa635343abc8cdd14416f52a) --- .../interactive_handshake_responder.ts | 125 ++++++++++++++++++ .../src/automine/delivery/onchain.test.ts | 57 +++++++- .../delivery/onchain_delivery_harness.ts | 56 ++++++-- .../src/handshake-registry/constants.ts | 13 ++ .../src/handshake-registry/index.ts | 1 + 5 files changed, 240 insertions(+), 12 deletions(-) create mode 100644 yarn-project/end-to-end/src/automine/delivery/interactive_handshake_responder.ts diff --git a/yarn-project/end-to-end/src/automine/delivery/interactive_handshake_responder.ts b/yarn-project/end-to-end/src/automine/delivery/interactive_handshake_responder.ts new file mode 100644 index 000000000000..e46d0fa2b4d5 --- /dev/null +++ b/yarn-project/end-to-end/src/automine/delivery/interactive_handshake_responder.ts @@ -0,0 +1,125 @@ +import { AztecAddress, type CompleteAddress } from '@aztec/aztec.js/addresses'; +import { DomainSeparator } from '@aztec/constants'; +import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto/poseidon'; +import { Schnorr, type SchnorrSignature } from '@aztec/foundation/crypto/schnorr'; +import { Fq, Fr } from '@aztec/foundation/curves/bn254'; +import type { GrumpkinScalar } from '@aztec/foundation/curves/grumpkin'; +import type { CustomRequest } from '@aztec/pxe/config'; +import { + INTERACTIVE_HANDSHAKE_REQUEST_KIND, + STANDARD_HANDSHAKE_REGISTRY_ADDRESS, +} from '@aztec/standard-contracts/handshake-registry/constants'; +import { type PublicKeys, derivePublicKeyFromSecretKey } from '@aztec/stdlib/keys'; + +/** + * The decoded payload of the registry's interactive-handshake signature request. Note it never carries the sender, + * so the recipient authorizes the handshake without learning who initiated it. + */ +export type InteractiveHandshakeRequest = { + /** The account whose authorization is being requested. */ + recipient: AztecAddress; + chainId: Fr; + version: Fr; + /** The x-coordinate of the handshake's ephemeral public key (its y-coordinate is positive by construction). */ + ephPkX: Fr; +}; + +/** + * The recipient's signed authorization for an interactive handshake. Mirrors the registry contract's + * `RecipientSignature` struct field for field. + */ +export type RecipientSignature = { + /** The recipient's master public keys, bound in-circuit to the recipient's address via `partialAddress`. */ + publicKeys: PublicKeys; + partialAddress: Fr; + /** The x-coordinate of the recipient's master message-signing public key. */ + mspkX: Fr; + /** Whether that key's y-coordinate is positive, so the circuit can reconstruct the point from `mspkX`. */ + mspkYIsPositive: boolean; + /** The schnorr signature over the handshake message. */ + signature: SchnorrSignature; +}; + +/** + * Parses and validates the registry's interactive-handshake signature request. + * + * @throws If the request kind is not {@link INTERACTIVE_HANDSHAKE_REQUEST_KIND}, the issuing contract is not the + * standard HandshakeRegistry, or the payload does not have the expected shape. + */ +export function parseInteractiveHandshakeRequest(request: CustomRequest): InteractiveHandshakeRequest { + if (!request.kind.equals(INTERACTIVE_HANDSHAKE_REQUEST_KIND)) { + throw new Error(`Not an interactive-handshake signature request: unexpected kind ${request.kind}`); + } + if (!request.contractAddress.equals(STANDARD_HANDSHAKE_REGISTRY_ADDRESS)) { + throw new Error( + `Interactive-handshake signature request issued by ${request.contractAddress}, expected the standard HandshakeRegistry at ${STANDARD_HANDSHAKE_REGISTRY_ADDRESS}`, + ); + } + if (request.payload.length !== 4) { + throw new Error(`Interactive-handshake signature request payload has ${request.payload.length} fields, expected 4`); + } + + const [recipient, chainId, version, ephPkX] = request.payload; + return { recipient: new AztecAddress(recipient), chainId, version, ephPkX }; +} + +/** + * Produces the recipient's signed authorization for an interactive handshake, signing with the master + * message-signing secret key. + */ +export async function signInteractiveHandshake( + request: InteractiveHandshakeRequest, + completeAddress: CompleteAddress, + masterMessageSigningSecretKey: GrumpkinScalar, +): Promise { + const mspk = await derivePublicKeyFromSecretKey(masterMessageSigningSecretKey); + const [mspkX, mspkYIsPositive] = mspk.toXAndSign(); + + const message = await computeInteractiveHandshakeSignatureMessage({ + chainId: request.chainId, + version: request.version, + registry: STANDARD_HANDSHAKE_REGISTRY_ADDRESS, + ephPkX: request.ephPkX, + }); + const signature = await new Schnorr().constructSignature(message, masterMessageSigningSecretKey); + + return { + publicKeys: completeAddress.publicKeys, + partialAddress: completeAddress.partialAddress, + mspkX, + mspkYIsPositive, + signature, + }; +} + +/** Serializes a {@link RecipientSignature} to the field layout the registry's in-circuit deserialization expects. */ +export function recipientSignatureToFields(recipientSignature: RecipientSignature): Fr[] { + const s = Fq.fromBuffer(recipientSignature.signature.s); + const e = Fq.fromBuffer(recipientSignature.signature.e); + return [ + ...recipientSignature.publicKeys.toFields(), + recipientSignature.partialAddress, + recipientSignature.mspkX, + new Fr(recipientSignature.mspkYIsPositive), + s.lo, + s.hi, + e.lo, + e.hi, + ]; +} + +/** + * The message an interactive-handshake authorization signs: the handshake's ephemeral key and chain context under + * `DomainSeparator.INTERACTIVE_HANDSHAKE_SIGNATURE`, exactly as the registry recomputes it in-circuit. + */ +function computeInteractiveHandshakeSignatureMessage(args: { + chainId: Fr; + version: Fr; + registry: AztecAddress; + ephPkX: Fr; +}): Promise { + return poseidon2HashWithSeparator( + [args.chainId, args.version, args.registry, args.ephPkX], + DomainSeparator.INTERACTIVE_HANDSHAKE_SIGNATURE, + ); +} diff --git a/yarn-project/end-to-end/src/automine/delivery/onchain.test.ts b/yarn-project/end-to-end/src/automine/delivery/onchain.test.ts index cdab97abcf31..0e0dfaeb5039 100644 --- a/yarn-project/end-to-end/src/automine/delivery/onchain.test.ts +++ b/yarn-project/end-to-end/src/automine/delivery/onchain.test.ts @@ -1,5 +1,15 @@ +import type { InitialAccountData } from '@aztec/accounts/testing'; +import type { CompleteAddress } from '@aztec/aztec.js/addresses'; import { Point } from '@aztec/foundation/curves/grumpkin'; +import type { ResolveCustomRequest } from '@aztec/pxe/config'; +import { deriveKeys } from '@aztec/stdlib/keys'; +import type { TestWallet } from '../../test-wallet/test_wallet.js'; +import { + parseInteractiveHandshakeRequest, + recipientSignatureToFields, + signInteractiveHandshake, +} from './interactive_handshake_responder.js'; import { buildMessageDeliveryTest } from './onchain_delivery_harness.js'; describe('onchain delivery', () => { @@ -34,8 +44,8 @@ describe('onchain delivery', () => { }, }); - // With the recipient registering the sender, - // the recipient PXE reconstructs the address-derived tag and discovers the delivery. + // With the recipient registering the sender, the recipient PXE reconstructs the address-derived tag + // and discovers the delivery. buildMessageDeliveryTest({ strategy: 'address-derived', mode: 'unconstrained', @@ -44,4 +54,47 @@ describe('onchain delivery', () => { await recipientWallet.registerSender(senderAddress); }, }); + + buildMessageDeliveryTest({ + strategy: 'interactive handshake', + mode: 'constrained', + senderHook: () => Promise.resolve({ type: 'interactive-handshake' }), + customRequestResponder: interactiveHandshakeResponder, + }); + + buildMessageDeliveryTest({ + strategy: 'interactive handshake', + mode: 'unconstrained', + senderHook: () => Promise.resolve({ type: 'interactive-handshake' }), + customRequestResponder: interactiveHandshakeResponder, + }); + + // Serves the registry's interactive-handshake signature request for the recipient: registers the handshake on the + // recipient PXE, then answers with the signed response. + function interactiveHandshakeResponder( + recipientWallet: TestWallet, + recipientAccount: InitialAccountData, + recipientCompleteAddress: CompleteAddress, + ): ResolveCustomRequest { + return async request => { + const parsed = parseInteractiveHandshakeRequest(request); + + // Register before signing. + await recipientWallet.registerTaggingSecretSource({ + kind: 'handshake', + recipient: parsed.recipient, + ephPk: parsed.ephPkX, + }); + + // The master message-signing secret key is deliberately never held by PXE or the key store; the wallet + // derives it client-side from the account secret. + const { masterMessageSigningSecretKey } = await deriveKeys(recipientAccount.secret); + const recipientSignature = await signInteractiveHandshake( + parsed, + recipientCompleteAddress, + masterMessageSigningSecretKey, + ); + return recipientSignatureToFields(recipientSignature); + }; + } }); diff --git a/yarn-project/end-to-end/src/automine/delivery/onchain_delivery_harness.ts b/yarn-project/end-to-end/src/automine/delivery/onchain_delivery_harness.ts index d10ae9196aea..6e00fca5969e 100644 --- a/yarn-project/end-to-end/src/automine/delivery/onchain_delivery_harness.ts +++ b/yarn-project/end-to-end/src/automine/delivery/onchain_delivery_harness.ts @@ -1,11 +1,12 @@ import { type InitialAccountData, generateSchnorrAccounts } from '@aztec/accounts/testing'; import type { FieldLike } from '@aztec/aztec.js/abi'; import { NO_FROM } from '@aztec/aztec.js/account'; -import type { AztecAddress } from '@aztec/aztec.js/addresses'; +import type { AztecAddress, CompleteAddress } from '@aztec/aztec.js/addresses'; import type { AztecNode } from '@aztec/aztec.js/node'; +import type { AccountManager } from '@aztec/aztec.js/wallet'; import { BlockNumber } from '@aztec/foundation/branded-types'; import { type DeliveryEvent, OnchainDeliveryTestContract } from '@aztec/noir-test-contracts.js/OnchainDeliveryTest'; -import type { PXECreationOptions } from '@aztec/pxe/server'; +import type { CustomRequest, ResolveCustomRequest, ResolveTaggingSecretStrategy } from '@aztec/pxe/config'; import type { AztecNodeDebug } from '@aztec/stdlib/interfaces/client'; import { jest } from '@jest/globals'; @@ -14,9 +15,13 @@ import { AUTOMINE_E2E_OPTS } from '../../fixtures/fixtures.js'; import { ensureHandshakeRegistryPublished, setup, setupPXEAndGetWallet } from '../../fixtures/setup.js'; import { TestWallet } from '../../test-wallet/test_wallet.js'; -// The wallet hook that selects a message's tagging-secret source. Derived from the exported PXE options -// rather than importing the hook type, which `@aztec/pxe/server` does not re-export. -export type SenderHook = NonNullable['resolveTaggingSecretStrategy']>; +// Builds the hook serving custom requests issued during the sender's simulations. Installed on the sender PXE at +// creation but built only once the recipient exists, since serving typically needs the recipient's wallet and keys. +export type CustomRequestResponder = ( + recipient: TestWallet, + recipientAccount: InitialAccountData, + recipientCompleteAddress: CompleteAddress, +) => ResolveCustomRequest; export type Mode = 'constrained' | 'unconstrained'; @@ -40,19 +45,22 @@ export function buildMessageDeliveryTest(opts: { strategy: string; mode: DeliveryMode; // Required: every cell states its source explicitly rather than leaning on the PXE default (covered by unit tests). - senderHook: SenderHook; + senderHook: ResolveTaggingSecretStrategy; // Recipient-side setup the source requires (e.g. registering a raw arbitrary secret); runs once after deployment. recipientRegistration?: ( recipient: TestWallet, recipientAddress: AztecAddress, sender: AztecAddress, ) => Promise; + // Serves the custom requests issued during the sender's simulations (e.g. the registry's interactive-handshake + // signature request). + customRequestResponder?: CustomRequestResponder; // Extra `it()`s to register in this cell's suite, e.g. assertions against state a custom `senderHook` recorded. // Called inside the same `describe`, after the two baseline assertions below, so it shares their `beforeAll` // instead of depending on Jest's cross-`describe` execution order. additionalTests?: () => void; }) { - const { strategy, mode, senderHook, recipientRegistration, additionalTests } = opts; + const { strategy, mode, senderHook, recipientRegistration, customRequestResponder, additionalTests } = opts; const description = `${strategy} x ${formatMode(mode)}`; describe(description, () => { @@ -84,11 +92,14 @@ export function buildMessageDeliveryTest(opts: { ? contractSender.methods.emit_note(recipient, value) : contractSender.methods.emit_note_unconstrained(recipient, value); + let additionallyFundedAccounts: InitialAccountData[]; + let recipientAccount: AccountManager | undefined; + let customRequestCount = 0; + beforeAll(async () => { // The sender PXE holds the sender and carries this cell's tagging-secret-strategy hook. The recipient is funded // at genesis here but created and deployed on the isolated recipient PXE below, so it carries no sender state // from other cells. - let additionallyFundedAccounts: InitialAccountData[]; ({ aztecNode, additionallyFundedAccounts, @@ -98,7 +109,26 @@ export function buildMessageDeliveryTest(opts: { } = await setup(1, { ...AUTOMINE_E2E_OPTS, additionallyFundedAccounts: await generateSchnorrAccounts(1, 'schnorr'), - pxeCreationOptions: { hooks: { resolveTaggingSecretStrategy: senderHook } }, + pxeCreationOptions: { + hooks: { + resolveTaggingSecretStrategy: senderHook, + resolveCustomRequest: async (request: CustomRequest) => { + if (!customRequestResponder) { + throw new Error('A custom request arrived but this test cell has no customRequestResponder configured'); + } + if (!recipientAccount) { + throw new Error('A custom request arrived before the recipient wallet was created'); + } + customRequestCount++; + const respond = customRequestResponder( + walletRecipient, + additionallyFundedAccounts[0], + await recipientAccount.getCompleteAddress(), + ); + return respond(request); + }, + }, + }, })); ({ wallet: walletRecipient, teardown: teardownRecipient } = await setupPXEAndGetWallet( @@ -108,7 +138,7 @@ export function buildMessageDeliveryTest(opts: { undefined, 'pxe-recipient', )); - const recipientAccount = await walletRecipient.createSchnorrAccount( + recipientAccount = await walletRecipient.createSchnorrAccount( additionallyFundedAccounts[0].secret, additionallyFundedAccounts[0].salt, additionallyFundedAccounts[0].signingKey, @@ -172,6 +202,12 @@ export function buildMessageDeliveryTest(opts: { expect(readNotes).toEqual(noteValues); }); + if (customRequestResponder) { + it('the custom request hook fires exactly once, on the send that bootstraps the tagging secret', () => { + expect(customRequestCount).toBe(1); + }); + } + additionalTests?.(); }); } diff --git a/yarn-project/standard-contracts/src/handshake-registry/constants.ts b/yarn-project/standard-contracts/src/handshake-registry/constants.ts index 48b4cd153728..0573b2b125d5 100644 --- a/yarn-project/standard-contracts/src/handshake-registry/constants.ts +++ b/yarn-project/standard-contracts/src/handshake-registry/constants.ts @@ -1,6 +1,8 @@ // Lightweight metadata leaf export for browser bundles: importing from // `@aztec/standard-contracts/handshake-registry/constants` avoids dragging in the // `HandshakeRegistry.json` static import. +import { sha256ToField } from '@aztec/foundation/crypto/sha256'; +import type { Fr } from '@aztec/foundation/curves/bn254'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; import { StandardContractAddress, StandardContractClassId, StandardContractSalt } from '../standard_contract_data.js'; @@ -8,3 +10,14 @@ import { StandardContractAddress, StandardContractClassId, StandardContractSalt export const STANDARD_HANDSHAKE_REGISTRY_ADDRESS: AztecAddress = StandardContractAddress.HandshakeRegistry; export const STANDARD_HANDSHAKE_REGISTRY_CLASS_ID = StandardContractClassId.HandshakeRegistry; export const STANDARD_HANDSHAKE_REGISTRY_SALT = StandardContractSalt.HandshakeRegistry; + +/** + * Request kind under which the HandshakeRegistry asks for a recipient's interactive-handshake signature through the + * `resolveCustomRequest` hook. Mirrors `INTERACTIVE_HANDSHAKE_REQUEST_KIND` in the registry contract. + */ +// TODO: remove this mirrored constant and read the value from the HandshakeRegistry artifact once the contract +// global can be `#[abi]`-exported. Fixed upstream but not yet released: +// https://github.com/noir-lang/noir/pull/12714 and https://github.com/noir-lang/noir/issues/12620. +export const INTERACTIVE_HANDSHAKE_REQUEST_KIND: Fr = sha256ToField([ + Buffer.from('HANDSHAKE_REGISTRY::INTERACTIVE_HANDSHAKE_REQUEST'), +]); diff --git a/yarn-project/standard-contracts/src/handshake-registry/index.ts b/yarn-project/standard-contracts/src/handshake-registry/index.ts index f10e7186a45f..2cc367647d2e 100644 --- a/yarn-project/standard-contracts/src/handshake-registry/index.ts +++ b/yarn-project/standard-contracts/src/handshake-registry/index.ts @@ -6,6 +6,7 @@ import { makeStandardContract } from '../make_standard_contract.js'; import type { StandardContract } from '../standard_contract.js'; export { + INTERACTIVE_HANDSHAKE_REQUEST_KIND, STANDARD_HANDSHAKE_REGISTRY_ADDRESS, STANDARD_HANDSHAKE_REGISTRY_CLASS_ID, STANDARD_HANDSHAKE_REGISTRY_SALT, From 77d31f8005562a681a3ea5eb8574165e275da563 Mon Sep 17 00:00:00 2001 From: aminsammara Date: Thu, 9 Jul 2026 09:50:05 +0000 Subject: [PATCH 17/35] fix(aztec.js): give waitForNode a bounded default timeout waitForNode called retryUntil without a timeout, and retryUntil treats a zero timeout as the never-time-out sentinel, so an unreachable node caused the call to poll getNodeInfo forever with no way for the caller to bound it. Add an optional { timeout, interval } argument and default it to DefaultWaitOpts (300s / 1s), matching the sibling waitForTx. An unreachable node now rejects with a TimeoutError; pass timeout: 0 to opt into the previous infinite-wait behavior. (cherry picked from commit 482a6b86532bb8fb5f83c57d37230025c581fcce) --- yarn-project/aztec.js/src/utils/node.test.ts | 24 ++++++++++++- yarn-project/aztec.js/src/utils/node.ts | 37 +++++++++++++------- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/yarn-project/aztec.js/src/utils/node.test.ts b/yarn-project/aztec.js/src/utils/node.test.ts index 18ea35d88e06..deec8998aba5 100644 --- a/yarn-project/aztec.js/src/utils/node.test.ts +++ b/yarn-project/aztec.js/src/utils/node.test.ts @@ -1,4 +1,5 @@ import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { TimeoutError } from '@aztec/foundation/error'; import { BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import { @@ -14,7 +15,7 @@ import { import { type MockProxy, mock } from 'jest-mock-extended'; -import { waitForTx } from './node.js'; +import { waitForNode, waitForTx } from './node.js'; describe('waitForTx', () => { let node: MockProxy; @@ -137,3 +138,24 @@ describe('waitForTx', () => { }); }); }); + +describe('waitForNode', () => { + let node: MockProxy; + + beforeEach(() => { + node = mock(); + }); + + it('resolves once the node becomes reachable', async () => { + node.getNodeInfo.mockRejectedValueOnce(new Error('not up yet')).mockResolvedValueOnce({} as any); + + await expect(waitForNode(node, undefined, { timeout: 5, interval: 0.01 })).resolves.toBeUndefined(); + expect(node.getNodeInfo).toHaveBeenCalledTimes(2); + }); + + it('rejects with a TimeoutError when the node stays unreachable', async () => { + node.getNodeInfo.mockRejectedValue(new Error('unreachable')); + + await expect(waitForNode(node, undefined, { timeout: 0.05, interval: 0.01 })).rejects.toThrow(TimeoutError); + }); +}); diff --git a/yarn-project/aztec.js/src/utils/node.ts b/yarn-project/aztec.js/src/utils/node.ts index ee88b68a7159..7f92a1a77948 100644 --- a/yarn-project/aztec.js/src/utils/node.ts +++ b/yarn-project/aztec.js/src/utils/node.ts @@ -6,18 +6,31 @@ import { SortedTxStatuses, TxStatus } from '@aztec/stdlib/tx'; import { DefaultWaitOpts, type WaitOpts } from '../contract/wait_opts.js'; -export const waitForNode = async (node: AztecNode, logger?: Logger) => { - await retryUntil(async () => { - try { - logger?.verbose('Attempting to contact Aztec node...'); - await node.getNodeInfo(); - logger?.verbose('Contacted Aztec node'); - return true; - } catch { - logger?.verbose('Failed to contact Aztec Node'); - } - return undefined; - }, 'RPC Get Node Info'); +/** + * Waits for an Aztec node to become reachable, polling {@link AztecNode.getNodeInfo} until it succeeds. + * @param node - The Aztec node to contact. + * @param logger - Optional logger for polling progress. + * @param opts - Optional timeout and interval (in seconds). `timeout` defaults to {@link DefaultWaitOpts.timeout}; + * pass `timeout: 0` to wait indefinitely. + * @throws TimeoutError if the node stays unreachable past the timeout. + */ +export const waitForNode = async (node: AztecNode, logger?: Logger, opts?: Pick) => { + await retryUntil( + async () => { + try { + logger?.verbose('Attempting to contact Aztec node...'); + await node.getNodeInfo(); + logger?.verbose('Contacted Aztec node'); + return true; + } catch { + logger?.verbose('Failed to contact Aztec Node'); + } + return undefined; + }, + 'RPC Get Node Info', + opts?.timeout ?? DefaultWaitOpts.timeout, + opts?.interval ?? DefaultWaitOpts.interval, + ); }; /** Returns true if the receipt status is at least the desired status level. */ From 6c22fe75d9080b378a7691a08a212ad0699b041f Mon Sep 17 00:00:00 2001 From: Martin Verzilli Date: Thu, 9 Jul 2026 14:21:44 +0200 Subject: [PATCH 18/35] refactor: cache Aztec node reads per execution (#24630) First part of integrating the enhancements explored at https://github.com/aztec-labs-eng/oxide/pull/257 Closes F-808 (cherry picked from commit 8a3f7e6b63ccce028fef95faa7a2a25423a4b158) --- .../aztec_node_read_cache.test.ts | 109 ++++++++++++++++++ .../aztec_node_read_cache.ts | 89 ++++++++++++++ .../oracle/utility_execution.test.ts | 102 +++++++++++++++- .../oracle/utility_execution_oracle.ts | 29 ++--- 4 files changed, 313 insertions(+), 16 deletions(-) create mode 100644 yarn-project/pxe/src/contract_function_simulator/aztec_node_read_cache.test.ts create mode 100644 yarn-project/pxe/src/contract_function_simulator/aztec_node_read_cache.ts diff --git a/yarn-project/pxe/src/contract_function_simulator/aztec_node_read_cache.test.ts b/yarn-project/pxe/src/contract_function_simulator/aztec_node_read_cache.test.ts new file mode 100644 index 000000000000..133602298e3c --- /dev/null +++ b/yarn-project/pxe/src/contract_function_simulator/aztec_node_read_cache.test.ts @@ -0,0 +1,109 @@ +import { ARCHIVE_HEIGHT } from '@aztec/constants'; +import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { promiseWithResolvers } from '@aztec/foundation/promise'; +import { MembershipWitness } from '@aztec/foundation/trees'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; +import { BlockHash } from '@aztec/stdlib/block'; +import type { AztecNode } from '@aztec/stdlib/interfaces/server'; +import { PublicDataWitness } from '@aztec/stdlib/trees'; +import { MinedTxReceipt, TxEffect, TxExecutionResult, TxHash, TxStatus } from '@aztec/stdlib/tx'; + +import { mock } from 'jest-mock-extended'; + +import { AztecNodeReadCache } from './aztec_node_read_cache.js'; + +describe('AztecNodeReadCache', () => { + let aztecNode: ReturnType>; + let cache: AztecNodeReadCache; + + const makeMinedReceipt = (txHash: TxHash) => + new MinedTxReceipt( + txHash, + TxStatus.FINALIZED, + TxExecutionResult.SUCCESS, + 0n, + BlockHash.random(), + BlockNumber(1), + SlotNumber(1), + 0, + EpochNumber(1), + TxEffect.empty(), + ); + + beforeEach(() => { + aztecNode = mock(); + cache = new AztecNodeReadCache(aztecNode); + }); + + it('shares concurrent tx receipt reads', async () => { + const txHash = TxHash.random(); + const deferred = promiseWithResolvers>(); + aztecNode.getTxReceipt.mockReturnValue(deferred.promise); + + const first = cache.getTxReceiptWithEffect(txHash); + const second = cache.getTxReceiptWithEffect(txHash); + const receipt = makeMinedReceipt(txHash); + deferred.resolve(receipt); + + await expect(Promise.all([first, second])).resolves.toEqual([receipt, receipt]); + expect(aztecNode.getTxReceipt).toHaveBeenCalledTimes(1); + }); + + it('evicts rejected reads so callers can retry', async () => { + const txHash = TxHash.random(); + const receipt = makeMinedReceipt(txHash); + aztecNode.getTxReceipt.mockRejectedValueOnce(new Error('temporary failure')); + aztecNode.getTxReceipt.mockResolvedValueOnce(receipt); + + await expect(cache.getTxReceiptWithEffect(txHash)).rejects.toThrow('temporary failure'); + await expect(cache.getTxReceiptWithEffect(txHash)).resolves.toBe(receipt); + expect(aztecNode.getTxReceipt).toHaveBeenCalledTimes(2); + }); + + it('keeps cache entries separate by method and arguments', async () => { + const blockHash = BlockHash.random(); + const leafSlot = Fr.random(); + const blockWitness = MembershipWitness.empty(ARCHIVE_HEIGHT); + const publicDataWitness = PublicDataWitness.random(); + aztecNode.getBlockHashMembershipWitness.mockResolvedValue(blockWitness); + aztecNode.getPublicDataWitness.mockResolvedValue(publicDataWitness); + + await expect(cache.getBlockHashMembershipWitness(blockHash, blockHash)).resolves.toBe(blockWitness); + await expect(cache.getPublicDataWitness(blockHash, leafSlot)).resolves.toBe(publicDataWitness); + + expect(aztecNode.getBlockHashMembershipWitness).toHaveBeenCalledTimes(1); + expect(aztecNode.getPublicDataWitness).toHaveBeenCalledTimes(1); + }); + + it('caches successful undefined results', async () => { + const referenceBlockHash = BlockHash.random(); + const blockHash = BlockHash.random(); + aztecNode.getBlockHashMembershipWitness.mockResolvedValue(undefined); + + await expect(cache.getBlockHashMembershipWitness(referenceBlockHash, blockHash)).resolves.toBeUndefined(); + await expect(cache.getBlockHashMembershipWitness(referenceBlockHash, blockHash)).resolves.toBeUndefined(); + + expect(aztecNode.getBlockHashMembershipWitness).toHaveBeenCalledTimes(1); + }); + + it('reuses cached slots across overlapping public storage ranges', async () => { + const blockHash = BlockHash.random(); + const contractAddress = await AztecAddress.random(); + const startStorageSlot = new Fr(100); + aztecNode.getPublicStorageAt.mockImplementation((_block, _contract, slot) => + Promise.resolve(new Fr(slot.value + 1n)), + ); + + await expect(cache.getPublicStorageRange(blockHash, contractAddress, startStorageSlot, 2)).resolves.toEqual([ + new Fr(101), + new Fr(102), + ]); + await expect(cache.getPublicStorageRange(blockHash, contractAddress, new Fr(101), 2)).resolves.toEqual([ + new Fr(102), + new Fr(103), + ]); + + expect(aztecNode.getPublicStorageAt).toHaveBeenCalledTimes(3); + }); +}); diff --git a/yarn-project/pxe/src/contract_function_simulator/aztec_node_read_cache.ts b/yarn-project/pxe/src/contract_function_simulator/aztec_node_read_cache.ts new file mode 100644 index 000000000000..ce7ff940fe61 --- /dev/null +++ b/yarn-project/pxe/src/contract_function_simulator/aztec_node_read_cache.ts @@ -0,0 +1,89 @@ +import { Fr } from '@aztec/foundation/curves/bn254'; +import type { AztecAddress } from '@aztec/stdlib/aztec-address'; +import type { BlockHash, BlockParameter } from '@aztec/stdlib/block'; +import type { AztecNode } from '@aztec/stdlib/interfaces/server'; +import type { TxHash } from '@aztec/stdlib/tx'; + +/** + * Per-execution cache for immutable Aztec node reads. + */ +export class AztecNodeReadCache { + private readonly cache = new Map>(); + + constructor(private readonly aztecNode: AztecNode) {} + + /** Fetches a block without reissuing the same node request. */ + public getBlock(block: BlockParameter) { + return this.#cachedRead(`block:${this.#keyPart(block)}`, () => this.aztecNode.getBlock(block)); + } + + /** Fetches a transaction receipt with its effect attached. */ + public getTxReceiptWithEffect(txHash: TxHash) { + return this.#cachedRead(`tx-receipt-with-effect:${txHash.toString()}`, () => + this.aztecNode.getTxReceipt(txHash, { includeTxEffect: true } as const), + ); + } + + /** Fetches an archive-tree witness for a block hash. */ + public getBlockHashMembershipWitness(referenceBlock: BlockParameter, blockHash: BlockHash) { + return this.#cachedRead( + `block-hash-membership-witness:${this.#keyPart(referenceBlock)}:${blockHash.toString()}`, + () => this.aztecNode.getBlockHashMembershipWitness(referenceBlock, blockHash), + ); + } + + /** Fetches a public-data-tree witness for a leaf slot. */ + public getPublicDataWitness(referenceBlock: BlockParameter, leafSlot: Fr) { + return this.#cachedRead(`public-data-witness:${this.#keyPart(referenceBlock)}:${leafSlot.toString()}`, () => + this.aztecNode.getPublicDataWitness(referenceBlock, leafSlot), + ); + } + + /** Fetches public storage for a single slot. */ + public getPublicStorageAt(referenceBlock: BlockParameter, contractAddress: AztecAddress, storageSlot: Fr) { + return this.#cachedRead( + `public-storage:${this.#keyPart(referenceBlock)}:${contractAddress.toString()}:${storageSlot.toString()}`, + () => this.aztecNode.getPublicStorageAt(referenceBlock, contractAddress, storageSlot), + ); + } + + /** Fetches a contiguous public storage range, reusing cached reads for overlapping slots. */ + public getPublicStorageRange( + referenceBlock: BlockParameter, + contractAddress: AztecAddress, + startStorageSlot: Fr, + numberOfElements: number, + ) { + const slots = Array(numberOfElements) + .fill(0) + .map((_, i) => new Fr(startStorageSlot.value + BigInt(i))); + + return Promise.all(slots.map(storageSlot => this.getPublicStorageAt(referenceBlock, contractAddress, storageSlot))); + } + + #cachedRead(key: string, fetch: () => Promise): Promise { + const cached = this.cache.get(key); + if (cached) { + return cached as Promise; + } + + const promise = fetch(); + promise.catch(() => this.cache.delete(key)); + this.cache.set(key, promise); + return promise; + } + + #keyPart(value: unknown): string { + if (['string', 'number', 'bigint', 'boolean'].includes(typeof value)) { + return String(value); + } + if (value && typeof value === 'object') { + const toString = (value as { toString?: () => string }).toString; + if (toString && toString !== Object.prototype.toString) { + return toString.call(value); + } + return JSON.stringify(value, (_key, nested) => (typeof nested === 'bigint' ? nested.toString() : nested)); + } + return String(value); + } +} diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts index 0e0c439e9f89..6e5c956302f8 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts @@ -1,7 +1,9 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { ARCHIVE_HEIGHT } from '@aztec/constants'; +import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { Grumpkin } from '@aztec/foundation/crypto/grumpkin'; import { Fr } from '@aztec/foundation/curves/bn254'; import { GrumpkinScalar } from '@aztec/foundation/curves/grumpkin'; +import { MembershipWitness } from '@aztec/foundation/trees'; import type { KeyStore } from '@aztec/key-store'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { StatefulTestContractArtifact } from '@aztec/noir-test-contracts.js/StatefulTest'; @@ -28,7 +30,17 @@ import { PublicKeys, deriveKeys, hashPublicKey } from '@aztec/stdlib/keys'; import { AppTaggingSecret, AppTaggingSecretKind, SiloedTag } from '@aztec/stdlib/logs'; import { Note, NoteDao } from '@aztec/stdlib/note'; import { makeL2Tips, randomContractInstanceWithAddress } from '@aztec/stdlib/testing'; -import { BlockHeader, CallContext, Capsule, GlobalVariables, TxHash } from '@aztec/stdlib/tx'; +import { + BlockHeader, + CallContext, + Capsule, + GlobalVariables, + MinedTxReceipt, + TxEffect, + TxExecutionResult, + TxHash, + TxStatus, +} from '@aztec/stdlib/tx'; import { mock } from 'jest-mock-extended'; import type { _MockProxy } from 'jest-mock-extended/lib/Mock.js'; @@ -657,6 +669,92 @@ describe('Utility Execution test suite', () => { }); }); + describe('node read cache', () => { + const makeMinedReceipt = (txHash: TxHash, txEffect = TxEffect.empty()) => + new MinedTxReceipt( + txHash, + TxStatus.FINALIZED, + TxExecutionResult.SUCCESS, + 0n, + BlockHash.random(), + BlockNumber(syncedBlockNumber), + SlotNumber(1), + 0, + EpochNumber(1), + txEffect, + ); + + it('reuses tx-effect reads within a utility execution', async () => { + const oracle = makeOracle({ scopes: [scope] }); + const txHash = TxHash.random(); + aztecNode.getTxReceipt.mockResolvedValue(makeMinedReceipt(txHash)); + + const first = await oracle.getTxEffect(txHash); + const second = await oracle.getTxEffect(txHash); + + expect(first).toEqual(second); + expect(aztecNode.getTxReceipt).toHaveBeenCalledTimes(1); + }); + + it('does not share cached reads across utility executions', async () => { + const txHash = TxHash.random(); + aztecNode.getTxReceipt.mockResolvedValue(makeMinedReceipt(txHash)); + + const firstOracle = makeOracle({ scopes: [scope] }); + const secondOracle = makeOracle({ scopes: [scope] }); + + // Cache works as usual + await firstOracle.getTxEffect(txHash); + await firstOracle.getTxEffect(txHash); + expect(aztecNode.getTxReceipt).toHaveBeenCalledTimes(1); + + // Same call in new oracle, goes to actual node since cache is clean + await secondOracle.getTxEffect(txHash); + expect(aztecNode.getTxReceipt).toHaveBeenCalledTimes(2); + }); + + it('does not cache failed tx-effect reads', async () => { + const oracle = makeOracle({ scopes: [scope] }); + const txHash = TxHash.random(); + aztecNode.getTxReceipt.mockRejectedValueOnce(new Error('temporary failure')); + aztecNode.getTxReceipt.mockResolvedValueOnce(makeMinedReceipt(txHash)); + + await expect(oracle.getTxEffect(txHash)).rejects.toThrow('temporary failure'); + + const result = await oracle.getTxEffect(txHash); + + expect(result.isSome()).toBe(true); + expect(aztecNode.getTxReceipt).toHaveBeenCalledTimes(2); + }); + + it('reuses archive witness reads within a utility execution', async () => { + const oracle = makeOracle({ scopes: [scope] }); + const referenceBlockHash = await anchorBlockHeader.hash(); + const blockHash = BlockHash.random(); + const witness = MembershipWitness.empty(ARCHIVE_HEIGHT); + aztecNode.getBlockHashMembershipWitness.mockResolvedValue(witness); + + const first = await oracle.getBlockHashMembershipWitness(referenceBlockHash, blockHash); + const second = await oracle.getBlockHashMembershipWitness(referenceBlockHash, blockHash); + + expect(first).toEqual(second); + expect(aztecNode.getBlockHashMembershipWitness).toHaveBeenCalledTimes(1); + }); + + it('reuses public storage reads within a utility execution', async () => { + const oracle = makeOracle({ scopes: [scope] }); + const blockHash = await anchorBlockHeader.hash(); + const startStorageSlot = Fr.random(); + aztecNode.getPublicStorageAt.mockResolvedValue(new Fr(7)); + + const first = await oracle.getFromPublicStorage(blockHash, contractAddress, startStorageSlot, 2); + const second = await oracle.getFromPublicStorage(blockHash, contractAddress, startStorageSlot, 2); + + expect(first).toEqual(second); + expect(aztecNode.getPublicStorageAt).toHaveBeenCalledTimes(2); + }); + }); + describe('fact store', () => { const service = new EphemeralArrayService(); const typeId = new Fr(10); diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts index 79c732bec2c4..0f318021b75f 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts @@ -56,6 +56,7 @@ import type { PrivateEventStore } from '../../storage/private_event_store/privat import type { RecipientTaggingStore } from '../../storage/tagging_store/recipient_tagging_store.js'; import type { TaggingSecretSourcesStore } from '../../storage/tagging_store/tagging_secret_sources_store.js'; import type { AnchoredContractData } from '../anchored_contract_data.js'; +import { AztecNodeReadCache } from '../aztec_node_read_cache.js'; import { EphemeralArrayService } from '../ephemeral_array_service.js'; import { BoundedVec } from '../noir-structs/bounded_vec.js'; import type { EmbeddedCurvePoint } from '../noir-structs/embedded_curve_point.js'; @@ -119,6 +120,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra private offchainEffects: OffchainEffect[] = []; private readonly ephemeralArrayService = new EphemeralArrayService(); protected readonly transientArrayService: TransientArrayService; + private readonly aztecNodeReadCache: AztecNodeReadCache; // We store oracle version to be able to show a nice error message when an oracle handler is missing. private contractOracleVersion: { major: number; minor: number } | undefined; @@ -172,6 +174,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra this.hooks = args.hooks; this.utilityExecutor = args.utilityExecutor; this.transientArrayService = args.transientArrayService; + this.aztecNodeReadCache = new AztecNodeReadCache(args.aztecNode); } public assertCompatibleOracleVersion(major: number, minor: number): void { @@ -265,7 +268,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra // hash at all. If the block hash did not exist by the reference block hash, then the node will not return the // membership witness as there is none. const witness = await this.#queryWithBlockHashNotAfterAnchor(referenceBlockHash, () => - this.aztecNode.getBlockHashMembershipWitness(referenceBlockHash, blockHash), + this.aztecNodeReadCache.getBlockHashMembershipWitness(referenceBlockHash, blockHash), ); return witness ? Option.some(witness) : Option.none(); } @@ -318,7 +321,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra */ public async getPublicDataWitness(blockHash: BlockHash, leafSlot: Fr): Promise { const witness = await this.#queryWithBlockHashNotAfterAnchor(blockHash, () => - this.aztecNode.getPublicDataWitness(blockHash, leafSlot), + this.aztecNodeReadCache.getPublicDataWitness(blockHash, leafSlot), ); if (!witness) { throw new Error(`Public data witness not found for slot ${leafSlot} at block hash ${blockHash.toString()}.`); @@ -511,16 +514,16 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra numberOfElements: number, ) { return this.#queryWithBlockHashNotAfterAnchor(blockHash, async () => { - const slots = Array(numberOfElements) - .fill(0) - .map((_, i) => new Fr(startStorageSlot.value + BigInt(i))); - - const values = await Promise.all( - slots.map(storageSlot => this.aztecNode.getPublicStorageAt(blockHash, contractAddress, storageSlot)), + const values = await this.aztecNodeReadCache.getPublicStorageRange( + blockHash, + contractAddress, + startStorageSlot, + numberOfElements, ); this.logger.debug( - `Oracle storage read: slots=[${slots.map(slot => slot.toString()).join(', ')}] address=${contractAddress.toString()} values=[${values.join(', ')}]`, + `Oracle storage read: start=${startStorageSlot.toString()} count=${numberOfElements} ` + + `address=${contractAddress.toString()} values=[${values.join(', ')}]`, ); return values; @@ -666,7 +669,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra throw new Error('Invalid tx hash passed into aztec_utl_getTxEffect oracle handler'); } - const receipt = await this.aztecNode.getTxReceipt(txHash, { includeTxEffect: true }); + const receipt = await this.aztecNodeReadCache.getTxReceiptWithEffect(txHash); if (!receipt.isMined() || !receipt.txEffect || receipt.blockNumber > this.anchorBlockHeader.getBlockNumber()) { return Option.none(); } @@ -1079,9 +1082,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra */ async #fetchTxEffects(txHashes: TxHash[]): Promise> { const uniqueTxHashes = uniqueBy(txHashes, h => h.toString()); - const fetched = await Promise.all( - uniqueTxHashes.map(h => this.aztecNode.getTxReceipt(h, { includeTxEffect: true })), - ); + const fetched = await Promise.all(uniqueTxHashes.map(h => this.aztecNodeReadCache.getTxReceiptWithEffect(h))); return new Map( uniqueTxHashes .map((h, i): [string, IndexedTxEffect | undefined] => { @@ -1115,7 +1116,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra const [response] = await Promise.all([ query(), (async () => { - const block = await this.aztecNode.getBlock(blockHash); + const block = await this.aztecNodeReadCache.getBlock(blockHash); const header = block?.header; if (!header) { throw new Error(`Could not find block header for block hash ${blockHash}`); From 9313618590872cb12e91b1d9bdf509021a6cd2b9 Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:59:22 +0100 Subject: [PATCH 19/35] fix(prover-node): rebuild pruned checkpoint provers and recover the epoch (#24436) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves A-1290. ## Problem A `CheckpointProver`'s sub-tree work forks world-state per block. When an L1 reorg prunes a base block, those fork reads fault and the prover permanently rejects its one-shot `blockProofs` promise. The store keeps the prover alive across a prune (`markPruned`) so a re-add of identical content can reuse its in-flight work. But a prune that removes the checkpoint also breaks the fork reads, so that reuse only ever hands back a poisoned prover: a re-add or a full `EpochSession` recreate re-references the same rejected `blockProofs` and fails immediately. The affected node then silently abandons the epoch until a process restart (a healthy prover elsewhere still proves it, so it's not a network stall). ## Fix **Drop the reuse machinery — a pruned prover cannot survive, so rebuild instead of reuse.** - `CheckpointStore.cancelAndRemoveAboveBlock` cancels and **removes** orphaned provers (replacing `markPrunedAboveBlock`); a re-add builds a fresh prover with a fresh `blockProofs`. - Removed the `pruned` flag / `markPruned` / `markCanonical` from `CheckpointProver`, and the `SlotWatcher` that only reaped lingering pruned provers. - `listCanonical` / `addOrUpdate` simplified — every prover in the store is now canonical. **Recover the epoch by rebuilding on re-add, rather than trying to prevent the terminal state.** A prune unwinds world-state as soon as it's detected — before the prover-node cancels the affected session — so a checkpoint prover mid-fork faults while still live and its session goes terminal `failed`. That race is inherent (the world-state rejection and the prover being cancelled are unordered), so instead of trying to reclassify it as a cancellation, we make the terminal state harmless: - `SessionManager.openFullSessionIfReady` no longer treats a terminal session as "already open" — it drops it and rebuilds. So re-adding an epoch's checkpoints reopens a fresh live session over fresh provers, via the checkpoint/prune triggers (ungated by the tick high-water mark). This reopen depends on the store no longer holding the poisoned prover: with the reuse machinery gone the rebuilt session gets a fresh prover with a healthy `blockProofs`, so it can actually prove rather than re-inheriting the rejected one. - `runSession` skips the failure-upload when the session's checkpoints no longer match canonical content — a prune-invalidated failure isn't a genuine proving failure, so it no longer emits a spurious post-mortem upload / alert. The only non-recovered case is "content pruned and never re-added" (chain stops publishing that epoch's checkpoints) — there's nothing to prove on this node then, so abandoning the epoch is correct. **Cancel the pruned prover's in-flight work promptly.** `CheckpointProver` now threads its abort signal into `PublicProcessor.process`, so a prune-driven cancel stops the current block's public execution immediately instead of running it to completion before the next `signal.aborted` check. This is independent of the recovery logic above — it just avoids wasting CPU re-executing a block whose checkpoint is being discarded. The in-flight block has not yet enqueued a broker proof, so aborting it loses no reusable work. ### Note: broker-level proof reuse is unaffected Removing the `CheckpointProver`-level reuse does not waste proving work. The expensive part — the SNARK proofs — is cached in the proving broker, content-addressed by job id, independent of world-state. `cancelJobsOnStop` defaults to `false`, so cancelling a prover does not abort its broker jobs; on an identical re-add the fresh prover regenerates identical inputs, and the broker dedups against the cached jobs/results (bounded by the broker's per-epoch cleanup). The reuse we deleted was witness-generation reuse, whose world-state forks cannot survive a prune anyway. ## Testing - `session-manager.test.ts`: pins the retry/recovery invariant — the periodic tick does not re-attempt a failed epoch (gated by `lastTickEpoch`), but a checkpoint re-add recovers it (ungated); a `failed` session's epoch is reopened once its checkpoints are re-added; a genuine proving failure still uploads a post-mortem; a failure coinciding with a canonical content change (prune) skips the upload. (Red/green verified for both the gate and the upload suppression.) - `checkpoint-store.test.ts` / `prover-node.test.ts`: pruned provers are cancelled and removed (not flagged); a re-add builds a fresh prover. - `checkpoint-prover.test.ts`: cancelling a prover mid-block aborts the signal its public execution is running under (red/green verified). - Full prover-node suite passes (166 tests); package typechecks clean; lint OK. - Updated `optimistic.parallel.test.ts` (the e2e that spawned this issue) to the cancelled+removed semantics. ## Notes - No operator/contract-dev-facing config or API changed, so no changelog entry. - The terminal `failed` state is not prevented, only recovered from: the world-state unwind and the prover cancellation are unordered, so the failure can't be reliably reclassified from the session side. Recovering on re-add is deterministic and needs no such race to be won. (cherry picked from commit c1e6229de4f2a5d7183100c8a20160d6c00c2b0d) --- .../proving/optimistic.parallel.test.ts | 20 +- yarn-project/prover-node/README.md | 113 ++++++----- .../prover-node/src/checkpoint-store.test.ts | 178 +++++------------- .../prover-node/src/checkpoint-store.ts | 148 ++++++--------- .../src/job/checkpoint-prover.test.ts | 74 +++++--- .../prover-node/src/job/checkpoint-prover.ts | 52 ++--- .../prover-node/src/job/epoch-session.test.ts | 3 - yarn-project/prover-node/src/metrics.ts | 2 +- .../prover-node/src/prover-node.test.ts | 14 +- yarn-project/prover-node/src/prover-node.ts | 7 +- .../prover-node/src/session-manager.test.ts | 162 ++++++++++++---- .../prover-node/src/session-manager.ts | 45 ++++- 12 files changed, 408 insertions(+), 410 deletions(-) diff --git a/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts b/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts index a3983288f7c1..2d3f2861f274 100644 --- a/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts @@ -338,22 +338,20 @@ describe('single-node/proving/optimistic', () => { timeout: 30, }); - // Verify the prover-node observes the prune. `markPruned()` fires reactively when - // the L2BlockStream emits the prune; the SlotWatcher then reaps the (now pruned) - // prover on its next tick (default 1s), so checking strictly for `isPruned()` would - // race against the reap. Identify the original by `(checkpointNumber, slot)` — - // checkpoint numbers refill sequentially after a reorg, so the replacement reuses - // the same number but lives at a different slot. Accept either state for the - // original: still in the store and pruned, or already reaped. + // Verify the prover-node observes the prune. The prune reactively cancels and removes the + // orphaned prover from the store when the L2BlockStream emits it, so the original should drop + // out of the store (or, if observed mid-race, be cancelled). Identify the original by + // `(checkpointNumber, slot)` — checkpoint numbers refill sequentially after a reorg, so the + // replacement reuses the same number but lives at a different slot. await retryUntil( () => { const prover = proverNode .getCheckpointStore() .listAll() .find(p => p.checkpoint.number === checkpointBeforeReorg && p.slotNumber === originalSlot); - return Promise.resolve(!prover || prover.isPruned()); + return Promise.resolve(!prover || prover.isCancelled()); }, - `prover marks original checkpoint ${checkpointBeforeReorg} (slot ${originalSlot}) as pruned (or reaps it)`, + `prover cancels and removes original checkpoint ${checkpointBeforeReorg} (slot ${originalSlot})`, 30, 0.2, ); @@ -383,7 +381,7 @@ describe('single-node/proving/optimistic', () => { proverNode .getCheckpointStore() .listAll() - .some(p => p.checkpoint.number === replacementCheckpoint && !p.isPruned()), + .some(p => p.checkpoint.number === replacementCheckpoint), ), `prover re-creates sub-tree for replacement checkpoint ${replacementCheckpoint}`, 30, @@ -875,7 +873,7 @@ describe('single-node/proving/optimistic', () => { // The session manager constructs a full session over the canonical content for the // anchored epoch when it completes, then proves it; the store retains the provers // until expiry. - const epochCheckpointsInStore = await proverNode.getCheckpointStore().listCanonicalForEpoch(epoch); + const epochCheckpointsInStore = await proverNode.getCheckpointStore().listForEpoch(epoch); const storedNumbers = new Set(epochCheckpointsInStore.map(p => p.checkpoint.number)); for (const n of preSpawnCheckpointNumbers) { expect(storedNumbers.has(n)).toBe(true); diff --git a/yarn-project/prover-node/README.md b/yarn-project/prover-node/README.md index d9521c69c340..abafeae777ee 100644 --- a/yarn-project/prover-node/README.md +++ b/yarn-project/prover-node/README.md @@ -35,7 +35,6 @@ flowchart TB SessionManager --> EpochTicker[(periodic tick)] SessionManager --> FullSessions[(fullSessions)] SessionManager --> PartialSessions[(partialSessions)] - CheckpointStore --> SlotWatcher FullSessions -.referenced checkpoints.-> CheckpointStore PartialSessions -.referenced checkpoints.-> CheckpointStore FullSessions --> TopTreeJob @@ -56,8 +55,9 @@ The prover-node splits responsibility between four classes: - **`CheckpointStore`** — a registry of `CheckpointProver` instances keyed by `(checkpointNumber, slot, archiveRoot)`. Each `CheckpointProver` runs its own sub-tree pipeline (tx gather → block processing → block-rollup proofs), starting eagerly the moment a - checkpoint is registered. The store is the single source of canonical-vs-pruned - checkpoint content that `EpochSession`s query when assembling their subsets. + checkpoint is registered. The store holds the canonical checkpoint content that + `EpochSession`s query when assembling their subsets; a prune cancels and removes the affected + provers, so every prover in the store is canonical. - **`SessionManager`** — owns every live `EpochSession`, the serial reconcile queue, the periodic tick, and all `EpochSession` lifecycle decisions. `ProverNode` calls into it via `onCheckpointAdded`, `onPrune`, and `startProof`. Every trigger it receives is @@ -81,43 +81,38 @@ A `CheckpointProver` is content-addressed by `(checkpoint.number, slot, archiveR where `archiveRoot` is the checkpoint's own archive root (its post-state). Keying on the post-state makes the identity precise: two checkpoints are "the same" iff they produce the same archive — so a reorg branch, or a replacement built on the same predecessor but -with different content, yields a different archive root and a distinct `CheckpointProver`, while an -identical re-add collapses to the same `CheckpointProver` and reuses its in-flight work. +with different content, yields a different archive root and a distinct `CheckpointProver`. A prune +cancels and removes a prover — its sub-tree work forks world-state per block and cannot survive the +prune — so a re-add, even of identical content, constructs a fresh `CheckpointProver` (block-rollup +jobs already completed in the proving broker are still reused, as they are content-addressed). ```mermaid stateDiagram-v2 [*] --> Created Created --> Proving: gather + execute Proving --> Proven: sub-tree resolves blockProofs - Proving --> Cancelled: cancel() + Proving --> Cancelled: cancel() (prune / shutdown / epoch-session error) + Proven --> Cancelled: cancel() (prune / reap) Proven --> Reaped: reapExpired(epoch) Cancelled --> [*] - - state "Pruned (side)" as Pruned - Proving --> Pruned: markPruned() - Pruned --> Proving: markCanonical() - Proven --> Pruned: markPruned() - Pruned --> Reaped: SlotWatcher (slot < syncedSlot) ``` -The **`Pruned`** state is a side flag, not a place in the main lifecycle: sub-tree -proving keeps running underneath, so a brief reorg that prunes and immediately -re-adds the same checkpoint avoids any re-proving. The flag only gates *eligibility* -to be included in an `EpochSession` — `EpochSession`s ask the store for *canonical* (non-pruned) -checkpoints when assembling their subsets. - -### Reaping rules - -- **Pruned**: the `SlotWatcher` (a `RunningPromise` polling - `l2BlockSource.getSyncedL2SlotNumber`) reaps a pruned `CheckpointProver` when the chain's - synced slot has moved past the `CheckpointProver`'s slot. Once the chain is past that slot, - a re-add with the same content is impossible. -- **Canonical**: `CheckpointStore.reapExpired(expiredEpoch)` drops any canonical - `CheckpointProver` whose epoch is at or below the supplied expired epoch. Once an epoch's - proof-submission window has closed, its proof can no longer be accepted on L1, - so the `CheckpointProver` is no longer needed. -- **Cancelled**: removed immediately by whichever path called `cancel()` (store - shutdown, prune past-slot, `EpochSession` error). +A prune **cancels and removes** the affected `CheckpointProver`s from the store: a prover's +sub-tree work forks world-state per block, and an L1 prune of a base block faults those reads, +so there is nothing to preserve across a reorg. A re-add — even of identical content — therefore +constructs a fresh `CheckpointProver`. `EpochSession`s ask the store for the checkpoints in a slot +range; every prover in the store is canonical. + +### Removal rules + +- **Pruned**: `CheckpointStore.cancelAndRemoveAboveBlock(target)` cancels and removes every + `CheckpointProver` whose last block is above the prune target, the moment a `chain-pruned` + event arrives. +- **Expired**: `CheckpointStore.reapExpired(expiredEpoch)` drops any `CheckpointProver` whose + epoch is at or below the supplied expired epoch. Once an epoch's proof-submission window has + closed, its proof can no longer be accepted on L1, so the `CheckpointProver` is no longer needed. +- **Shutdown**: `CheckpointStore.stop()` cancels every remaining prover and awaits its teardown + (including teardowns still in flight for provers already removed by a prune or reap). ### Eager tx gathering @@ -288,8 +283,8 @@ sequenceDiagram alt content key new CS->>CP: new CheckpointProver(args) CP->>CP: eager gather + sub-tree start - else content key matches - CS->>CP: markCanonical() + else content key already live + CS->>CP: reuse existing prover end PN->>SM: onCheckpointAdded(epoch) SM->>SM: queue reconcile({kind:'checkpoint', epoch}) @@ -306,9 +301,9 @@ sequenceDiagram participant CS as CheckpointStore participant SM as SessionManager - L2->>PN: chain-pruned{checkpoint} - PN->>CS: markPrunedAfter(checkpoint.number) - CS->>CS: flip every CheckpointProver above threshold to pruned (sub-tree keeps running) + L2->>PN: chain-pruned{block} + PN->>CS: cancelAndRemoveAboveBlock(prunedToBlock) + CS->>CS: cancel + remove every CheckpointProver whose last block is above the target PN->>SM: onPrune(affectedEpochs) SM->>SM: queue reconcile({kind:'prune', affectedEpochs}) SM->>SM: walk EpochSessions, cancel-and-recreate those with shifted content @@ -371,30 +366,28 @@ indexing) leave the mark in place and the next tick retries. ### checkpoint-added → prune → checkpoint-added (reorg resilience) -State: epoch N has checkpoints c1..c4 all canonical (slots s1..s4). `fullSessions[N]` -holds `EpochSession` **A** with spec `{kind:'full', N, fromSlot:s1, toSlot:s4}`, referencing -checkpoints `[c1, c2, c3, c4]`. +State: epoch N has checkpoints c1..c4 (slots s1..s4). `fullSessions[N]` holds `EpochSession` +**A** with spec `{kind:'full', N, fromSlot:s1, toSlot:s4}`, referencing `[c1, c2, c3, c4]`. -1. **chain-pruned arrives, target c3.** Store flips c4 to pruned. Reconcile fires: - for `EpochSession` A, canonical content for `(s1, s4)` is now `[c1, c2, c3]` (c4 pruned). - The frozen set `[c1, c2, c3, c4]` no longer matches → `A.cancel('canonical content - changed')`. Epoch N still complete on L1 → reconcile constructs `EpochSession` **B** with - the same spec `{full, N, s1, s4}` but checkpoints `[c1, c2, c3]`. +1. **chain-pruned arrives, target c3.** The store cancels and removes c4 (its last block is + above the prune target). Reconcile fires: `EpochSession` A's frozen set `[c1, c2, c3, c4]` + includes the now-cancelled c4, so it no longer matches the store's `[c1, c2, c3]` → + `A.cancel('canonical content changed')`. Epoch N still complete on L1 → reconcile constructs + `EpochSession` **B** with the same spec `{full, N, s1, s4}` but checkpoints `[c1, c2, c3]`. 2. **`EpochSession` B starts top-tree proving over [c1, c2, c3].** -3. **chain-checkpointed arrives, target c4_re (same content key as old c4).** The - store finds the existing `CheckpointProver` at `(c4.number, s4, c4.archive.root)` - and calls `markCanonical()`. The sub-tree work that never stopped is visible to - `EpochSession`s again. (A re-add with *different* content would have a different archive - root and so get a fresh `CheckpointProver` instead.) +3. **chain-checkpointed arrives, target c4_re (same content key as old c4).** The old c4 + prover was removed on the prune, so the store constructs a **fresh** `CheckpointProver` for + c4_re and starts its sub-tree work from scratch. (Its block-rollup jobs are content-addressed + in the proving broker, so any the old c4 already completed are reused rather than re-proved.) -4. **Reconcile fires.** `EpochSession` B's canonical content for `(s1, s4)` is now `[c1, c2, - c3, c4]`, doesn't match its frozen `[c1, c2, c3]` → `B.cancel(...)`. Construct - `EpochSession` **C** with same spec but checkpoints `[c1, c2, c3, c4]`. +4. **Reconcile fires.** `EpochSession` B's content for `(s1, s4)` is now `[c1, c2, c3, c4_re]`, + doesn't match its frozen `[c1, c2, c3]` → `B.cancel(...)`. Construct `EpochSession` **C** with + the same spec but checkpoints `[c1, c2, c3, c4_re]`. -5. **`EpochSession` C reuses the long-lived c1..c4 `CheckpointProver` instances.** Sub-tree - work may already be complete; only the top-tree is recomputed. The chonk cache +5. **`EpochSession` C reuses the long-lived c1..c3 `CheckpointProver` instances and the fresh + c4_re.** Only c4_re's witnesses are regenerated; the top-tree is recomputed. The chonk cache survived the reorg because no epoch in this range has expired yet. @@ -466,19 +459,19 @@ Keying by tx hash makes the cache survive any reorg up to finality; releasing on finality means we don't grow the cache indefinitely while still keeping every reorg-relevant proof. -### Why does the slot watcher only reap pruned `CheckpointProver`s? +### Why is a pruned `CheckpointProver` removed rather than kept for a possible re-add? -Canonical `CheckpointProver`s can't be reaped on a slot heuristic — they're still part of the -proven-chain story. Pruned `CheckpointProver`s, on the other hand, are only kept around in -case the chain re-adds the same content; once the synced slot has moved past, that -re-add is impossible, and the `CheckpointProver` can go. Finality is the right signal for -canonical reaping, because finality is the only state that rules out future reorgs. +A prover's sub-tree work forks world-state per block, and an L1 prune of a base block faults +those in-flight reads — so the work cannot survive the prune, and there is nothing to preserve +by keeping the prover around. Removing it on prune and rebuilding on re-add is therefore both +correct and simpler; the expensive block-rollup proofs are still reused when content re-appears, +because the proving broker is content-addressed independently of the prover's lifetime. ## Configuration | Env var | Description | |---|---| -| `PROVER_NODE_POLLING_INTERVAL_MS` | Polling interval for the L2BlockStream, the checkpoint-store slot watcher, and the SessionManager periodic tick. Default 1000 ms. | +| `PROVER_NODE_POLLING_INTERVAL_MS` | Polling interval for the L2BlockStream and the SessionManager periodic tick. Default 1000 ms. | | `PROVER_NODE_MAX_PENDING_JOBS` | Cap on the number of non-terminal `EpochSession`s (full + partial). When at limit, reconcile defers opening new full `EpochSession`s; explicit `startProof` calls throw. | | `PROVER_NODE_EPOCH_PROVING_DELAY_MS` | Optional sleep at the start of each `EpochSession`, before the TopTreeJob is constructed. Used in tests to give late events time to land. | | `TX_GATHERING_TIMEOUT_MS` | Per-block tx gather deadline used by each `CheckpointProver`. | diff --git a/yarn-project/prover-node/src/checkpoint-store.test.ts b/yarn-project/prover-node/src/checkpoint-store.test.ts index 0c9c98e8de4f..bd823eb83847 100644 --- a/yarn-project/prover-node/src/checkpoint-store.test.ts +++ b/yarn-project/prover-node/src/checkpoint-store.test.ts @@ -9,12 +9,12 @@ import { EmptyL1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { mock } from 'jest-mock-extended'; -import { type CheckpointProverFactory, CheckpointStore } from './checkpoint-store.js'; +import { CheckpointStore } from './checkpoint-store.js'; import type { CheckpointProver } from './job/checkpoint-prover.js'; describe('CheckpointStore', () => { - let store: TestCheckpointStore; - let blockSource: ReturnType>>; + let store: CheckpointStore; + let blockSource: ReturnType>>; /** Track stub provers we hand back from the factory. */ const stubs: StubProver[] = []; @@ -22,14 +22,13 @@ describe('CheckpointStore', () => { const l1Constants = { ...EmptyL1RollupConstants, epochDuration: 1 }; beforeEach(() => { - blockSource = mock>(); + blockSource = mock>(); blockSource.getL1Constants.mockResolvedValue(l1Constants); stubs.length = 0; - store = new TestCheckpointStore( + store = new CheckpointStore( blockSource, // The deps are not exercised — the factory below ignores them. {} as any, - { slotWatcherPollIntervalMs: 100 }, undefined, (args, _deps) => { const stub = makeStubProver(args.checkpoint, args.epochNumber); @@ -51,21 +50,35 @@ describe('CheckpointStore', () => { expect(stubs.length).toBe(1); }); - it('addOrUpdate is idempotent for the same content key (re-add after prune)', async () => { + it('addOrUpdate reuses the prover for a still-canonical duplicate registration', async () => { + // An at-least-once re-registration of a checkpoint that has NOT been pruned reuses the running + // prover rather than rebuilding its in-flight work. const cp = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1 }); const first = await store.addOrUpdate(cp, makeRegisterData()); - expect(first.isPruned()).toBe(false); - store.markPrunedAboveBlock(BlockNumber(0)); - expect(first.isPruned()).toBe(true); - - // Re-adding the identical checkpoint (same archive root) reuses the existing prover. const second = await store.addOrUpdate(cp, makeRegisterData()); expect(second).toBe(first); - expect(second.isPruned()).toBe(false); expect(stubs.length).toBe(1); }); + it('addOrUpdate rebuilds a fresh prover for a re-add after prune', async () => { + const cp = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1 }); + + const first = await store.addOrUpdate(cp, makeRegisterData()); + expect(first.isCancelled()).toBe(false); + store.cancelAndRemoveAboveBlock(BlockNumber(0)); + // The pruned prover is cancelled and dropped from the store. + expect(first.isCancelled()).toBe(true); + expect(store.getByCheckpoint(cp)).toBeUndefined(); + + // Re-adding the identical checkpoint (same archive root) constructs a fresh prover — the pruned + // one's forked world-state reads did not survive, so there is nothing to reuse. + const second = await store.addOrUpdate(cp, makeRegisterData()); + expect(second).not.toBe(first); + expect(second.isCancelled()).toBe(false); + expect(stubs.length).toBe(2); + }); + it('addOrUpdate refuses a conflicting canonical checkpoint at the same slot', async () => { // Two canonical checkpoints sharing a slot would be a parallel chain. The store rejects // the second; the caller must prune the first (via the chain-pruned event) before the @@ -76,20 +89,20 @@ describe('CheckpointStore', () => { const proverA = await store.addOrUpdate(a, makeRegisterData()); await expect(store.addOrUpdate(b, makeRegisterData())).rejects.toThrow( - /canonical checkpoint already occupies this slot/i, + /a different checkpoint already occupies this slot/i, ); - // After the predecessor is pruned, the replacement is accepted and keys to a distinct - // prover (different archive root → different content id). - store.markPrunedAboveBlock(BlockNumber(0)); - expect(proverA.isPruned()).toBe(true); + // After the predecessor is pruned (cancelled and removed), the replacement is accepted and keys + // to a distinct prover (different archive root → different content id). + store.cancelAndRemoveAboveBlock(BlockNumber(0)); + expect(proverA.isCancelled()).toBe(true); const proverB = await store.addOrUpdate(b, makeRegisterData()); expect(proverB).not.toBe(proverA); - expect(proverB.isPruned()).toBe(false); + expect(proverB.isCancelled()).toBe(false); expect(stubs.length).toBe(2); }); - it('markPrunedAboveBlock marks every prover holding a block above the target and returns them', async () => { + it('cancelAndRemoveAboveBlock cancels and removes every prover holding a block above the target and returns them', async () => { // Four single-block checkpoints occupying blocks 1..4 (one block each). Pruning to block 2 orphans the // checkpoints whose last block is above 2 — checkpoints 3 and 4 — and leaves 1 and 2 canonical. const cps = await timesAsync(4, i => @@ -102,23 +115,26 @@ describe('CheckpointStore', () => { for (const cp of cps) { await store.addOrUpdate(cp, makeRegisterData()); } - const affected = store.markPrunedAboveBlock(BlockNumber(2)); + const affected = store.cancelAndRemoveAboveBlock(BlockNumber(2)); expect(affected.map(p => p.checkpoint.number)).toEqual([3, 4]); - expect(store.listCanonical().map(p => p.checkpoint.number)).toEqual([1, 2]); + expect(affected.every(p => p.isCancelled())).toBe(true); + // The orphaned provers are gone from the store; only 1 and 2 remain. + expect(store.list().map(p => p.checkpoint.number)).toEqual([1, 2]); }); - it('markPrunedAboveBlock marks a checkpoint whose block range straddles the target (partially orphaned)', async () => { + it('cancelAndRemoveAboveBlock removes a checkpoint whose block range straddles the target (partially orphaned)', async () => { // A single checkpoint spanning blocks 5..8. A prune to block 6 lands mid-checkpoint: the checkpoint is partially - // orphaned (blocks 7, 8 are gone) and must be marked, since its last block (8) is above the target. + // orphaned (blocks 7, 8 are gone) and must be removed, since its last block (8) is above the target. const cp = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 4, startBlockNumber: 5 }); await store.addOrUpdate(cp, makeRegisterData()); - const affected = store.markPrunedAboveBlock(BlockNumber(6)); + const affected = store.cancelAndRemoveAboveBlock(BlockNumber(6)); expect(affected.map(p => p.checkpoint.number)).toEqual([1]); - expect(store.listCanonical()).toEqual([]); + expect(affected[0].isCancelled()).toBe(true); + expect(store.list()).toEqual([]); }); - it('reapExpired drops canonical provers whose epoch is ≤ expiredEpoch', async () => { + it('reapExpired drops provers whose epoch is ≤ expiredEpoch', async () => { // With epochDuration=1 each checkpoint's slot is also its epoch number. const cps = await Promise.all([ Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: SlotNumber(1) }), @@ -133,82 +149,15 @@ describe('CheckpointStore', () => { expect(remainingNumbers).toEqual([3]); }); - it('reapExpired leaves pruned provers in place', async () => { - const cp = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: SlotNumber(1) }); - await store.addOrUpdate(cp, makeRegisterData()); - store.markPrunedAboveBlock(BlockNumber(0)); - store.reapExpired(EpochNumber(10)); - expect(store.listAll().map(p => p.checkpoint.number)).toEqual([1]); - }); - - // ---------------- slot watcher ---------------- - - it('slot watcher reaps pruned provers whose slot is strictly before the synced slot', async () => { - const cp1 = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: SlotNumber(1) }); - const cp2 = await Checkpoint.random(CheckpointNumber(2), { numBlocks: 1, slotNumber: SlotNumber(2) }); - const cp3 = await Checkpoint.random(CheckpointNumber(3), { numBlocks: 1, slotNumber: SlotNumber(3) }); - for (const cp of [cp1, cp2, cp3]) { - await store.addOrUpdate(cp, makeRegisterData()); - } - // Prune everything above checkpoint 0 ⇒ all three flip to pruned. - store.markPrunedAboveBlock(BlockNumber(0)); - blockSource.getSyncedL2SlotNumber.mockResolvedValue(SlotNumber(3)); - - await store.triggerSlotWatcherTick(); - - // Slots 1 and 2 are < 3 and get reaped; slot 3 is not strictly less, so it stays. - expect(store.listAll().map(p => p.checkpoint.number)).toEqual([3]); - // Reaped stubs were cancelled by the watcher. - expect(stubs.find(s => s.checkpoint.number === 1)!.cancelled).toBe(true); - expect(stubs.find(s => s.checkpoint.number === 2)!.cancelled).toBe(true); - expect(stubs.find(s => s.checkpoint.number === 3)!.cancelled).toBe(false); - }); - - it('slot watcher leaves canonical provers in place even when their slot is past the synced slot', async () => { - // Canonical provers must survive — only pruned provers are eligible for reaping. - const cp = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: SlotNumber(1) }); - await store.addOrUpdate(cp, makeRegisterData()); - blockSource.getSyncedL2SlotNumber.mockResolvedValue(SlotNumber(10)); - - await store.triggerSlotWatcherTick(); - - expect(store.listAll().map(p => p.checkpoint.number)).toEqual([1]); - expect(stubs[0].cancelled).toBe(false); - }); - - it('slot watcher no-ops when getSyncedL2SlotNumber returns undefined', async () => { - const cp = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: SlotNumber(1) }); - await store.addOrUpdate(cp, makeRegisterData()); - store.markPrunedAboveBlock(BlockNumber(0)); - blockSource.getSyncedL2SlotNumber.mockResolvedValue(undefined); - - await store.triggerSlotWatcherTick(); - - // No synced slot yet ⇒ watcher doesn't know whether the chain has moved past, so it - // keeps the pruned prover around for a possible re-add. - expect(store.listAll().map(p => p.checkpoint.number)).toEqual([1]); - expect(stubs[0].cancelled).toBe(false); - }); - - it('slot watcher swallows getSyncedL2SlotNumber errors instead of crashing the tick', async () => { - const cp = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: SlotNumber(1) }); - await store.addOrUpdate(cp, makeRegisterData()); - store.markPrunedAboveBlock(BlockNumber(0)); - blockSource.getSyncedL2SlotNumber.mockRejectedValue(new Error('archiver unavailable')); - - await expect(store.triggerSlotWatcherTick()).resolves.toBeUndefined(); - expect(store.listAll().map(p => p.checkpoint.number)).toEqual([1]); - }); - - it('listCanonicalForEpoch returns only canonical provers in the epoch slot range', async () => { + it('listForEpoch returns only provers in the epoch slot range', async () => { // With epochDuration=1, each epoch's slot range is exactly [slot, slot]. const cp1 = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: SlotNumber(10) }); const cp2 = await Checkpoint.random(CheckpointNumber(2), { numBlocks: 1, slotNumber: SlotNumber(11) }); await store.addOrUpdate(cp1, makeRegisterData()); await store.addOrUpdate(cp2, makeRegisterData()); - const epoch10 = await store.listCanonicalForEpoch(EpochNumber(10)); - const epoch11 = await store.listCanonicalForEpoch(EpochNumber(11)); + const epoch10 = await store.listForEpoch(EpochNumber(10)); + const epoch11 = await store.listForEpoch(EpochNumber(11)); expect(epoch10.map(p => p.checkpoint.number)).toEqual([1]); expect(epoch11.map(p => p.checkpoint.number)).toEqual([2]); }); @@ -220,12 +169,8 @@ type StubProver = { checkpoint: Checkpoint; slotNumber: SlotNumber; epochNumber: EpochNumber; - pruned: boolean; cancelled: boolean; - isPruned(): boolean; isCancelled(): boolean; - markPruned(): void; - markCanonical(): void; cancel(opts?: { routine?: boolean }): void; whenDone(): Promise; }; @@ -237,20 +182,10 @@ function makeStubProver(checkpoint: Checkpoint, epochNumber: EpochNumber): StubP checkpoint, slotNumber: checkpoint.header.slotNumber, epochNumber, - pruned: false, cancelled: false, - isPruned() { - return this.pruned; - }, isCancelled() { return this.cancelled; }, - markPruned() { - this.pruned = true; - }, - markCanonical() { - this.pruned = false; - }, cancel() { this.cancelled = true; }, @@ -268,24 +203,3 @@ function makeRegisterData() { previousArchiveSiblingPath: makeTuple(ARCHIVE_HEIGHT, () => Fr.ZERO), }; } - -/** - * Subclass that exposes the protected `reapPrunedPastSlot` so tests can drive a single - * SlotWatcher tick directly — avoids spinning up the underlying `RunningPromise` and - * waiting on its polling interval. - */ -class TestCheckpointStore extends CheckpointStore { - constructor( - blockSource: ConstructorParameters[0], - proverDeps: ConstructorParameters[1], - options: ConstructorParameters[2], - bindings: ConstructorParameters[3], - factory: CheckpointProverFactory, - ) { - super(blockSource, proverDeps, options, bindings, factory); - } - - public triggerSlotWatcherTick(): Promise { - return this.reapPrunedPastSlot(); - } -} diff --git a/yarn-project/prover-node/src/checkpoint-store.ts b/yarn-project/prover-node/src/checkpoint-store.ts index 4006041c6950..c675c1fe1a86 100644 --- a/yarn-project/prover-node/src/checkpoint-store.ts +++ b/yarn-project/prover-node/src/checkpoint-store.ts @@ -1,6 +1,5 @@ import type { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log'; -import { RunningPromise } from '@aztec/foundation/promise'; import type { L2BlockSource } from '@aztec/stdlib/block'; import type { Checkpoint } from '@aztec/stdlib/checkpoint'; import { type L1RollupConstants, getEpochAtSlot, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; @@ -19,54 +18,64 @@ export type CheckpointProverFactory = (args: CheckpointProverArgs, deps: Checkpo * * The store survives every epoch / session boundary. A prover lives from its first * `addOrUpdate` call until either: - * - it has been pruned and the L2 chain has moved past its slot (no re-add possible), or + * - its checkpoint is pruned by an L1 reorg (`cancelAndRemoveAboveBlock`), or * - its epoch's proof-submission window has closed (`reapExpired`), so the proof could no * longer be accepted on L1 even if produced. * - * A re-add of a checkpoint that matches an existing prover's content key reuses the - * existing prover (and flips it back to canonical); the in-flight sub-tree work never - * stops, so a prune-then-re-add of the same content avoids re-proving entirely. + * A prover's sub-tree work forks world-state per block and does not survive a prune of a base + * block, so there is nothing to preserve across a reorg: a pruned prover is cancelled and + * dropped, and a re-add (even of identical content) constructs a fresh prover. */ export class CheckpointStore { private readonly provers = new Map(); - private readonly slotWatcher: RunningPromise; + /** + * Teardowns of provers already removed from `provers` (by prune or reap), awaited on `stop()`. + * Keyed by a monotonic id rather than the prover's content id: a prune-then-re-add can leave two + * teardowns for the same content id in flight at once, which a content-id key would clobber. + */ + private readonly pendingTeardowns = new Map>(); + private nextTeardownId = 0; private readonly log: Logger; constructor( - private readonly l2BlockSource: Pick, + private readonly l2BlockSource: Pick, private readonly proverDeps: Omit, - private readonly options: { slotWatcherPollIntervalMs: number }, bindings?: LoggerBindings, private readonly proverFactoryFn: CheckpointProverFactory = (args, deps) => new CheckpointProver(args, deps), ) { this.log = createLogger('prover-node:checkpoint-store', bindings); - this.slotWatcher = new RunningPromise( - () => this.reapPrunedPastSlot(), - this.log, - this.options.slotWatcherPollIntervalMs, - ); } public start(): Promise { - this.slotWatcher.start(); return Promise.resolve(); } public async stop(): Promise { - await this.slotWatcher.stop(); - // Cancel every live prover; await teardown. + // Cancel every live prover, then await both their teardown and any still in flight for provers + // already removed by a prune or reap. const provers = Array.from(this.provers.values()); this.provers.clear(); for (const prover of provers) { prover.cancel(); } - await Promise.allSettled(provers.map(p => p.whenDone())); + await Promise.allSettled([...provers.map(p => p.whenDone()), ...this.pendingTeardowns.values()]); + } + + /** + * Tracks the teardown of a prover just removed from the store so `stop()` can await it. The entry + * removes itself once teardown settles, so the map stays bounded by the number in flight. + */ + private trackTeardown(prover: CheckpointProver): void { + const id = this.nextTeardownId++; + const done = prover.whenDone(); + this.pendingTeardowns.set(id, done); + void done.finally(() => this.pendingTeardowns.delete(id)); } /** * Registers a checkpoint with the store. If a prover already exists for the - * `(number, slot, archive root)` content key, it is reused and marked canonical; - * otherwise a new prover is constructed. + * `(number, slot, archive root)` content key it is reused (an at-least-once re-registration of + * still-canonical content); otherwise a new prover is constructed. */ public async addOrUpdate(checkpoint: Checkpoint, data: RegisterCheckpointData): Promise { const l1Constants = await this.l2BlockSource.getL1Constants(); @@ -75,18 +84,18 @@ export class CheckpointStore { const existing = this.provers.get(id); if (existing) { - existing.markCanonical(); return existing; } - // At most one canonical checkpoint per slot. A different canonical checkpoint at the - // same slot means the caller forgot to prune the old chain before adding the replacement - // — surface it rather than silently creating a parallel canonical chain. + // At most one canonical checkpoint per slot. A different checkpoint at the same slot means the + // caller forgot to prune the old chain before adding the replacement — surface it rather than + // silently creating a parallel canonical chain. A pruned checkpoint has already been removed, + // so every prover still in the store is canonical. for (const prover of this.provers.values()) { - if (prover.slotNumber === checkpoint.header.slotNumber && !prover.isPruned()) { + if (prover.slotNumber === checkpoint.header.slotNumber) { throw new Error( `Cannot add checkpoint ${checkpoint.number} (archive ${checkpoint.archive.root}) at slot ${checkpoint.header.slotNumber}: ` + - `a different canonical checkpoint already occupies this slot. Prune it first.`, + `a different checkpoint already occupies this slot. Prune it first.`, ); } } @@ -97,20 +106,25 @@ export class CheckpointStore { } /** - * Marks every canonical prover that holds a block above the prune target as pruned. A checkpoint is orphaned by a - * prune to block `targetBlockNumber` iff its last block sits above the target — including a checkpoint whose range - * straddles the target (partially orphaned), which block-range marking catches without boundary ambiguity. Keying - * off the surviving block number (rather than a checkpoint number) is correct even when the source has already + * Cancels and removes every prover that holds a block above the prune target. A checkpoint is orphaned by a prune to + * block `targetBlockNumber` iff its last block sits above the target — including a checkpoint whose range straddles + * the target (partially orphaned), which block-range marking catches without boundary ambiguity. Keying off the + * surviving block number (rather than a checkpoint number) is correct even when the source has already * re-checkpointed past the divergence: the prune event reports the highest surviving block, which by construction * survives on the source, whereas the source's current checkpointed tip can sit above the prune target. - * Sub-tree work keeps running so a re-add of the same content can pick it up. Returns the affected provers. + * + * The prover's in-flight sub-tree work forks world-state per block and faults once its base block is pruned, so it + * cannot be reused; it is cancelled (aborting the fork reads) and dropped. A subsequent re-add constructs a fresh + * prover. Returns the removed provers. */ - public markPrunedAboveBlock(targetBlockNumber: BlockNumber): CheckpointProver[] { + public cancelAndRemoveAboveBlock(targetBlockNumber: BlockNumber): CheckpointProver[] { const affected: CheckpointProver[] = []; - for (const prover of this.provers.values()) { + for (const [id, prover] of Array.from(this.provers.entries())) { const lastBlockNumber = prover.checkpoint.blocks.at(-1)!.number; - if (lastBlockNumber > targetBlockNumber && !prover.isPruned()) { - prover.markPruned(); + if (lastBlockNumber > targetBlockNumber) { + prover.cancel(); + this.trackTeardown(prover); + this.provers.delete(id); affected.push(prover); } } @@ -118,20 +132,17 @@ export class CheckpointStore { } /** - * Drops canonical (non-pruned) provers whose epoch is at or below the supplied expired - * epoch. Once an epoch's proof-submission window has closed, its proof can no longer be - * accepted on L1, so the prover is no longer needed. + * Drops provers whose epoch is at or below the supplied expired epoch. Once an epoch's + * proof-submission window has closed, its proof can no longer be accepted on L1, so the + * prover is no longer needed. */ public reapExpired(expiredEpoch: EpochNumber): void { const reaped: { id: string; checkpointNumber: CheckpointNumber; epochNumber: EpochNumber }[] = []; for (const [id, prover] of Array.from(this.provers.entries())) { - if (prover.isPruned()) { - continue; - } if (prover.epochNumber <= expiredEpoch) { reaped.push({ id, checkpointNumber: prover.checkpoint.number, epochNumber: prover.epochNumber }); prover.cancel({ routine: true }); - void prover.whenDone(); + this.trackTeardown(prover); this.provers.delete(id); } } @@ -154,63 +165,28 @@ export class CheckpointStore { return this.provers.get(CheckpointProver.idFor(checkpoint)); } - /** Every prover currently in the store (canonical and pruned), in insertion order. */ + /** Every prover currently in the store, in insertion order. */ public listAll(): CheckpointProver[] { return Array.from(this.provers.values()); } - /** Canonical (non-pruned) provers in the store, sorted by checkpoint number. */ - public listCanonical(): CheckpointProver[] { - return Array.from(this.provers.values()) - .filter(p => !p.isPruned()) - .sort((a, b) => a.checkpoint.number - b.checkpoint.number); + /** Provers in the store, sorted by checkpoint number. */ + public list(): CheckpointProver[] { + return Array.from(this.provers.values()).sort((a, b) => a.checkpoint.number - b.checkpoint.number); } /** - * Canonical provers whose slot is in the supplied epoch's slot range, sorted by - * checkpoint number. + * Provers whose slot is in the supplied epoch's slot range, sorted by checkpoint number. */ - public async listCanonicalForEpoch(epoch: EpochNumber): Promise { + public async listForEpoch(epoch: EpochNumber): Promise { const l1Constants = await this.l2BlockSource.getL1Constants(); const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, l1Constants); - return this.listCanonicalInSlotRange(fromSlot, toSlot); + return this.listInSlotRange(fromSlot, toSlot); } - /** Canonical provers whose slot falls within `[fromSlot, toSlot]`, sorted by checkpoint number. */ - public listCanonicalInSlotRange(fromSlot: SlotNumber, toSlot: SlotNumber): CheckpointProver[] { - return this.listCanonical().filter(p => p.slotNumber >= fromSlot && p.slotNumber <= toSlot); - } - - /** - * SlotWatcher tick: reap pruned provers whose slot has passed the chain's synced - * slot. Once the chain has moved past, no re-add can revive the prover and its - * content key is unique enough that an actual re-add would create a new entry. - * - * Protected so unit tests can drive a single tick without spinning up the - * `RunningPromise` and waiting on its interval. - */ - protected async reapPrunedPastSlot(): Promise { - let syncedSlot: SlotNumber | undefined; - try { - syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber(); - } catch (err) { - this.log.debug(`SlotWatcher could not read synced slot`, { error: `${err}` }); - return; - } - if (syncedSlot === undefined) { - return; - } - for (const [id, prover] of Array.from(this.provers.entries())) { - if (prover.isPruned() && prover.slotNumber < syncedSlot) { - this.log.info(`Reaping pruned CheckpointProver ${id}: slot ${prover.slotNumber} < synced ${syncedSlot}`, { - checkpointNumber: prover.checkpoint.number, - slotNumber: prover.slotNumber, - }); - prover.cancel(); - void prover.whenDone(); - this.provers.delete(id); - } - } + /** Provers whose slot falls within `[fromSlot, toSlot]`, sorted by checkpoint number. */ + public listInSlotRange(fromSlot: SlotNumber, toSlot: SlotNumber): CheckpointProver[] { + return this.list().filter(p => p.slotNumber >= fromSlot && p.slotNumber <= toSlot); } } diff --git a/yarn-project/prover-node/src/job/checkpoint-prover.test.ts b/yarn-project/prover-node/src/job/checkpoint-prover.test.ts index 4a19f8a9c9a0..f516b5f33485 100644 --- a/yarn-project/prover-node/src/job/checkpoint-prover.test.ts +++ b/yarn-project/prover-node/src/job/checkpoint-prover.test.ts @@ -96,7 +96,6 @@ describe('CheckpointProver', () => { expect(prover.l1ToL2Messages).toEqual([]); expect(prover.isCancelled()).toBe(false); expect(prover.isCompleted()).toBe(false); - expect(prover.isPruned()).toBe(false); await cleanup(prover); }); @@ -108,31 +107,6 @@ describe('CheckpointProver', () => { }); }); - // ---------------- prune/canonical flag ---------------- - - describe('markPruned / markCanonical', () => { - it('markPruned flips isPruned() and is idempotent', async () => { - const prover = makeProver(); - expect(prover.isPruned()).toBe(false); - prover.markPruned(); - expect(prover.isPruned()).toBe(true); - prover.markPruned(); - expect(prover.isPruned()).toBe(true); - await cleanup(prover); - }); - - it('markCanonical restores isPruned() to false and is idempotent on a non-pruned prover', async () => { - const prover = makeProver(); - prover.markPruned(); - prover.markCanonical(); - expect(prover.isPruned()).toBe(false); - // No-op when already canonical. - prover.markCanonical(); - expect(prover.isPruned()).toBe(false); - await cleanup(prover); - }); - }); - // ---------------- cancellation ---------------- describe('cancel', () => { @@ -188,6 +162,54 @@ describe('CheckpointProver', () => { }); }); + // ---------------- cancellation short-circuits execution ---------------- + + describe('cancellation short-circuits execution', () => { + it('threads its abort signal into public execution so a cancel stops the current block', async () => { + // Drive execution into the block loop: gather resolves, the sub-tree and forks are stubbed, + // and public processing parks until its signal aborts. The captured signal must be the + // prover's own abort signal, so cancelling the prover interrupts the in-flight block rather + // than letting it run to completion before the next `signal.aborted` check. + txProvider.getTxsForBlock.mockReset(); + txProvider.getTxsForBlock.mockResolvedValue({ txs: [], missingTxs: [] }); + + const subTree = { + getSubTreeResult: () => new Promise(() => {}), + startNewBlock: () => Promise.resolve(), + startChonkVerifierCircuits: () => Promise.resolve(), + addTxs: () => Promise.resolve(), + setBlockCompleted: () => Promise.resolve(), + cancel: () => {}, + stop: () => Promise.resolve(), + }; + proverFactory.createCheckpointSubTreeOrchestrator.mockResolvedValue(subTree as any); + dbProvider.fork.mockResolvedValue({ + appendLeaves: () => Promise.resolve(), + close: () => Promise.resolve(), + } as any); + + const processReached = promiseWithResolvers(); + const publicProcessor = { + process: (_txs: unknown, limits: { signal?: AbortSignal }) => { + processReached.resolve(limits.signal!); + // Park until the signal aborts, mirroring PublicProcessor's per-tx abort check. + return new Promise(resolve => { + limits.signal?.addEventListener('abort', () => resolve([[], [], [], [], []])); + }); + }, + }; + publicProcessorFactory.create.mockReturnValue(publicProcessor as any); + + const prover = makeProver(); + const signal = await processReached.promise; + expect(signal.aborted).toBe(false); + + prover.cancel(); + expect(signal.aborted).toBe(true); + await expect(prover.whenDone()).resolves.toBeUndefined(); + }); + }); + // ---------------- gather failure ---------------- describe('gather failures', () => { diff --git a/yarn-project/prover-node/src/job/checkpoint-prover.ts b/yarn-project/prover-node/src/job/checkpoint-prover.ts index c1cb4628c8ae..2f39dd15d8df 100644 --- a/yarn-project/prover-node/src/job/checkpoint-prover.ts +++ b/yarn-project/prover-node/src/job/checkpoint-prover.ts @@ -59,16 +59,16 @@ export type CheckpointProverArgs = { * The store creates a CheckpointProver once per content-key. Keying on the checkpoint's * own archive root (its post-state) means two checkpoints are "the same" iff they * produce the same archive — so a reorg branch, or a replacement built on the same - * predecessor but with different content, keys to a distinct prover; an identical - * re-add keys to the same one and reuses its in-flight sub-tree work. + * predecessor but with different content, keys to a distinct prover. * * The prover eagerly starts its own tx gather and sub-tree work in the constructor, so * callers only need to call `whenBlockProofsReady()` to obtain the resulting block-rollup * proofs. * - * The prover survives prune/re-add cycles via `markPruned()` / `markCanonical()` — - * sub-tree proving keeps running underneath, so a checkpoint that is re-added after - * a brief reorg can be re-consumed with no re-proving. + * A CheckpointProver does not survive a prune: its sub-tree work forks world-state per + * block, and an L1 prune of a base block faults those reads. The store therefore cancels and + * discards a prover when its checkpoint is pruned, and a re-add (even of identical content) + * constructs a fresh prover. * * `cancel()` is idempotent. It aborts the gather + sub-tree, rejects the block-proof * promise, and exposes a `whenDone()` that resolves once teardown has unwound. @@ -92,8 +92,6 @@ export class CheckpointProver { private cancelled = false; private subTree?: CheckpointSubTreeOrchestrator; private completed = false; - /** Pruned in the canonical chain but not yet reaped — sub-tree continues running. */ - private pruned = false; private readonly abortController = new AbortController(); /** Tracks the eager gather+execute task so `cancel()` and `whenDone()` can await its unwind. */ @@ -146,37 +144,6 @@ export class CheckpointProver { return this.completed; } - public isPruned(): boolean { - return this.pruned; - } - - /** - * Mark this prover as no longer present in the canonical chain. Sub-tree proving keeps - * running so the work survives if the checkpoint is re-added. Idempotent. - */ - public markPruned(): void { - if (this.pruned) { - return; - } - this.pruned = true; - this.deps.log.info(`Marking CheckpointProver ${this.id} as pruned`, { - checkpointNumber: this.checkpoint.number, - slotNumber: this.slotNumber, - }); - } - - /** Mark this prover as part of the canonical chain again after a re-add. Idempotent. */ - public markCanonical(): void { - if (!this.pruned) { - return; - } - this.pruned = false; - this.deps.log.info(`Marking CheckpointProver ${this.id} as canonical`, { - checkpointNumber: this.checkpoint.number, - slotNumber: this.slotNumber, - }); - } - /** AbortSignal that fires on cancel — for callers that want to wire their own tasks. */ public getAbortSignal(): AbortSignal { return this.abortController.signal; @@ -432,7 +399,14 @@ export class CheckpointProver { } private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise { - const [processedTxs, failedTxs] = await publicProcessor.process(txs, { deadline: this.deps.deadline }); + // Pass the abort signal so a prune-driven cancel stops the current block's public execution + // immediately, rather than running it to completion before the next `signal.aborted` check. + // On abort `process` returns a partial result, the length check below throws, and + // `gatherAndExecute` swallows it via its `cancelled` guard. + const [processedTxs, failedTxs] = await publicProcessor.process(txs, { + deadline: this.deps.deadline, + signal: this.abortController.signal, + }); if (failedTxs.length) { const failedTxHashes = await Promise.all(failedTxs.map(({ tx }) => tx.getTxHash())); diff --git a/yarn-project/prover-node/src/job/epoch-session.test.ts b/yarn-project/prover-node/src/job/epoch-session.test.ts index 9a78535ddbfb..dea4db077bca 100644 --- a/yarn-project/prover-node/src/job/epoch-session.test.ts +++ b/yarn-project/prover-node/src/job/epoch-session.test.ts @@ -442,9 +442,6 @@ function makeStubProver(checkpoint: Checkpoint, opts: { blockProofsError?: Error whenBlockProofsReady: () => blockProofs, isCancelled: () => false, isCompleted: () => false, - isPruned: () => false, - markPruned: () => {}, - markCanonical: () => {}, cancel: () => {}, whenDone: () => Promise.resolve(), getAbortSignal: () => new AbortController().signal, diff --git a/yarn-project/prover-node/src/metrics.ts b/yarn-project/prover-node/src/metrics.ts index 699c75b807bb..78311379b590 100644 --- a/yarn-project/prover-node/src/metrics.ts +++ b/yarn-project/prover-node/src/metrics.ts @@ -96,7 +96,7 @@ export class ProverNodeJobMetrics { this.activeCheckpoints = this.meter.createObservableGauge(Metrics.PROVER_NODE_ACTIVE_CHECKPOINTS); this.activeEpochSessions = this.meter.createObservableGauge(Metrics.PROVER_NODE_ACTIVE_EPOCH_SESSIONS); this.stateObserver = (observer: BatchObservableResult) => { - observer.observe(this.activeCheckpoints!, checkpointStore.listCanonical().length); + observer.observe(this.activeCheckpoints!, checkpointStore.listAll().length); let full = 0; let partial = 0; for (const session of sessionManager.allSessions()) { diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index 74486a092ba0..f13b79002c4d 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -128,7 +128,7 @@ describe('ProverNode', () => { expect(proverNode.getLastProcessedCheckpoint()).toEqual(CheckpointNumber(100)); }); - it('dispatches chain-pruned through markPrunedAboveBlock and notifies the session manager only when affected', async () => { + it('dispatches chain-pruned through cancelAndRemoveAboveBlock and notifies the session manager only when affected', async () => { // No registered checkpoints — nothing to prune. await proverNode.handleBlockStreamEvent({ type: 'chain-pruned', @@ -139,7 +139,7 @@ describe('ProverNode', () => { expect(sessionManager.onPrune).not.toHaveBeenCalled(); // Register a checkpoint (cp 2 at block 2), then prune to block 1. The checkpoint's only block (2) is above the - // prune target, so it is marked pruned and its epoch (2) is reported. + // prune target, so it is cancelled and removed and its epoch (2) is reported. setupNotFullyProven(); await proverNode.handleBlockStreamEvent(mineCheckpoint(makeCheckpoint(2, 2, 2))); // The prune target (block 1) resolves to checkpoint 1, clamping the cursor to checkpoint 0. @@ -182,8 +182,9 @@ describe('ProverNode', () => { proven: makeTipId(2), }); - // The orphaned prover for checkpoint 3 is marked pruned, and the cursor was clamped below 3. - expect(originalProver.isPruned()).toBe(true); + // The orphaned prover for checkpoint 3 is cancelled and removed from the store, and the cursor was clamped below 3. + expect(originalProver.isCancelled()).toBe(true); + expect(proverNode.getCheckpointStore().getByCheckpoint(original)).toBeUndefined(); expect(proverNode.getLastProcessedCheckpoint()).toEqual(CheckpointNumber(1)); // The rebuilt checkpoint 3 (distinct archive root) is now served by the source. A fresh chain-checkpointed(3) @@ -197,7 +198,7 @@ describe('ProverNode', () => { }); it('throws on a prune whose target block data is missing, leaving provers and cursor untouched for retry', async () => { - // The cursor floor is resolved before any prover is marked, so a missing-data prune throws without side effects + // The cursor floor is resolved before any prover is removed, so a missing-data prune throws without side effects // and the next pass retries the whole handler (the tips cursor only advances on success). setupNotFullyProven(); await proverNode.handleBlockStreamEvent(mineCheckpoint(makeCheckpoint(3, 3, 3))); @@ -214,7 +215,8 @@ describe('ProverNode', () => { }), ).rejects.toThrow(/No block data found for prune target/); - expect(registeredProver.isPruned()).toBe(false); + expect(registeredProver.isCancelled()).toBe(false); + expect(proverNode.getCheckpointStore().listAll()).toContain(registeredProver); expect(sessionManager.onPrune).not.toHaveBeenCalled(); expect(proverNode.getLastProcessedCheckpoint()).toEqual(CheckpointNumber(3)); }); diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index 95c6e1a0de8a..6eb4c1b7fede 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -171,7 +171,6 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra txGatheringTimeoutMs: this.config.txGatheringTimeoutMs, deadline: undefined, }, - { slotWatcherPollIntervalMs: this.config.proverNodePollingIntervalMs }, this.log.getBindings(), ); } @@ -374,8 +373,8 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra private async handlePruneEvent(prunedToBlock: L2BlockId) { this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, { prunedToBlock }); - // Resolve the cursor floor BEFORE marking provers: markPrunedAboveBlock returns only newly-marked provers, so a - // throw after marking would leave a retry pass with nothing to act on. Resolving first means a throw leaves + // Resolve the cursor floor BEFORE removing provers: cancelAndRemoveAboveBlock returns only the provers it removed, + // so a throw after removing would leave a retry pass with nothing to act on. Resolving first means a throw leaves // everything untouched and the next pass retries the whole handler (the tips cursor only advances on success). let cursorFloor: CheckpointNumber; if (prunedToBlock.number === 0) { @@ -393,7 +392,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1)); } - const affected = this.checkpointStore.markPrunedAboveBlock(prunedToBlock.number); + const affected = this.checkpointStore.cancelAndRemoveAboveBlock(prunedToBlock.number); if (this.lastProcessedCheckpoint > cursorFloor) { this.lastProcessedCheckpoint = cursorFloor; diff --git a/yarn-project/prover-node/src/session-manager.test.ts b/yarn-project/prover-node/src/session-manager.test.ts index c907cf8b3151..606ebac446a0 100644 --- a/yarn-project/prover-node/src/session-manager.test.ts +++ b/yarn-project/prover-node/src/session-manager.test.ts @@ -31,6 +31,8 @@ describe('SessionManager', () => { /** Mirror of fullSessions/partialSessions whose entries are stubs we control. */ let stubs: StubSession[]; + /** Sessions the manager passed to the failure-upload callback. */ + let sessionFailures: EpochSession[]; /** Resolves whenever the manager constructs a stub session. */ let onConstruct: ((stub: StubSession) => void) | undefined; @@ -51,10 +53,11 @@ describe('SessionManager', () => { l2BlockSource.getL1Constants.mockResolvedValue(l1Constants); l2BlockSource.isEpochComplete.mockResolvedValue(false); l2BlockSource.getCheckpoints.mockResolvedValue([]); - store.listCanonicalInSlotRange.mockReturnValue([]); - store.listCanonicalForEpoch.mockResolvedValue([]); + store.listInSlotRange.mockReturnValue([]); + store.listForEpoch.mockResolvedValue([]); stubs = []; + sessionFailures = []; onConstruct = undefined; manager = new TestSessionManager( @@ -67,6 +70,10 @@ describe('SessionManager', () => { metrics, dateProvider: new DateProvider(), config: { maxPendingJobs: 0, tickIntervalMs: 60_000, finalizationDelayMs: undefined }, + onSessionFailed: session => { + sessionFailures.push(session); + return Promise.resolve(); + }, }, (spec, provers) => { const stub = makeStubSession(spec, provers); @@ -114,7 +121,7 @@ describe('SessionManager', () => { l2BlockSource.isEpochComplete.mockResolvedValue(true); l2BlockSource.getCheckpoints.mockResolvedValue([archiverCp(1, 6), archiverCp(2, 7)]); // Store only has checkpoint 1. - store.listCanonicalInSlotRange.mockReturnValue([proverForCheckpoint(1, 6)]); + store.listInSlotRange.mockReturnValue([proverForCheckpoint(1, 6)]); await manager.onCheckpointAdded(epoch); expect(stubs.length).toBe(0); expect(manager.getFullSession(epoch)).toBeUndefined(); @@ -126,7 +133,7 @@ describe('SessionManager', () => { const provers = [proverForCheckpoint(1, 6), proverForCheckpoint(2, 7)]; l2BlockSource.isEpochComplete.mockResolvedValue(true); l2BlockSource.getCheckpoints.mockResolvedValue([archiverCp(1, 6), archiverCp(2, 7)]); - store.listCanonicalInSlotRange.mockReturnValue(provers); + store.listInSlotRange.mockReturnValue(provers); await manager.onCheckpointAdded(epoch); @@ -176,7 +183,7 @@ describe('SessionManager', () => { ), ), ); - store.listCanonicalInSlotRange.mockImplementation((fromSlot: SlotNumber) => { + store.listInSlotRange.mockImplementation((fromSlot: SlotNumber) => { if (Number(fromSlot) === 6) { return epoch3Provers; } @@ -203,7 +210,7 @@ describe('SessionManager', () => { const provers = [proverForCheckpoint(1, 6)]; l2BlockSource.isEpochComplete.mockResolvedValue(true); l2BlockSource.getCheckpoints.mockResolvedValue([archiverCp(1, 6)]); - store.listCanonicalInSlotRange.mockReturnValue(provers); + store.listInSlotRange.mockReturnValue(provers); await manager.onTick(); expect(manager.getFullSession(EpochNumber(3))).toBeDefined(); @@ -229,7 +236,7 @@ describe('SessionManager', () => { const provers = [proverForCheckpoint(1, 6)]; l2BlockSource.isEpochComplete.mockResolvedValue(true); l2BlockSource.getCheckpoints.mockResolvedValue([archiverCp(1, 6)]); - store.listCanonicalInSlotRange.mockReturnValue(provers); + store.listInSlotRange.mockReturnValue(provers); await manager.onTick(); expect(stubs.length).toBe(1); @@ -246,7 +253,7 @@ describe('SessionManager', () => { const provers = [proverForCheckpoint(1, 6)]; l2BlockSource.isEpochComplete.mockResolvedValue(true); l2BlockSource.getCheckpoints.mockResolvedValue([archiverCp(1, 6)]); - store.listCanonicalInSlotRange.mockReturnValue(provers); + store.listInSlotRange.mockReturnValue(provers); await manager.onTick(); expect(stubs.length).toBe(1); @@ -260,6 +267,37 @@ describe('SessionManager', () => { expect(stubs.length).toBe(1); }); + it('onTick does not re-attempt a failed epoch, but a checkpoint re-add recovers it', async () => { + // Pins the invariant: a genuinely-failed epoch is never re-attempted by the periodic tick + // (gated by lastTickEpoch), yet the SAME epoch is recovered when a chain event fires, because + // checkpoint/prune triggers reach openFullSessionIfReady ungated. This is how a failed proof is + // not resubmitted on a loop while a pruned-then-re-added epoch still gets reproven. + mockNextUnprovenSlot(2, 6); // proven tip block 2 → first unproven slot 6 → epoch 3 + const provers = [proverForCheckpoint(1, 6)]; + l2BlockSource.isEpochComplete.mockResolvedValue(true); + l2BlockSource.getCheckpoints.mockResolvedValue([archiverCp(1, 6)]); + store.listInSlotRange.mockReturnValue(provers); + + // Tick opens the session; it fails. lastTickEpoch is now pinned at epoch 3. + await manager.onTick(); + expect(stubs.length).toBe(1); + stubs[0].terminate('failed'); + await flushSessionCompletion(); + + // Further ticks must NOT re-attempt epoch 3 — the gate holds, no new session is constructed. + await manager.onTick(); + await manager.onTick(); + expect(stubs.length).toBe(1); + expect(manager.getFullSession(EpochNumber(3))).toBeUndefined(); + + // A checkpoint (re-)added to the epoch reaches openFullSessionIfReady ungated → fresh session. + await manager.onCheckpointAdded(EpochNumber(3)); + const recreated = manager.getFullSession(EpochNumber(3)) as unknown as StubSession | undefined; + expect(recreated).toBeDefined(); + expect(recreated!.isTerminal()).toBe(false); + expect(stubs.length).toBe(2); + }); + it('onTick keeps retrying the same epoch while a transient blocker prevents opening', async () => { // The archiver is still indexing — getCheckpoints returns a checkpoint we don't yet // have in the store. openFullSessionIfReady should bail without creating a session, @@ -267,7 +305,7 @@ describe('SessionManager', () => { mockNextUnprovenSlot(2, 6); l2BlockSource.isEpochComplete.mockResolvedValue(true); l2BlockSource.getCheckpoints.mockResolvedValue([archiverCp(1, 6)]); - store.listCanonicalInSlotRange.mockReturnValue([]); // store hasn't indexed it yet + store.listInSlotRange.mockReturnValue([]); // store hasn't indexed it yet await manager.onTick(); expect(stubs.length).toBe(0); // no session created @@ -275,7 +313,7 @@ describe('SessionManager', () => { expect(stubs.length).toBe(0); // still no session — the tick keeps trying // Archiver catches up; the next tick succeeds. - store.listCanonicalInSlotRange.mockReturnValue([proverForCheckpoint(1, 6)]); + store.listInSlotRange.mockReturnValue([proverForCheckpoint(1, 6)]); await manager.onTick(); expect(stubs.length).toBe(1); expect(manager.getFullSession(EpochNumber(3))).toBe(stubs[0] as unknown as EpochSession); @@ -291,7 +329,7 @@ describe('SessionManager', () => { const original = stubs[0]; // Now the store reports only the first prover. - store.listCanonicalInSlotRange.mockReturnValue([initial[0]]); + store.listInSlotRange.mockReturnValue([initial[0]]); await manager.onPrune([epoch]); expect(original.cancelled).toBe(true); @@ -313,7 +351,7 @@ describe('SessionManager', () => { await openCanonicalFullSession(epoch, [proverForCheckpoint(1, 6)]); const original = stubs[0]; - store.listCanonicalInSlotRange.mockReturnValue([]); + store.listInSlotRange.mockReturnValue([]); await manager.onPrune([epoch]); expect(original.cancelled).toBe(true); @@ -333,7 +371,7 @@ describe('SessionManager', () => { const original = stubs[0]; // Reorg removes every checkpoint of the epoch → session dropped, not recreated. - store.listCanonicalInSlotRange.mockReturnValue([]); + store.listInSlotRange.mockReturnValue([]); await manager.onPrune([epoch]); expect(original.cancelled).toBe(true); expect(original.state).toBe('cancelled'); @@ -351,6 +389,56 @@ describe('SessionManager', () => { expect(stubs.length).toBe(2); }); + it('reopens an epoch whose session failed once its checkpoints are re-added', async () => { + // A prune can fault a checkpoint mid-proof before it is reconciled, driving the session to + // `failed`. That terminal state must not strand the epoch — a re-add of its checkpoints reopens a + // fresh, live session (via the checkpoint trigger, ungated by the tick high-water mark). + const epoch = EpochNumber(3); + const prover = proverForCheckpoint(1, 6); + await openCanonicalFullSession(epoch, [prover]); + const original = stubs[0]; + + original.terminate('failed'); + await flushSessionCompletion(); + expect(original.state).toBe('failed'); + + await openCanonicalFullSession(epoch, [prover]); + const recreated = manager.getFullSession(epoch) as unknown as StubSession | undefined; + expect(recreated).toBeDefined(); + expect(recreated).not.toBe(original); + expect(recreated!.uuid).not.toBe(original.uuid); + expect(recreated!.isTerminal()).toBe(false); + expect(stubs.length).toBe(2); + }); + + it('uploads a post-mortem for a genuine proving failure (canonical content unchanged)', async () => { + const epoch = EpochNumber(3); + const prover = proverForCheckpoint(1, 6); + await openCanonicalFullSession(epoch, [prover]); + const session = manager.getFullSession(epoch); + // The session's checkpoints still match canonical → the failure was genuine, not a prune. + store.listInSlotRange.mockReturnValue([prover]); + + stubs[0].terminate('failed'); + await flushSessionCompletion(); + + expect(sessionFailures).toEqual([session]); + }); + + it('skips the failure upload when the failure coincides with a canonical content change', async () => { + const epoch = EpochNumber(3); + const prover = proverForCheckpoint(1, 6); + await openCanonicalFullSession(epoch, [prover]); + // A prune removed the checkpoint from the store, so the session's content no longer matches + // canonical — the failure was caused by the content changing under it, not a proving fault. + store.listInSlotRange.mockReturnValue([]); + + stubs[0].terminate('failed'); + await flushSessionCompletion(); + + expect(sessionFailures).toEqual([]); + }); + it('drops terminal sessions on the next reconcile', async () => { const epoch = EpochNumber(3); await openCanonicalFullSession(epoch, [proverForCheckpoint(1, 6)]); @@ -372,8 +460,8 @@ describe('SessionManager', () => { it('cancels and recreates a partial session whose canonical content changed', async () => { const epoch = EpochNumber(7); const initial = [proverForCheckpoint(1, 14)]; - store.listCanonicalForEpoch.mockResolvedValue(initial); - store.listCanonicalInSlotRange.mockReturnValue(initial); + store.listForEpoch.mockResolvedValue(initial); + store.listInSlotRange.mockReturnValue(initial); const stubPromise = awaitNextStub(); const startPromise = manager.startProof(epoch); @@ -383,7 +471,7 @@ describe('SessionManager', () => { // The store now reports a different prover at the same slot. const swapped = [proverForCheckpoint(2, 14)]; - store.listCanonicalInSlotRange.mockReturnValue(swapped); + store.listInSlotRange.mockReturnValue(swapped); const recreatePromise = awaitNextStub(); await manager.onTick(); @@ -406,14 +494,14 @@ describe('SessionManager', () => { it('drops a partial session and does not recreate when canonical content goes empty', async () => { const epoch = EpochNumber(7); const initial = [proverForCheckpoint(1, 14)]; - store.listCanonicalForEpoch.mockResolvedValue(initial); - store.listCanonicalInSlotRange.mockReturnValue(initial); + store.listForEpoch.mockResolvedValue(initial); + store.listInSlotRange.mockReturnValue(initial); const stubPromise = awaitNextStub(); const startPromise = manager.startProof(epoch); const original = await stubPromise; - store.listCanonicalInSlotRange.mockReturnValue([]); + store.listInSlotRange.mockReturnValue([]); await manager.onTick(); expect(original.cancelled).toBe(true); @@ -427,8 +515,8 @@ describe('SessionManager', () => { it('drops terminal partial sessions on the next reconcile', async () => { const epoch = EpochNumber(7); const canonical = [proverForCheckpoint(1, 14)]; - store.listCanonicalForEpoch.mockResolvedValue(canonical); - store.listCanonicalInSlotRange.mockReturnValue(canonical); + store.listForEpoch.mockResolvedValue(canonical); + store.listInSlotRange.mockReturnValue(canonical); const stubPromise = awaitNextStub(); const startPromise = manager.startProof(epoch); @@ -457,8 +545,8 @@ describe('SessionManager', () => { terminalFull.terminate('failed'); expect(terminalFull.isTerminal()).toBe(true); - store.listCanonicalForEpoch.mockResolvedValue(canonical); - store.listCanonicalInSlotRange.mockReturnValue(canonical); + store.listForEpoch.mockResolvedValue(canonical); + store.listInSlotRange.mockReturnValue(canonical); const stubPromise = awaitNextStub(); const startPromise = manager.startProof(epoch); @@ -476,8 +564,8 @@ describe('SessionManager', () => { it('startProof ignores a terminal partial session and constructs a fresh one', async () => { const epoch = EpochNumber(7); const canonical = [proverForCheckpoint(1, 14)]; - store.listCanonicalForEpoch.mockResolvedValue(canonical); - store.listCanonicalInSlotRange.mockReturnValue(canonical); + store.listForEpoch.mockResolvedValue(canonical); + store.listInSlotRange.mockReturnValue(canonical); // Open a partial, settle it terminally, then call startProof again. const firstPromise = awaitNextStub(); @@ -508,8 +596,8 @@ describe('SessionManager', () => { const epoch = EpochNumber(7); // Epoch 7 covers slots [14, 15]. Single canonical prover at slot 14. const canonical = [proverForCheckpoint(1, 14)]; - store.listCanonicalForEpoch.mockResolvedValue(canonical); - store.listCanonicalInSlotRange.mockReturnValue(canonical); + store.listForEpoch.mockResolvedValue(canonical); + store.listInSlotRange.mockReturnValue(canonical); // Arm the construction trigger before calling startProof — no need to sleep waiting // for reconcile to land. @@ -535,7 +623,7 @@ describe('SessionManager', () => { }); it('startProof throws when the epoch has no canonical content', async () => { - store.listCanonicalForEpoch.mockResolvedValue([]); + store.listForEpoch.mockResolvedValue([]); await expect(manager.startProof(EpochNumber(7))).rejects.toThrow(/No blocks found/); }); @@ -543,7 +631,7 @@ describe('SessionManager', () => { const epoch = EpochNumber(7); // proverForCheckpoint builds a checkpoint whose single block number equals the checkpoint // number (1 here). A proven tip at or beyond that block means the epoch is already proven. - store.listCanonicalForEpoch.mockResolvedValue([proverForCheckpoint(1, 14)]); + store.listForEpoch.mockResolvedValue([proverForCheckpoint(1, 14)]); l2BlockSource.getBlockNumber.mockResolvedValue(BlockNumber(1)); await expect(manager.startProof(epoch)).rejects.toThrow(/already proven/i); @@ -559,7 +647,7 @@ describe('SessionManager', () => { expect(stubs.length).toBe(1); const fullSession = stubs[0]; - store.listCanonicalForEpoch.mockResolvedValue(provers); + store.listForEpoch.mockResolvedValue(provers); const doneId = await manager.startProof(epoch); fullSession.terminate('completed'); @@ -571,8 +659,8 @@ describe('SessionManager', () => { it('startProof dedupes against an existing partial session with the same spec', async () => { const epoch = EpochNumber(7); const canonical = [proverForCheckpoint(1, 14)]; - store.listCanonicalForEpoch.mockResolvedValue(canonical); - store.listCanonicalInSlotRange.mockReturnValue(canonical); + store.listForEpoch.mockResolvedValue(canonical); + store.listInSlotRange.mockReturnValue(canonical); const firstId = await manager.startProof(epoch); expect(stubs).toHaveLength(1); @@ -605,7 +693,7 @@ describe('SessionManager', () => { ), ), ); - store.listCanonicalInSlotRange.mockImplementation((fromSlot: SlotNumber) => { + store.listInSlotRange.mockImplementation((fromSlot: SlotNumber) => { if (Number(fromSlot) === 6) { return epoch3Provers; } @@ -668,7 +756,7 @@ describe('SessionManager', () => { async function openCanonicalFullSession(epoch: EpochNumber, provers: CheckpointProver[]): Promise { l2BlockSource.isEpochComplete.mockResolvedValueOnce(true); l2BlockSource.getCheckpoints.mockResolvedValueOnce(provers.map(p => ({ checkpoint: p.checkpoint }) as any)); - store.listCanonicalInSlotRange.mockReturnValueOnce(provers); + store.listInSlotRange.mockReturnValueOnce(provers); await manager.onCheckpointAdded(epoch); } @@ -678,6 +766,11 @@ describe('SessionManager', () => { * after an action that schedules a reconcile — the manager itself signals "session ready" * via the factory call. */ + /** Lets `runSession`'s post-`start()` continuation (failure upload, logging) run after a stub terminates. */ + function flushSessionCompletion(): Promise { + return new Promise(resolve => setImmediate(resolve)); + } + function awaitNextStub(): Promise { const { promise, resolve } = promiseWithResolvers(); onConstruct = stub => { @@ -831,7 +924,6 @@ function proverForCheckpoint(number: number, slot: number): CheckpointProver { id: CheckpointProver.idFor(checkpoint), checkpoint, slotNumber: SlotNumber(slot), - isPruned: () => false, isCancelled: () => false, } as unknown as CheckpointProver; } diff --git a/yarn-project/prover-node/src/session-manager.ts b/yarn-project/prover-node/src/session-manager.ts index 3430815c205f..1162958f53c2 100644 --- a/yarn-project/prover-node/src/session-manager.ts +++ b/yarn-project/prover-node/src/session-manager.ts @@ -185,7 +185,7 @@ export class SessionManager { * slot. Dedupes against any existing session covering the same range, returning its id. */ public async startProof(epoch: EpochNumber): Promise { - const canonical = await this.deps.checkpointStore.listCanonicalForEpoch(epoch); + const canonical = await this.deps.checkpointStore.listForEpoch(epoch); if (canonical.length === 0) { throw new EmptyEpochError(epoch); } @@ -268,7 +268,7 @@ export class SessionManager { this.fullSessions.delete(key); continue; } - const canonical = this.canonicalCheckpointsForSpec(session.getSpec()); + const canonical = this.checkpointsForSpec(session.getSpec()); if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) { this.fireAndForgetCancel(session, 'canonical content changed'); this.fullSessions.delete(key); @@ -284,7 +284,7 @@ export class SessionManager { this.partialSessions.delete(key); continue; } - const canonical = this.canonicalCheckpointsForSpec(session.getSpec()); + const canonical = this.checkpointsForSpec(session.getSpec()); if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) { this.fireAndForgetCancel(session, 'canonical content changed'); this.partialSessions.delete(key); @@ -298,6 +298,8 @@ export class SessionManager { } private async openFullSessionIfReady(epoch: EpochNumber): Promise { + // `recreateInvalidSessions` runs at the top of every reconcile and deletes terminal sessions + // before this is called, so a session present here is live and already covers the epoch. if (this.fullSessions.has(epoch)) { return; } @@ -314,7 +316,7 @@ export class SessionManager { return; } const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, l1Constants); - const canonical = this.deps.checkpointStore.listCanonicalInSlotRange(fromSlot, toSlot); + const canonical = this.deps.checkpointStore.listInSlotRange(fromSlot, toSlot); if (!this.archiverFullyCovered(archiverCps, canonical)) { this.log.debug(`Skipping full-session open for epoch ${epoch}: archiver checkpoints not all in store`, { archiverCount: archiverCps.length, @@ -329,7 +331,7 @@ export class SessionManager { } private openPartialSession(spec: SessionSpec): void { - const canonical = this.deps.checkpointStore.listCanonicalInSlotRange(spec.fromSlot, spec.toSlot); + const canonical = this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot); if (canonical.length === 0) { return; } @@ -403,6 +405,17 @@ export class SessionManager { const state = await session.start(); this.log.info(`Session ${session.getId()} exited with state ${state}`); if (state === 'failed' && this.deps.onSessionFailed) { + // Best-effort suppression of the spurious post-mortem upload a prune produces: if the session's + // checkpoints no longer match the store's current set, the failure was caused by the content + // changing under it, not a genuine proving fault, so skip the upload. This is inherently racy — + // the store lags the world-state unwind, so a fault observed before the prune is reconciled here + // still uploads. The epoch is recovered regardless by recreating the session on re-add. + if (!this.checkpointsMatch(session.getCheckpoints(), this.checkpointsForSpec(session.getSpec()))) { + this.log.info(`Skipping failure upload for session ${session.getId()}: canonical content changed`, { + ...session.getSpec(), + }); + return; + } try { await this.deps.onSessionFailed(session); } catch (err) { @@ -447,6 +460,24 @@ export class SessionManager { return live >= max; } + /** + * Maps a reconcile trigger to the epochs whose full session should be (re)opened. + * + * This is where the "don't retry a genuinely-failed epoch, but do recover a pruned one" invariant + * lives — enforced by which triggers are gated by `lastTickEpoch`: + * + * - The periodic `tick` IS gated: once a tick has opened a session for an epoch, `lastTickEpoch` + * advances to it and later ticks skip it (`epoch <= lastTickEpoch`). So a failed attempt is never + * resubmitted on a loop by the tick. + * - `checkpoint` and `prune` are deliberately NOT gated. They only fire when the epoch's canonical + * content actually changes — a checkpoint arrives, or a reorg prunes/replaces one — which is + * exactly when re-attempting is correct. + * + * A genuine proving failure produces no content change, hence no checkpoint/prune event, so only + * the gated tick could reopen it — and it won't. A prune + re-add fires ungated events, so the + * epoch is reopened through this path (and `openFullSessionIfReady` rebuilds over the fresh + * provers). See the "onTick does not retry ... but recovers ... re-added" test. + */ private async epochsForTrigger(trigger: ReconcileTrigger): Promise { switch (trigger.kind) { case 'checkpoint': @@ -481,8 +512,8 @@ export class SessionManager { return getEpochAtSlot(header.getSlot(), await this.getL1Constants()); } - private canonicalCheckpointsForSpec(spec: SessionSpec): CheckpointProver[] { - return this.deps.checkpointStore.listCanonicalInSlotRange(spec.fromSlot, spec.toSlot); + private checkpointsForSpec(spec: SessionSpec): CheckpointProver[] { + return this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot); } private fireAndForgetCancel(session: EpochSession, reason: string): void { From 6565898546a4a30ddc5ceb4c49bffcd6fba139ea Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Mon, 13 Jul 2026 07:24:30 -0400 Subject: [PATCH 20/35] fix(prover-node): do not abort in-flight proving jobs on a clean shutdown (#24579) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v5 version of #24562 (based on `merge-train/spartan-v5`). Stacked on #24578 — review/merge that first. Note: this is a fresh implementation, not a port — the v5 prover-node architecture differs from `next` (it uses `EpochSession` / `TopTreeJob` rather than `EpochProvingJob` / `ProvingOrchestrator`), which is the structure actually running on testnet. ## Problem On a clean shutdown, `SessionManager.stop()` cancels every live session with reason `'prover-node stopping'`. That flows `EpochSession.cancel` → `TopTreeJob.cancel()` → `topTree.cancel({ abortJobs: true })` — the `abortJobs: true` was **hardcoded**, so a deploy/restart aborted the in-flight top-tree broker jobs. Aborting on a clean restart is wasteful: the proofs are still valid, agents are mid-flight, and a restarted node re-orchestrating the same epoch produces the same deterministically-hashed job ids, so it re-proves from scratch. ## Fix Thread an `abortJobs` decision from the cancel reason down to the broker: - `SessionManager.stop()` cancels with `{ abortJobs: false }` — a clean shutdown preserves the jobs. - `EpochSession.cancel(reason, { abortJobs })` forwards it to `TopTreeJob.cancel(abortJobs)` → `topTree.cancel({ abortJobs })`. - All other cancels (reorg / supersede / deadline) keep the default `abortJobs: true`, since their inputs are stale. Composes with #24578: a clean redeploy mid-epoch neither cancels the in-flight jobs nor (if one ever were aborted) poisons them, so the epoch keeps proving across the restart. ## Tests - `session-manager.test.ts`: `stop()` cancels every session with `abortJobs: false`. - `epoch-session.test.ts`: a normal cancel forwards `abortJobs: true` to the top-tree orchestrator; a clean-shutdown cancel forwards `abortJobs: false`. Note: not run locally in this session — the prover-node suite needs a full noir/wasm bootstrap that wasn't available here — so relied on CI. --- *Created by [claudebox](https://claudebox.work/v2/sessions/6fa5e242ed9ceae7) · group: `slackbot`* --------- Co-authored-by: Phil Windle (cherry picked from commit c44e74e5aecafbf5184f6088573cad9cf8e8c63f) --- yarn-project/end-to-end/src/fixtures/setup.ts | 3 +- .../end-to-end/src/single-node/README.md | 2 +- .../proving/prover_restart.parallel.test.ts | 424 ++++++++++++++++++ .../single-node/single_node_test_context.ts | 10 +- .../prover-node/src/job/epoch-session.test.ts | 20 + .../prover-node/src/job/epoch-session.ts | 6 +- .../prover-node/src/job/top-tree-job.ts | 8 +- .../prover-node/src/session-manager.test.ts | 10 +- .../prover-node/src/session-manager.ts | 4 +- 9 files changed, 473 insertions(+), 14 deletions(-) create mode 100644 yarn-project/end-to-end/src/single-node/proving/prover_restart.parallel.test.ts diff --git a/yarn-project/end-to-end/src/fixtures/setup.ts b/yarn-project/end-to-end/src/fixtures/setup.ts index 8281c8104cce..797fbac1c83a 100644 --- a/yarn-project/end-to-end/src/fixtures/setup.ts +++ b/yarn-project/end-to-end/src/fixtures/setup.ts @@ -37,7 +37,7 @@ import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree'; import type { P2PClientDeps } from '@aztec/p2p'; import { MockGossipSubNetwork, getMockPubSubP2PServiceFactory } from '@aztec/p2p/test-helpers'; import { protocolContractsHash } from '@aztec/protocol-contracts'; -import type { ProverNodeConfig } from '@aztec/prover-node'; +import type { ProverNodeConfig, ProverNodeDeps } from '@aztec/prover-node'; import { type PXEConfig, type PXECreationOptions, getPXEConfig } from '@aztec/pxe/server'; import type { SequencerClient } from '@aztec/sequencer-client'; import { AuthRegistryArtifact, getStandardAuthRegistry } from '@aztec/standard-contracts/auth-registry'; @@ -872,6 +872,7 @@ export function createAndSyncProverNode( telemetry?: TelemetryClient; dateProvider: DateProvider; p2pClientDeps?: P2PClientDeps; + proverNodeDeps?: Partial; }, options: { genesis?: GenesisData; dontStart?: boolean }, ): Promise<{ proverNode: AztecNodeService }> { diff --git a/yarn-project/end-to-end/src/single-node/README.md b/yarn-project/end-to-end/src/single-node/README.md index 16b4f7b7b212..2ef57fd67fc3 100644 --- a/yarn-project/end-to-end/src/single-node/README.md +++ b/yarn-project/end-to-end/src/single-node/README.md @@ -58,7 +58,7 @@ top-level `it`; CI splits each `it` into its own job. | `cross-chain/` | L1↔L2 messaging on a single node, on the `CrossChainMessagingTest` harness (`cross_chain_messaging_test.ts`), which extends `SingleNodeTestContext` and owns a `CrossChainTestHarness` plus the L1 inbox/outbox handles (it auto-proves via an `EpochTestSettler` when no prover node runs). The shared L1→L2 message helpers live in `message_test_helpers.ts` (`createL1ToL2MessageHelpers`, whose injected `markAsProven` lets each suite plug in its own proving policy). `l1_to_l2` (L1→L2 message readiness and duplicate messages, over private and public scope), `l1_to_l2_inbox_drift` (inbox checkpoint drift after a rollup reorg, over private and public scope), `l2_to_l1` (L2→L1 message inclusion across single/multi-message txs and multi-block checkpoints, subtree-root balancing, and a reorg-and-remine case), `token_bridge` (private and public L1→L2 deposits and L2→L1 withdrawals via the TokenBridge — including mint-on-behalf — plus the withdrawal/claim failure cases, merged from the former three `token_bridge_*` files). | | `bot/` | Transaction bot implementations. `bot` (transfer bot, AMM bot, and cross-chain bot; exercises fee-juice portal deposits, L2→L1 messages, and bot contract reuse). | | `sync/` | World-state sync stress and snapshot sync. `synching` builds fixture block data (env-gated, slow) and replays it for sync benchmarks and prune/reorg scenarios (only the outer `it.each` runs in CI); `snapshot_sync` exercises the node snapshot upload/download path, syncing fresh nodes from one or multiple snapshot URLs, including fallback from a corrupted snapshot. | -| `proving/` | Epoch and proof lifecycle. `default_node` (basic proving/node coverage that shares one context-default node across three suites: consecutive epochs prove and finalized blocks are purged from world state beyond the checkpoint-history window, a proof is submitted even with no txs, and the node returns initial genesis-block data — the former `world_state_pruning`, `empty_blocks`, and `node_block_api` files), `long_proving_time` (a prover delay spanning multiple epochs), `multi_proof` (multiple prover nodes prove one epoch), `optimistic.parallel` (checkpoint-driven proving across the happy path and several mid-epoch / last-slot / during-proving reorg cases), `proof_fails.parallel` (proof not accepted after epoch end; proving aborts when the next epoch ends), `cross_chain_public_message` (an epoch with a public tx that consumes an L1→L2 message in the block it lands, guarding against a sequencer/prover state-root mismatch), `upload_failed_proof` (a failed proving job's state is uploaded and re-run on a fresh instance). | +| `proving/` | Epoch and proof lifecycle. `default_node` (basic proving/node coverage that shares one context-default node across three suites: consecutive epochs prove and finalized blocks are purged from world state beyond the checkpoint-history window, a proof is submitted even with no txs, and the node returns initial genesis-block data — the former `world_state_pruning`, `empty_blocks`, and `node_block_api` files), `long_proving_time` (a prover delay spanning multiple epochs), `multi_proof` (multiple prover nodes prove one epoch), `optimistic.parallel` (checkpoint-driven proving across the happy path and several mid-epoch / last-slot / during-proving reorg cases), `proof_fails.parallel` (proof not accepted after epoch end; proving aborts when the next epoch ends), `cross_chain_public_message` (an epoch with a public tx that consumes an L1→L2 message in the block it lands, guarding against a sequencer/prover state-root mismatch), `upload_failed_proof` (a failed proving job's state is uploaded and re-run on a fresh instance), `prover_restart.parallel` (a clean prover-node shutdown leaves its in-flight broker jobs untouched, and a restart against the same shared broker resumes proving from them rather than aborting and re-proving — one case gates the top tree and withholds the checkpoint-root proofs so those top-tree jobs are the in-flight, revived ones, the other starves agents from the start so real transaction base-rollup proofs are the in-flight, revived jobs). | | `prover/` | Real-proof exercises on the `FullProverTest` harness (real Barretenberg when `FAKE_PROOFS=0`, fake otherwise), split by where the proving runs so CI can size their containers independently: `client/client` (client-side proof generation and `verifyProof` for private and public transfers, no on-chain submission) and `server/full` (the end-to-end pipeline: client proves, node builds blocks, prover node generates epoch proofs, L1 verifies them). | | `partial-proofs/` | Manually driven partial-proof submission. `single_root` (the prover node's `startProof` path on a single root) and `multi_root` (three partial-proof roots are staged and messages consume against any covering root, exercising the multi-root Outbox semantics). | | `l1-reorgs/` | Behavior under L1 reorgs, split by what reorgs. `blocks.parallel` (prune L2 blocks when a reorg drops a proof, hold when a replacement proof lands in the window, restore blocks when a proof reappears, prune pending-chain blocks, and see new blocks added by a reorg) and `messages.parallel` (L1→L2 messages updated by a reorg, and a missed message inserted by one). `setup.ts` holds the shared `FAST_REORG_TIMING` profile and delayer wiring. | diff --git a/yarn-project/end-to-end/src/single-node/proving/prover_restart.parallel.test.ts b/yarn-project/end-to-end/src/single-node/proving/prover_restart.parallel.test.ts new file mode 100644 index 000000000000..319ffee9e94f --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/proving/prover_restart.parallel.test.ts @@ -0,0 +1,424 @@ +import type { Logger } from '@aztec/aztec.js/log'; +import type { RollupContract } from '@aztec/ethereum/contracts'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { promiseWithResolvers } from '@aztec/foundation/promise'; +import { retryUntil } from '@aztec/foundation/retry'; +import { type ProvingBroker, createAndStartProvingBroker } from '@aztec/prover-client/broker'; +import type { TestProverNode } from '@aztec/prover-node/test'; +import { EthAddress } from '@aztec/stdlib/block'; +import { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers'; +import type { + AztecNode, + GetProvingJobResponse, + ProofUri, + ProvingJob, + ProvingJobBroker, + ProvingJobFilter, + ProvingJobId, + ProvingJobStatus, +} from '@aztec/stdlib/interfaces/server'; +import { ProvingRequestType } from '@aztec/stdlib/proofs'; +import { getTelemetryClient } from '@aztec/telemetry-client'; + +import { expect, jest } from '@jest/globals'; + +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { proveInteraction } from '../../test-wallet/utils.js'; +import { NO_REORG_SUBMISSION_EPOCHS, setupWithProver } from '../setup.js'; +import { FAST_REORG_TIMING, SingleNodeTestContext } from '../single_node_test_context.js'; + +jest.setTimeout(1000 * 60 * 20); + +const ALL_PROVING_TYPES = Object.values(ProvingRequestType).filter( + (t): t is ProvingRequestType => typeof t === 'number', +); + +const isParity = (type: ProvingRequestType) => + type === ProvingRequestType.PARITY_BASE || type === ProvingRequestType.PARITY_ROOT; + +const isTxBaseRollup = (type: ProvingRequestType) => + type === ProvingRequestType.PRIVATE_TX_BASE_ROLLUP || type === ProvingRequestType.PUBLIC_TX_BASE_ROLLUP; + +// Checkpoint-root proofs are enqueued by the top-tree orchestrator (unlike parity, which is a +// block/sub-tree proof). A single-block checkpoint uses the SINGLE_BLOCK variant. Both are the +// "non-parity top-tree" jobs we withhold from agents to catch them in flight. +const CHECKPOINT_ROOT_TYPES = [ + ProvingRequestType.CHECKPOINT_ROOT_ROLLUP, + ProvingRequestType.CHECKPOINT_ROOT_SINGLE_BLOCK_ROLLUP, +]; +const isCheckpointRoot = (type: ProvingRequestType) => CHECKPOINT_ROOT_TYPES.includes(type); + +/** + * A thin proxy over a real {@link ProvingBroker} that (a) records every job enqueue and cancel so the + * test can assert what the prover node did, and (b) can starve agents on demand so proving jobs pile + * up unproven at the broker. It deliberately has no `stop()` method, so the prover node's shutdown + * (`tryStop` on its job producer) does not stop the underlying broker — the broker outlives the node + * and carries the in-flight jobs across a restart, exactly like a production external broker service. + */ +class RecordingBrokerProxy implements ProvingJobBroker { + /** When true, agents get no work (getProvingJob returns undefined) and the piggybacked next job on a report is withheld. */ + public agentsPaused = false; + /** + * Job types withheld from agents: their dependencies still prove, but these jobs are never handed + * out, so they pile up `in-queue` — letting the test catch a specific layer (e.g. checkpoint-root + * proofs) in flight without stalling everything else. + */ + public captureTypes = new Set(); + /** Every job passed to enqueueProvingJob, with the status the broker returned at the start of that call. */ + public readonly enqueues: { job: ProvingJob; returnedStatus: ProvingJobStatus['status'] }[] = []; + /** Every job id passed to cancelProvingJob. On a clean shutdown this must stay empty. */ + public readonly cancels: ProvingJobId[] = []; + + constructor(private readonly inner: ProvingBroker) {} + + // Narrow the agent's allow-list to exclude the captured types, so the broker never hands those jobs + // to an agent (via getProvingJob or the piggybacked next-job on a report). + private withCapture(filter?: ProvingJobFilter): ProvingJobFilter | undefined { + if (this.captureTypes.size === 0) { + return filter; + } + const base = filter?.allowList?.length ? filter.allowList : ALL_PROVING_TYPES; + return { allowList: base.filter(t => !this.captureTypes.has(t)) }; + } + + async enqueueProvingJob(job: ProvingJob): Promise { + const status = await this.inner.enqueueProvingJob(job); + this.enqueues.push({ job, returnedStatus: status.status }); + return status; + } + + cancelProvingJob(id: ProvingJobId): Promise { + this.cancels.push(id); + return this.inner.cancelProvingJob(id); + } + + getProvingJobStatus(id: ProvingJobId): Promise { + return this.inner.getProvingJobStatus(id); + } + + getCompletedJobs(ids: ProvingJobId[]): Promise { + return this.inner.getCompletedJobs(ids); + } + + getProvingJob(filter?: ProvingJobFilter): Promise { + return this.agentsPaused ? Promise.resolve(undefined) : this.inner.getProvingJob(this.withCapture(filter)); + } + + async reportProvingJobSuccess( + id: ProvingJobId, + result: ProofUri, + filter?: ProvingJobFilter, + ): Promise { + // Always settle the reported job (its result must be cached for reuse), but while paused withhold + // the next job the broker hands back so no new work starts. + const next = await this.inner.reportProvingJobSuccess(id, result, this.withCapture(filter)); + return this.agentsPaused ? undefined : next; + } + + async reportProvingJobError( + id: ProvingJobId, + err: string, + retry?: boolean, + filter?: ProvingJobFilter, + ): Promise { + const next = await this.inner.reportProvingJobError(id, err, retry, this.withCapture(filter)); + return this.agentsPaused ? undefined : next; + } + + reportProvingJobProgress( + id: ProvingJobId, + startedAt: number, + filter?: ProvingJobFilter, + ): Promise { + return this.inner.reportProvingJobProgress(id, startedAt, filter); + } +} + +/** + * E2E tests for a clean prover-node restart with jobs in flight at a shared broker. + * + * A clean prover-node shutdown must NOT abort its in-flight broker jobs — sessions are cancelled with + * `abortJobs: false` and checkpoint sub-trees with `cancelJobsOnStop: false` — so a restarted node + * re-requests them and the broker returns the existing jobs rather than a fresh `not-found`. The + * end-state alone ("epoch proven after restart") does not prove this: aborted jobs are revivable at + * the broker, so proving would recover even if the jobs were wrongly aborted. The discriminating + * assertions are that the shutdown issued zero `cancelProvingJob` calls and that the in-flight jobs + * remain `in-queue` (not `aborted`) across it. + * + * The broker is a test-owned object shared across both prover-node incarnations (production's external + * broker topology), so the in-flight jobs survive the restart in memory. + * + * Two scenarios, covering different layers of the proof tree in flight at shutdown: + * - top-tree checkpoint-root proofs: gate the top tree, then withhold only the checkpoint-root jobs + * from agents so everything below proves and the checkpoint roots sit in flight. These are enqueued + * by the top-tree orchestrator, so this exercises the `abortJobs: false` top-tree cancel path. + * - sub-tree transaction proofs: starve agents from the start so the transaction base rollups sit in + * flight, preserved by the checkpoint sub-tree's `cancelJobsOnStop: false`. + */ +describe('single-node/proving/prover_restart', () => { + let test: SingleNodeTestContext; + let context: EndToEndContext; + let node: AztecNode; + let rollup: RollupContract; + let logger: Logger; + let L2_SLOT_DURATION_IN_S: number; + + let realBroker: ProvingBroker; + let broker: RecordingBrokerProxy; + + const PINNED_PROVER_ID = EthAddress.fromNumber(1); + + // A shared in-memory broker (no dataDirectory) that outlives each prover node. + const createSharedBroker = async () => { + realBroker = await createAndStartProvingBroker( + { ...context.config, dataDirectory: undefined }, + getTelemetryClient(), + ); + broker = new RecordingBrokerProxy(realBroker); + }; + + afterEach(async () => { + await test?.teardown(); + await realBroker?.stop(); + }); + + describe('in-flight checkpoint-root proofs', () => { + beforeEach(async () => { + test = await setupWithProver({ + ...FAST_REORG_TIMING, + // We own prover-node creation so we can inject the shared broker and drive the stop/restart. + startProverNode: false, + maxSpeedUpAttempts: 0, + cancelTxOnTimeout: false, + minTxsPerBlock: 0, + aztecProofSubmissionEpochs: NO_REORG_SUBMISSION_EPOCHS, + // Recover promptly from any job left in-progress by a withheld piggyback, and never fail a job + // for retrying while agents are paused. + proverBrokerJobTimeoutMs: 2_000, + proverBrokerPollIntervalMs: 500, + proverBrokerJobMaxRetries: 1_000, + }); + ({ rollup, logger, context } = test); + ({ L2_SLOT_DURATION_IN_S } = test); + node = context.aztecNode; + await createSharedBroker(); + }); + + it('preserves and revives in-flight checkpoint-root proofs across a clean prover-node restart', async () => { + // Prover node #1, wired to the shared broker and a pinned prover id (so a restart re-requests the + // exact same content-addressed job ids). + const node1 = await test.createProverNode({ proverNodeDeps: { broker }, proverId: PINNED_PROVER_ID }); + const proverNode1 = node1.getProverNode() as TestProverNode; + + // Gate top-tree proving of the first full session so it blocks until we release it, giving us a + // deterministic point at which the top tree begins enqueuing its jobs. + const { promise: provingGate, resolve: releaseProvingGate } = promiseWithResolvers(); + let gatedSession: ReturnType[number] | undefined; + proverNode1.setSessionHooks({ + beforeTopTreeProve: async () => { + // EpochSession flips to `awaiting-root` before awaiting this hook, so the gating session is + // the live full session in that state. First one to arrive is the one we gate; later ones + // sail through once the gate is released. + const session = proverNode1.sessionManager + .allSessions() + .find(s => s.getKind() === 'full' && s.getState() === 'awaiting-root'); + if (!session) { + return; + } + gatedSession ??= session; + logger.warn(`Top-tree proving gated for epoch ${session.getEpochNumber()} — waiting for test to release`); + await provingGate; + logger.warn(`Proving gate released for epoch ${session.getEpochNumber()}`); + }, + }); + + // Wait for a full session to complete its checkpoints and block at the top-tree gate. + const inFlightSession = await retryUntil( + () => Promise.resolve(gatedSession), + 'full session blocks at the top-tree proving gate', + L2_SLOT_DURATION_IN_S * 12, + 0.5, + ); + const gatedEpoch = inFlightSession.getEpochNumber(); + const checkpoints = inFlightSession.getCheckpoints(); + const epochEndCheckpoint = checkpoints[checkpoints.length - 1].checkpoint.number; + logger.info(`Epoch ${gatedEpoch} is gated at top-tree proving (ends at checkpoint ${epochEndCheckpoint})`); + + // Stop block production so the system goes quiescent (no new sub-tree work), then withhold the + // checkpoint-root proofs from agents. Everything below (tx/block/parity sub-tree work) still + // proves, so the top tree reaches and enqueues its checkpoint-root jobs — which then sit + // `in-queue`, leaving the top tree mid-proof at shutdown. This is the fix's real target: a + // top-tree job (not a leaf parity job) in flight. + await context.aztecNodeAdmin!.setConfig({ skipPublishingCheckpointsPercent: 100 }); + broker.captureTypes = new Set(CHECKPOINT_ROOT_TYPES); + releaseProvingGate(); + + // Wait until at least one checkpoint-root job for the gated epoch is pending at the broker. + const pendingCheckpointRootIds = await retryUntil( + async () => { + const candidates = broker.enqueues + .filter(e => e.job.epochNumber === gatedEpoch && isCheckpointRoot(e.job.type)) + .map(e => e.job.id); + const inQueue: ProvingJobId[] = []; + for (const id of new Set(candidates)) { + if ((await broker.getProvingJobStatus(id)).status === 'in-queue') { + inQueue.push(id); + } + } + return inQueue.length > 0 ? inQueue : undefined; + }, + 'checkpoint-root jobs are enqueued and pending at the broker', + 60, + 0.5, + ); + logger.info( + `${pendingCheckpointRootIds.length} checkpoint-root job(s) pending at the broker for epoch ${gatedEpoch}`, + ); + + // Clean shutdown: this drives SessionManager.stop() -> session.cancel({ abortJobs: false }) -> + // TopTreeJob.cancel(false), which must NOT abort the in-flight checkpoint-root jobs. + const cancelsBeforeStop = broker.cancels.length; + await node1.stop(); + + // The fix under test: a clean shutdown leaves the in-flight jobs untouched. Pre-fix, the shutdown + // would abort them — a non-empty `cancels` and an `aborted` status. + expect(broker.cancels.length).toBe(cancelsBeforeStop); + for (const id of pendingCheckpointRootIds) { + expect((await broker.getProvingJobStatus(id)).status).toBe('in-queue'); + } + logger.info('Clean shutdown preserved the in-flight checkpoint-root jobs at the broker'); + + // Restart: a fresh prover node against the SAME broker and the same prover id. It resyncs from L1 + // and re-drives the epoch; re-requesting the preserved jobs reuses them rather than re-proving. + const enqueuesBeforeRestart = broker.enqueues.length; + const node2 = await test.createProverNode({ proverNodeDeps: { broker }, proverId: PINNED_PROVER_ID }); + expect((node2.getProverNode() as TestProverNode).getProverId()).toEqual(PINNED_PROVER_ID); + broker.captureTypes = new Set(); + + // Proving resumes automatically and the epoch lands on L1. + await test.waitUntilProvenCheckpointNumber(epochEndCheckpoint, 240); + expect(await rollup.getProvenCheckpointNumber()).toBeGreaterThanOrEqual(epochEndCheckpoint); + logger.info(`Epoch ${gatedEpoch} proven on L1 up to checkpoint ${epochEndCheckpoint} after restart`); + + // Reuse: the checkpoint-root proofs that were pending before the stop are re-requested and reused + // (returned with an existing status, never a fresh `not-found`, and never `aborted`). These are + // top-tree, non-parity proofs. + const reRequests = broker.enqueues.slice(enqueuesBeforeRestart); + const revivedCheckpointRoots = reRequests.filter( + e => pendingCheckpointRootIds.includes(e.job.id) && e.returnedStatus !== 'not-found', + ); + expect(revivedCheckpointRoots.length).toBeGreaterThan(0); + expect(revivedCheckpointRoots.every(e => isCheckpointRoot(e.job.type) && !isParity(e.job.type))).toBe(true); + expect(reRequests.every(e => e.returnedStatus !== 'aborted')).toBe(true); + }); + }); + + describe('in-flight transaction proofs', () => { + beforeEach(async () => { + test = await setupWithProver({ + ...FAST_REORG_TIMING, + numberOfAccounts: 1, + startProverNode: false, + maxSpeedUpAttempts: 0, + cancelTxOnTimeout: false, + minTxsPerBlock: 0, + aztecProofSubmissionEpochs: NO_REORG_SUBMISSION_EPOCHS, + // Keep the frozen epoch's jobs around while block production advances during the freeze. + proverBrokerMaxEpochsToKeepResultsFor: 10, + }); + ({ rollup, logger, context } = test); + node = context.aztecNode; + await createSharedBroker(); + }); + + it('preserves and revives in-flight checkpoint prover jobs across a clean prover-node restart', async () => { + // Starve agents before the prover node exists: it will enqueue jobs at the broker but prove none. + broker.agentsPaused = true; + + const node1 = await test.createProverNode({ proverNodeDeps: { broker }, proverId: PINNED_PROVER_ID }); + expect((node1.getProverNode() as TestProverNode).getProverId()).toEqual(PINNED_PROVER_ID); + + // Anchor on a fresh epoch, then land a couple of real txs in it so the broker gets actual + // transaction base-rollup jobs (not just the empty-block parity/root jobs). + await test.waitUntilNextEpochStarts(); + const contract = await test.registerTestContract(context.wallet); + const receipts = []; + for (let i = 0; i < 2; i++) { + const provenTx = await proveInteraction(context.wallet, contract.methods.emit_nullifier(new Fr(i + 1)), { + from: context.accounts[0], + }); + receipts.push(await provenTx.send()); + } + const txCheckpoint = (await node.getBlock(receipts[receipts.length - 1].blockNumber!))!.checkpointNumber; + const txCp = await retryUntil( + async () => (await node.getCheckpoints(txCheckpoint, 1))[0], + `archiver indexes checkpoint ${txCheckpoint}`, + 30, + 0.2, + ); + const txEpoch = getEpochAtSlot(txCp.header.slotNumber, test.constants); + logger.info(`Landed 2 txs in checkpoint ${txCheckpoint} (epoch ${txEpoch})`); + + // Wait until the transaction base-rollup jobs are enqueued and pending (in-queue) at the broker — + // these are the non-parity proofs we want to see revived. Agents are starved, so they cannot + // complete. + const pendingTxProofIds = await retryUntil( + async () => { + const ids = [...new Set(broker.enqueues.filter(e => isTxBaseRollup(e.job.type)).map(e => e.job.id))]; + const inQueue: ProvingJobId[] = []; + for (const id of ids) { + if ((await broker.getProvingJobStatus(id)).status === 'in-queue') { + inQueue.push(id); + } + } + return inQueue.length > 0 ? inQueue : undefined; + }, + 'transaction base-rollup jobs are enqueued and pending at the broker', + 60, + 0.5, + ); + logger.info( + `${pendingTxProofIds.length} transaction base-rollup job(s) pending at the broker for epoch ${txEpoch}`, + ); + + // Complete the epoch on L1 (so a restart can prove it), then stop producing so no further epochs + // pile up jobs while the prover is down. + await test.warpToEpochStart(txEpoch + 1); + await context.aztecNodeAdmin!.setConfig({ skipPublishingCheckpointsPercent: 100 }); + const epochEndCheckpoint = (await test.monitor.run(true)).checkpointNumber; + expect(epochEndCheckpoint).toBeGreaterThanOrEqual(txCheckpoint); + + // Clean shutdown: this must not abort any broker jobs. + const cancelsBeforeStop = broker.cancels.length; + const enqueuesBeforeRestart = broker.enqueues.length; + await node1.stop(); + + expect(broker.cancels.length).toBe(cancelsBeforeStop); + for (const id of pendingTxProofIds) { + expect((await broker.getProvingJobStatus(id)).status).toBe('in-queue'); + } + logger.info('Clean shutdown preserved the in-flight transaction proofs at the broker'); + + // Restart against the same broker with the same prover id and let agents run. + const node2 = await test.createProverNode({ proverNodeDeps: { broker }, proverId: PINNED_PROVER_ID }); + expect((node2.getProverNode() as TestProverNode).getProverId()).toEqual(PINNED_PROVER_ID); + broker.agentsPaused = false; + + // Proving resumes automatically and the epoch lands on L1. + await test.waitUntilProvenCheckpointNumber(epochEndCheckpoint, 240); + expect(await rollup.getProvenCheckpointNumber()).toBeGreaterThanOrEqual(epochEndCheckpoint); + logger.info(`Epoch ${txEpoch} proven on L1 up to checkpoint ${epochEndCheckpoint} after restart`); + + // The transaction proofs that were pending before the stop are re-requested and reused (returned + // with an existing status, never a fresh `not-found`, and never `aborted`). + const reRequests = broker.enqueues.slice(enqueuesBeforeRestart); + const revivedTxProofs = reRequests.filter( + e => pendingTxProofIds.includes(e.job.id) && e.returnedStatus !== 'not-found', + ); + expect(revivedTxProofs.length).toBeGreaterThan(0); + // The revived work is genuinely non-parity (the leaf parity jobs are not what we're asserting on). + expect(revivedTxProofs.every(e => !isParity(e.job.type))).toBe(true); + expect(reRequests.every(e => e.returnedStatus !== 'aborted')).toBe(true); + }); + }); +}); diff --git a/yarn-project/end-to-end/src/single-node/single_node_test_context.ts b/yarn-project/end-to-end/src/single-node/single_node_test_context.ts index 83360a4f2d61..2e73c4fee273 100644 --- a/yarn-project/end-to-end/src/single-node/single_node_test_context.ts +++ b/yarn-project/end-to-end/src/single-node/single_node_test_context.ts @@ -27,7 +27,7 @@ import { executeTimeout } from '@aztec/foundation/timer'; import { SpamContract } from '@aztec/noir-test-contracts.js/Spam'; import { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { getMockPubSubP2PServiceFactory } from '@aztec/p2p/test-helpers'; -import type { ProverNodeConfig } from '@aztec/prover-node'; +import type { ProverNodeConfig, ProverNodeDeps } from '@aztec/prover-node'; import type { PXEConfig } from '@aztec/pxe/config'; import { type Sequencer, type SequencerClient, type SequencerEvents, SequencerState } from '@aztec/sequencer-client'; import { type BlockParameter, EthAddress } from '@aztec/stdlib/block'; @@ -371,8 +371,11 @@ export class SingleNodeTestContext { return accountManager.address; } - public async createProverNode(opts: { dontStart?: boolean } & Partial = {}) { + public async createProverNode( + opts: { dontStart?: boolean; proverNodeDeps?: Partial } & Partial = {}, + ) { this.logger.warn('Creating and syncing a simulated prover node...'); + const { proverNodeDeps, ...configOverrides } = opts; const proverNodePrivateKey = this.getNextPrivateKey(); const proverIndex = this.proverNodes.length + 1; const { mockGossipSubNetwork } = this.context; @@ -385,7 +388,7 @@ export class SingleNodeTestContext { p2pEnabled: this.context.config.p2pEnabled || mockGossipSubNetwork !== undefined, proverId: EthAddress.fromNumber(proverIndex), dontStart: opts.dontStart, - ...opts, + ...configOverrides, }, { dataDirectory: join(this.context.config.dataDirectory!, randomBytes(8).toString('hex')), @@ -398,6 +401,7 @@ export class SingleNodeTestContext { : undefined, rpcTxProviders: [this.context.aztecNode], }, + proverNodeDeps, }, { genesis: this.context.genesis, diff --git a/yarn-project/prover-node/src/job/epoch-session.test.ts b/yarn-project/prover-node/src/job/epoch-session.test.ts index dea4db077bca..c13701bd4d99 100644 --- a/yarn-project/prover-node/src/job/epoch-session.test.ts +++ b/yarn-project/prover-node/src/job/epoch-session.test.ts @@ -199,6 +199,26 @@ describe('EpochSession', () => { expect(topTree.stop).toHaveBeenCalled(); }); + it('aborts the in-flight broker jobs on a normal cancel (abortJobs defaults to true)', async () => { + const proveGate = promiseWithResolvers(); + const session = makeSession({ hooks: { topTreeProveOverride: () => proveGate.promise } }); + const startResult = session.start(); + await topTreeConstructed.promise; + await session.cancel('canonical content changed'); + await expect(startResult).resolves.toBe('cancelled'); + expect(topTree.cancel).toHaveBeenCalledWith({ abortJobs: true }); + }); + + it('preserves the in-flight broker jobs when cancelled with abortJobs=false (clean shutdown)', async () => { + const proveGate = promiseWithResolvers(); + const session = makeSession({ hooks: { topTreeProveOverride: () => proveGate.promise } }); + const startResult = session.start(); + await topTreeConstructed.promise; + await session.cancel('prover-node stopping', { abortJobs: false }); + await expect(startResult).resolves.toBe('cancelled'); + expect(topTree.cancel).toHaveBeenCalledWith({ abortJobs: false }); + }); + it('cancel after start has settled leaves the existing terminal state in place', async () => { publishingService.submit.mockResolvedValue('published'); const session = makeSession(); diff --git a/yarn-project/prover-node/src/job/epoch-session.ts b/yarn-project/prover-node/src/job/epoch-session.ts index a6a325ee1ce7..e174d034953f 100644 --- a/yarn-project/prover-node/src/job/epoch-session.ts +++ b/yarn-project/prover-node/src/job/epoch-session.ts @@ -228,7 +228,7 @@ export class EpochSession implements Traceable { * Cancels the session. Idempotent. Withdraws any submitted candidate from the * publishing service so the in-flight publisher (if any) is interrupted. */ - public async cancel(reason = 'cancelled'): Promise { + public async cancel(reason = 'cancelled', { abortJobs = true }: { abortJobs?: boolean } = {}): Promise { if (this.isTerminal()) { return; } @@ -247,7 +247,9 @@ export class EpochSession implements Traceable { if (this.topTreeJob && !this.topTreeJob.isCancelled()) { const job = this.topTreeJob; this.topTreeJob = undefined; - job.cancel(); + // On a clean shutdown we leave the in-flight broker jobs alone so a restart can reuse them; + // other cancellations (reorg, supersede, deadline) abort them since their inputs are stale. + job.cancel(abortJobs); this.pendingTopTreeCleanups.push(job); } await this.teardownTopTreeIfNeeded(); diff --git a/yarn-project/prover-node/src/job/top-tree-job.ts b/yarn-project/prover-node/src/job/top-tree-job.ts index d7ddc67463f9..1290ea9de381 100644 --- a/yarn-project/prover-node/src/job/top-tree-job.ts +++ b/yarn-project/prover-node/src/job/top-tree-job.ts @@ -128,7 +128,7 @@ export class TopTreeJob { * via `whenDone()` — the parent collects the cancelled job and awaits all * pending top-tree teardowns at the end of the epoch. */ - public cancel(): void { + public cancel(abortJobs = true): void { if (this.cancelled) { return; } @@ -146,7 +146,7 @@ export class TopTreeJob { // Fire and forget: parent awaits the cancel-driven teardown via whenDone(); the // chained .catch swallows rejections so the unawaited promise doesn't surface // as an unhandled rejection. - this.cancelPromise = this.runCancel().catch(() => {}); + this.cancelPromise = this.runCancel(abortJobs).catch(() => {}); } /** Resolves once the cancel-driven teardown of the underlying orchestrator has unwound. */ @@ -156,9 +156,9 @@ export class TopTreeJob { } } - private async runCancel(): Promise { + private async runCancel(abortJobs: boolean): Promise { try { - this.topTree.cancel({ abortJobs: true }); + this.topTree.cancel({ abortJobs }); } catch (err) { this.deps.log.error('Error cancelling top tree', err); } diff --git a/yarn-project/prover-node/src/session-manager.test.ts b/yarn-project/prover-node/src/session-manager.test.ts index 606ebac446a0..dde4d6c793d8 100644 --- a/yarn-project/prover-node/src/session-manager.test.ts +++ b/yarn-project/prover-node/src/session-manager.test.ts @@ -715,6 +715,8 @@ describe('SessionManager', () => { // stop() passes 'prover-node stopping' as the cancel reason — verify every session // saw it, so a future caller can grep logs for that string. expect(stubs.map(s => s.cancelReasons)).toEqual([['prover-node stopping'], ['prover-node stopping']]); + // A clean shutdown must preserve the in-flight broker jobs so a restart reuses them. + expect(stubs.map(s => s.cancelAbortJobs)).toEqual([[false], [false]]); }); it('stop awaits sessions whose cancel is in flight', async () => { @@ -822,6 +824,8 @@ type StubSession = { cancelled: boolean; /** Reasons captured for every cancel(reason) call. Lets assertions verify "why" the cancel fired. */ cancelReasons: string[]; + /** abortJobs captured for every cancel() call. Lets assertions verify a clean shutdown preserves jobs. */ + cancelAbortJobs: boolean[]; /** Optional gate held by tests that want to drive a cancel mid-flight. */ cancelBlocker?: Promise; /** Resolves the first time cancel() is invoked — tests use it to know when stop's cancel call lands. */ @@ -836,7 +840,7 @@ type StubSession = { getEpochNumber(): EpochNumber; getCheckpoints(): readonly CheckpointProver[]; isTerminal(): boolean; - cancel(reason?: string): Promise; + cancel(reason?: string, opts?: { abortJobs?: boolean }): Promise; start(): Promise; whenDone(): Promise; }; @@ -852,6 +856,7 @@ function makeStubSession(spec: SessionSpec, provers: readonly CheckpointProver[] state: 'awaiting-checkpoints', cancelled: false, cancelReasons: [], + cancelAbortJobs: [], cancelStarted: promiseWithResolvers(), donePromise: promise, resolveDone: resolve, @@ -885,8 +890,9 @@ function makeStubSession(spec: SessionSpec, provers: readonly CheckpointProver[] ]; return terminal.includes(this.state); }, - async cancel(reason?: string) { + async cancel(reason?: string, opts?: { abortJobs?: boolean }) { this.cancelReasons.push(reason ?? 'cancelled'); + this.cancelAbortJobs.push(opts?.abortJobs ?? true); this.cancelStarted.resolve(); if (this.cancelBlocker) { await this.cancelBlocker; diff --git a/yarn-project/prover-node/src/session-manager.ts b/yarn-project/prover-node/src/session-manager.ts index 1162958f53c2..e6721f921951 100644 --- a/yarn-project/prover-node/src/session-manager.ts +++ b/yarn-project/prover-node/src/session-manager.ts @@ -227,7 +227,9 @@ export class SessionManager { await this.epochTicker?.stop(); await this.reconcileQueue.cancel(); const sessions = this.allSessions(); - await Promise.allSettled(sessions.map(s => s.cancel('prover-node stopping'))); + // A clean shutdown is just a restart, so preserve the in-flight broker jobs (abortJobs: false) + // for the restarted node to reuse rather than re-proving the epoch from scratch. + await Promise.allSettled(sessions.map(s => s.cancel('prover-node stopping', { abortJobs: false }))); } // ---------------- reconcile ---------------- From c06ccd8a944cec70537a3721a40b5d022d9ec006 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Mon, 13 Jul 2026 18:52:00 -0300 Subject: [PATCH 21/35] docs: fee readme improvements (#24666) Additions to the README on gas and fees (cherry picked from commit 4d53a9f48b9d335426f841f9343b0be38372c7ed) --- yarn-project/stdlib/src/gas/README.md | 144 +++++++++++++++++++------- 1 file changed, 108 insertions(+), 36 deletions(-) diff --git a/yarn-project/stdlib/src/gas/README.md b/yarn-project/stdlib/src/gas/README.md index 982252ce703b..1bce97106cdb 100644 --- a/yarn-project/stdlib/src/gas/README.md +++ b/yarn-project/stdlib/src/gas/README.md @@ -1,47 +1,69 @@ # Aztec Gas and Fee Model The minimum fee per mana and its components are computed on L1 in -`l1-contracts/src/core/libraries/rollup/FeeLib.sol`. This document describes the -formulas, the oracle lag/lifetime mechanism, and the TypeScript types in this directory. +`l1-contracts/src/core/libraries/rollup/FeeLib.sol` (`fee_math.ts` in this directory is a +TypeScript port of those formulas, used to predict fees a few slots ahead). This document +describes the formulas, the oracle lag/lifetime mechanism, how a transaction's fee is +derived from them, the gas and data limits, and the TypeScript types in this directory. ## Mana -Aztec uses **mana** as its unit of work (analogous to Ethereum gas). Transactions consume -mana in two dimensions: **DA** (data availability) and **L2** (execution). The total fee -is `gasUsed * feePerMana` summed across both dimensions. +Aztec meters work as gas in two dimensions: **DA** (data availability, i.e. blob data +published to L1) and **L2** (execution). The L2 dimension is called **mana** (analogous to +Ethereum gas): block headers track `totalManaUsed`, and the fee model below prices one unit +of mana. The total fee is `gasUsed * feePerGas` summed across both dimensions. + +### The DA dimension is priced at zero + +Only the L2 dimension currently carries a price. When building checkpoint global variables, +the sequencer sets `feePerDaGas = 0` and sets `feePerL2Gas` to the L1-computed minimum fee +per mana (`sequencer-client/src/global_variable_builder/global_builder.ts` and +`fee_provider.ts` in the same directory). The cost of publishing data is still recovered: +the blob gas a checkpoint pays on L1 is part of the *sequencer cost* component of the mana +fee below. DA gas remains fully metered and limited (see +[Gas and Data Limits](#gas-and-data-limits)) — it bounds how much data a tx or checkpoint +may publish — it just contributes nothing to the fee today. ## Fee Components -The minimum fee per mana has four components: +The minimum fee per mana has four components. All are computed in ETH (wei) per mana and +converted to the fee asset at the end (see [Fee Asset Price](#fee-asset-price)). ### Sequencer Cost L1 cost to propose a checkpoint (calldata gas + blob data), amortized over `manaTarget`: ``` -sequencerCost = ((L1_GAS_PER_CHECKPOINT_PROPOSED * baseFee) +sequencerCost = ceil(((L1_GAS_PER_CHECKPOINT_PROPOSED * baseFee) + (BLOBS_PER_CHECKPOINT * BLOB_GAS_PER_BLOB * blobFee)) - / manaTarget + / manaTarget) ``` +Note that `BLOBS_PER_CHECKPOINT` here is FeeLib's own constant (3), not the protocol blob +capacity of the same name (6) — see the note under [Key Constants](#key-constants). + ### Prover Cost L1 cost to verify an epoch proof, amortized over epoch duration and `manaTarget`, plus a governance-set proving cost that compensates for off-chain proof generation: ``` -proverCost = (L1_GAS_PER_EPOCH_VERIFIED * baseFee / epochDuration) / manaTarget +proverCost = ceil(ceil((L1_GAS_PER_EPOCH_VERIFIED * baseFee) / epochDuration) / manaTarget) + provingCostPerMana ``` +Updates to `provingCostPerMana` are rate-limited on L1 (`FeeLib.updateProvingCostPerMana`): +at most one update every 30 days, each moving the value by at most ×1.5 (or ÷1.5), with a +floor of 2 wei per mana. + ### Congestion Cost An exponential surcharge when the network is congested (inspired by EIP-1559; the implementation uses the `fakeExponential` Taylor series approximation from EIP-4844): ``` -baseCost = sequencerCost + proverCost -congestionCost = baseCost * congestionMultiplier / MINIMUM_CONGESTION_MULTIPLIER - baseCost +baseCost = sequencerCost + proverCost +congestionCost = floor(baseCost * congestionMultiplier / MINIMUM_CONGESTION_MULTIPLIER) - baseCost ``` When there is no congestion the multiplier equals `MINIMUM_CONGESTION_MULTIPLIER` (1e9) @@ -50,11 +72,15 @@ and congestion cost is zero. ### Congestion Multiplier ``` -excessMana = max(0, prevExcessMana + prevManaUsed - manaTarget) -congestionMultiplier = fakeExponential(MINIMUM_CONGESTION_MULTIPLIER, excessMana, denominator) +excessMana = max(0, prevExcessMana + prevManaUsed - manaTarget) +denominator = manaTarget * 854,700,854 / 1e8 ≈ 8.547 * manaTarget +congestionMultiplier = fakeExponential(MINIMUM_CONGESTION_MULTIPLIER, + min(excessMana, 100 * denominator), denominator) ``` -Each additional `manaTarget` of excess mana increases the multiplier by ~12.5%. +Each additional `manaTarget` of excess mana multiplies the fee by `e^(1/8.547) ≈ 1.124`, +i.e. ~12.5%. The exponent is capped at 100 (multiplier ≤ ~2.7e43 × the minimum) to keep +the Taylor series from overflowing. ### Total @@ -62,6 +88,11 @@ Each additional `manaTarget` of excess mana increases the multiplier by ~12.5%. minFeePerMana = sequencerCost + proverCost + congestionCost ``` +Each component is converted from ETH to the fee asset individually (rounding up) before +summing. The sum is capped at `type(uint128).max` (`FeeLib.summedMinFee`) so it always fits +the proposal header's `feePerL2Gas` field — without the cap, extreme congestion could +produce a fee no valid header can represent, halting the chain. + ## L1 Gas Oracle: Lag and Lifetime The oracle feeds Ethereum's `baseFee` and `blobFee` into the fee model using a two-phase @@ -70,9 +101,9 @@ The oracle feeds Ethereum's `baseFee` and `blobFee` into the fee model using a t - **LAG = 2 slots** — when new L1 fees are observed, they activate `LAG` slots later (`slotOfChange = currentSlot + LAG`). This gives mempool transactions time to land before fees change. -- **LIFETIME = 5 slots** — after an oracle update, the next update is rejected until - `slotOfChange + (LIFETIME - LAG)` = 3 more slots have passed. This rate-limits how - frequently L1 fee data can change. +- **LIFETIME = 5 slots** — after an oracle update, further updates are ignored + (`updateL1GasFeeOracle` returns without effect) until `slotOfChange + (LIFETIME - LAG)` + = 3 more slots have passed. This rate-limits how frequently L1 fee data can change. Fee resolution at a given timestamp: @@ -84,6 +115,12 @@ else → use post (new fees) **Net effect**: L1 fee changes reach L2 with a 2-slot delay and can update at most once every 5 slots. +Because queued values only activate `LAG` slots later, the min fee for the next `LAG` +slots is fully determined by current on-chain state. `fee_math.ts` ports the FeeLib +formulas so the sequencer can predict min fees over that window (`FeePredictor` in +`sequencer-client/src/global_variable_builder`), under a configurable mana-usage +assumption (`ManaUsageEstimate`: none / target / limit). + ### Worked Example Suppose the oracle is updated at slot 10 with new L1 fees. Here is the timeline: @@ -117,34 +154,68 @@ Key observations: ## Fee Asset Price -Fees are computed in ETH internally but converted to the fee asset (Fee Juice) via -`ethPerFeeAsset` (1e12 precision). The price updates at most ±1% (±100 bps) per -checkpoint: +Fees are computed in ETH (wei) internally and converted to the fee asset (Fee Juice) via +`ethPerFeeAsset` (1e12 precision), rounding up (`PriceLib.toFeeAsset` in +`l1-contracts/src/core/libraries/compressed-data/fees/FeeConfig.sol`). + +Each checkpoint proposal carries a fee-asset price modifier (`OracleInput`) chosen by the +proposer, bounded to ±1% (±100 bps) per checkpoint: ``` newPrice = currentPrice * (10000 + modifierBps) / 10000 ``` +The result is clamped to [`MIN_ETH_PER_FEE_ASSET` = 100, `MAX_ETH_PER_FEE_ASSET` = 1e14], +i.e. 1e-10 to 100 ETH per fee asset. The floor of 100 guarantees a ±1% step always moves +the integer price by at least 1. + +## Transaction Fees + +The checkpoint's `gasFees` (the L1-computed min fee per mana, with DA priced at zero) act +as a base fee. Senders declare `GasSettings`: gas limits, teardown gas limits, +`maxFeesPerGas`, and `maxPriorityFeesPerGas`. A tx is only includable if `maxFeesPerGas` +covers the checkpoint's `gasFees` in both dimensions. The effective fee adds an +EIP-1559-style priority fee on top of the base, capped by the max +(`computeEffectiveGasFees` in `stdlib/src/fees/transaction_fee.ts`): + +``` +effectiveFeePerGas = gasFees + min(maxPriorityFeePerGas, maxFeesPerGas - gasFees) (per dimension) +transactionFee = billedGas.daGas * effectiveFeePerDaGas + billedGas.l2Gas * effectiveFeePerL2Gas +``` + +`billedGas` is actual consumption except for the teardown phase, which is billed at the +declared `teardownGasLimits` rather than actual usage — the fee is charged during teardown +itself, before actual teardown consumption is known. + ## Maximum Fee Change Rate -| Component | Bound | -| ---------------------- | ------------------------------------------------------- | -| L1 base fee / blob fee | At most once every 5 slots (oracle LIFETIME) | -| Fee asset price | ±1% per checkpoint | +| Component | Bound | +| ---------------------- | -------------------------------------------------------- | +| L1 base fee / blob fee | At most once every 5 slots (oracle LIFETIME) | +| Fee asset price | ±1% per checkpoint | +| Proving cost per mana | At most ×1.5 (or ÷1.5) per update, one update per 30 days | | Congestion multiplier | Depends on excess mana accumulation/drain per checkpoint | -| Sequencer/prover costs | Scale linearly with L1 fees | +| Sequencer/prover costs | Scale linearly with L1 fees | ## Key Constants -| Constant | Value | -| ------------------------------ | -------------- | -| `L1_GAS_PER_CHECKPOINT_PROPOSED` | 300,000 | -| `L1_GAS_PER_EPOCH_VERIFIED` | 3,600,000 | -| `BLOBS_PER_CHECKPOINT` | 3 | -| `BLOB_GAS_PER_BLOB` | 2^17 | -| `MINIMUM_CONGESTION_MULTIPLIER` | 1e9 | -| `LAG` | 2 slots | -| `LIFETIME` | 5 slots | +| Constant | Value | +| ------------------------------- | -------------- | +| `L1_GAS_PER_CHECKPOINT_PROPOSED` | 300,000 | +| `L1_GAS_PER_EPOCH_VERIFIED` | 3,600,000 | +| `BLOBS_PER_CHECKPOINT` (FeeLib) | 3 | +| `BLOB_GAS_PER_BLOB` | 2^17 | +| `MINIMUM_CONGESTION_MULTIPLIER` | 1e9 | +| `LAG` | 2 slots | +| `LIFETIME` | 5 slots | +| `MIN_ETH_PER_FEE_ASSET` | 100 (1e-10 ETH) | +| `MAX_ETH_PER_FEE_ASSET` | 1e14 (100 ETH) | + +⚠️ Name clash: `FeeLib.sol` (and its port `fee_math.ts`) defines `BLOBS_PER_CHECKPOINT = 3`, +used only to price the sequencer's blob costs. The protocol constant of the same name in +`@aztec/constants` is **6** — the actual blob capacity of a checkpoint, used throughout the +limits section below. The FeeLib value is a holdover from the pre-checkpoint +`BLOBS_PER_BLOCK`, so the fee model prices half of a full checkpoint's blobs. ## Gas and Data Limits @@ -196,7 +267,7 @@ advertises them in `NodeInfo.txsLimits` (a required field); wallets read it and `GasSettings.fallback` as the default gas limits when sending without explicit limits, and they are enforced by `GasLimitsValidator` (clamped to the per-tx protocol maxima) at three points: RPC tx acceptance (`aztec-node/src/aztec-node/server.ts`), gossip validation (`p2p/src/services/libp2p/libp2p_service.ts`), -and pending-pool admission (`p2p/src/client/factory.ts`). They are deliberately *not* enforced at reqresp or +and pending-pool admission (`p2p/src/client/factory.ts`). They are deliberately *not* enforced at req/resp or block-proposal validation — admission is relay policy, not block validity. ### Per-block builder budgets @@ -240,7 +311,8 @@ The outermost limits, enforced as proposal validity in `validateCheckpointLimits ## TypeScript Types -- **`Gas`** — mana quantity in two dimensions (`daGas`, `l2Gas`). +- **`Gas`** — gas quantity in two dimensions (`daGas`, `l2Gas`). - **`GasFees`** — per-unit price in each dimension (`feePerDaGas`, `feePerL2Gas`). - **`GasSettings`** — sender-chosen fee parameters: gas limits, teardown limits, max fees, priority fees. - **`GasUsed`** — actual consumption after execution. Note: `billedGas` uses the teardown gas *limit*, not actual usage. +- **`fee_math.ts`** — TypeScript port of the FeeLib formulas, used for fee prediction over the oracle LAG window. From cb6d8e41a2c38bc24dcf028a0c86d0edc620db82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Venturo?= Date: Mon, 13 Jul 2026 11:11:04 -0300 Subject: [PATCH 22/35] fix: prevent access to secrets not in scope (#24616) Some stores were being accessed without checking if the associated accounts were in scope or not. For logs I added this to the service, but we're not yet consistent in how we use services (call-lived, tx-lived?) so in other cases it is just inlined in the oracle. (cherry picked from commit ce12e293c4d3aa8eca8a3336453a9c39967df8fc) --- .../oracle/utility_execution.test.ts | 29 ++++++++++++------- .../oracle/utility_execution_oracle.ts | 10 ++++++- yarn-project/pxe/src/logs/log_service.test.ts | 25 +++++++++++++--- yarn-project/pxe/src/logs/log_service.ts | 4 +++ yarn-project/pxe/src/notes/note_service.ts | 2 +- 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts index 6e5c956302f8..48beacfb3d23 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts @@ -564,8 +564,8 @@ describe('Utility Execution test suite', () => { const contractAddressA = await AztecAddress.random(); const contractAddressB = await AztecAddress.random(); - const oracleA = makeOracle({ contractAddress: contractAddressA }); - const oracleB = makeOracle({ contractAddress: contractAddressB }); + const oracleA = makeOracle({ contractAddress: contractAddressA, scopes: [owner] }); + const oracleB = makeOracle({ contractAddress: contractAddressB, scopes: [owner] }); const ephPksArray = EphemeralArray.fromValues(service, [ephPk]); const responseA = await oracleA.getSharedSecrets(owner, ephPksArray, contractAddressA); @@ -593,15 +593,14 @@ describe('Utility Execution test suite', () => { ); }); - it('returns no secrets when the PXE does not hold the keys for the address', async () => { + it('rejects a recipient outside the allowed scopes', async () => { const ephSk = GrumpkinScalar.random(); const ephPk = await Grumpkin.mul(Grumpkin.generator, ephSk); - const foreignAddress = await AztecAddress.random(); const ephPksArray = EphemeralArray.fromValues(service, [ephPk]); - const response = await utilityExecutionOracle.getSharedSecrets(foreignAddress, ephPksArray, contractAddress); - - expect(response.readAll(service)).toEqual([]); + await expect(utilityExecutionOracle.getSharedSecrets(owner, ephPksArray, contractAddress)).rejects.toThrow( + /not in the allowed scopes/, + ); }); }); @@ -643,10 +642,8 @@ describe('Utility Execution test suite', () => { ); }); - const result = await utilityExecutionOracle.getPendingTaggedLogsV2( - owner, - EphemeralArray.fromValues(service, providedSecrets), - ); + const oracle = makeOracle({ scopes: [owner] }); + const result = await oracle.getPendingTaggedLogsV2(owner, EphemeralArray.fromValues(service, providedSecrets)); const queried = aztecNode.getPrivateLogsByTags.mock.calls.flatMap(([query]) => query.tags.map(entry => ('tag' in entry ? entry.tag.value.toString() : entry.value.toString())), @@ -667,6 +664,16 @@ describe('Utility Execution test suite', () => { }, ]); }); + + it('rejects a scope outside the allowed scopes', async () => { + const outOfScope = await AztecAddress.random(); + await expect( + utilityExecutionOracle.getPendingTaggedLogsV2( + outOfScope, + EphemeralArray.fromValues(service, []), + ), + ).rejects.toThrow(/not in the allowed scopes/); + }); }); describe('node read cache', () => { diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts index 0f318021b75f..ca1849da99a5 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts @@ -48,6 +48,7 @@ import { TxResolverService } from '../../messages/tx_resolver_service.js'; import { NoteService } from '../../notes/note_service.js'; import { ORACLE_VERSION_MAJOR } from '../../oracle_version.js'; import type { AddressStore } from '../../storage/address_store/address_store.js'; +import { assertAllowedScope } from '../../storage/allowed_scopes.js'; import type { CapsuleService } from '../../storage/capsule_store/capsule_service.js'; import { FactCollectionKey, FactCollectionTypeKey, anchoredTipBlockNumbers } from '../../storage/fact_store/index.js'; import type { FactService, OriginBlock } from '../../storage/fact_store/index.js'; @@ -598,6 +599,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra this.recipientTaggingStore, this.taggingSecretSourcesStore, this.addressStore, + this.scopes, this.jobId, this.logger.getBindings(), ); @@ -857,7 +859,8 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra * @param ephPks - Ephemeral array containing the serialized Points. * @param contractAddress - The contract address for app-siloing (validated against execution context). * @returns A new ephemeral array containing the computed shared secrets, or an empty array when the PXE does not - * hold the keys for `address`, signaling that no secrets can be derived for it. + * hold the keys for `address`. + * @throws If `address` is not in the execution's allowed scopes. */ public async getSharedSecrets( address: AztecAddress, @@ -869,6 +872,11 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra `getSharedSecrets called with contract address ${contractAddress}, expected ${this.contractAddress}`, ); } + + assertAllowedScope(address, this.scopes); + + // An address can be in scope without the PXE holding its keys (e.g. syncing a registered but non-owned account), + // in which case no secrets can be derived and we return an empty array rather than failing. const recipientCompleteAddress = await this.addressStore.getCompleteAddress(address); if (!recipientCompleteAddress) { this.logger.warn( diff --git a/yarn-project/pxe/src/logs/log_service.test.ts b/yarn-project/pxe/src/logs/log_service.test.ts index 39b5ff298001..b3dc03c6005b 100644 --- a/yarn-project/pxe/src/logs/log_service.test.ts +++ b/yarn-project/pxe/src/logs/log_service.test.ts @@ -321,13 +321,17 @@ describe('LogService', () => { contractAddress = await AztecAddress.random(); const l2TipsProvider = mock(); - ({ aztecNode, keyStore, taggingSecretSourcesStore, addressStore, logService } = - await createTestLogService(l2TipsProvider)); + const scopes: AztecAddress[] = []; + ({ aztecNode, keyStore, taggingSecretSourcesStore, addressStore, logService } = await createTestLogService( + l2TipsProvider, + scopes, + )); l2TipsProvider.getL2Tips.mockResolvedValue(makeL2Tips(0)); const completeAddress = await keyStore.addAccount(await deriveKeys(Fr.random()), Fr.random()); await addressStore.addCompleteAddress(completeAddress); recipient = completeAddress.address; + scopes.push(recipient); sharedSecret = await Point.random(); }); @@ -396,6 +400,13 @@ describe('LogService', () => { expect(txHashes).not.toContainEqual(directionalLog.txHash); }); + it('rejects a recipient outside the allowed scopes', async () => { + const outOfScope = await AztecAddress.random(); + await expect(logService.fetchTaggedLogs(contractAddress, outOfScope, [])).rejects.toThrow( + /not in the allowed scopes/, + ); + }); + function handshakeTags(secret: Point, app: AztecAddress): Promise { return Promise.all( [AppTaggingSecretKind.UNCONSTRAINED, AppTaggingSecretKind.CONSTRAINED].map(async kind => @@ -421,7 +432,8 @@ describe('LogService', () => { contractAddress = await AztecAddress.random(); const l2TipsProvider = mock(); - const testContext = await createTestLogService(l2TipsProvider); + const scopes: AztecAddress[] = []; + const testContext = await createTestLogService(l2TipsProvider, scopes); ({ aztecNode, keyStore, taggingSecretSourcesStore, addressStore, logService } = testContext); l2TipsProvider.getL2Tips.mockResolvedValue(makeL2Tips(testContext.anchorBlockHeader.globalVariables.blockNumber)); @@ -429,6 +441,7 @@ describe('LogService', () => { recipientCompleteAddress = await keyStore.addAccount(await deriveKeys(new Fr(1)), Fr.random()); recipient = recipientCompleteAddress.address; await addressStore.addCompleteAddress(recipientCompleteAddress); + scopes.push(recipient); sender = await AztecAddress.random(); @@ -469,7 +482,10 @@ describe('LogService', () => { }); }); -async function createTestLogService(l2TipsProvider: MockProxy = mock()) { +async function createTestLogService( + l2TipsProvider: MockProxy = mock(), + scopes: AztecAddress[] = [], +) { const keyStore = new KeyStore(await openTmpStore('test')); const recipientTaggingStore = new RecipientTaggingStore(await openTmpStore('test')); const taggingSecretSourcesStore = new TaggingSecretSourcesStore(await openTmpStore('test')); @@ -486,6 +502,7 @@ async function createTestLogService(l2TipsProvider: MockProxy = recipientTaggingStore, taggingSecretSourcesStore, addressStore, + scopes, 'test', ); diff --git a/yarn-project/pxe/src/logs/log_service.ts b/yarn-project/pxe/src/logs/log_service.ts index 18dd7d2d2a55..2654f928fa3e 100644 --- a/yarn-project/pxe/src/logs/log_service.ts +++ b/yarn-project/pxe/src/logs/log_service.ts @@ -24,6 +24,7 @@ import type { LogRetrievalResponse } from '../contract_function_simulator/noir-s import type { PendingTaggedLog } from '../contract_function_simulator/noir-structs/pending_tagged_log.js'; import { ResolvedTx } from '../contract_function_simulator/noir-structs/resolved_tx.js'; import { AddressStore } from '../storage/address_store/address_store.js'; +import { assertAllowedScope } from '../storage/allowed_scopes.js'; import type { RecipientTaggingStore } from '../storage/tagging_store/recipient_tagging_store.js'; import type { TaggingSecretSourcesStore } from '../storage/tagging_store/tagging_secret_sources_store.js'; import { @@ -47,6 +48,7 @@ export class LogService { private readonly recipientTaggingStore: RecipientTaggingStore, private readonly taggingSecretSourcesStore: TaggingSecretSourcesStore, private readonly addressStore: AddressStore, + private readonly scopes: AztecAddress[], private readonly jobId: string, bindings?: LoggerBindings, ) { @@ -189,6 +191,8 @@ export class LogService { recipient: AztecAddress, providedSecrets: AppTaggingSecret[], ): Promise { + assertAllowedScope(recipient, this.scopes); + this.log.verbose( `Fetching tagged logs for contract ${contractAddress.toString()} and recipient ${recipient.toString()}`, ); diff --git a/yarn-project/pxe/src/notes/note_service.ts b/yarn-project/pxe/src/notes/note_service.ts index cbe1ff82624b..b84ef27739f2 100644 --- a/yarn-project/pxe/src/notes/note_service.ts +++ b/yarn-project/pxe/src/notes/note_service.ts @@ -26,7 +26,7 @@ export class NoteService { * * @param owner - The owner of the notes. If undefined, returns notes for all known owners. * @param status - The status of notes to fetch. - * @param scopes - The accounts whose notes we can access in this call. Currently optional and will default to all. + * @param scopes - The accounts whose notes we can access in this call. An empty list matches no notes. */ public async getNotes( contractAddress: AztecAddress, From 0f18c6ec0d52f71d996080b28303af5e10a96b1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Venturo?= Date: Mon, 13 Jul 2026 11:32:41 -0300 Subject: [PATCH 23/35] fix: prevent reception of messages too far into the future (#24645) I added a sanity check to avoid having messages implausibly far into the future, which would lead to them not getting evicted etc., but more importantly which would signal some inconsistency somewhere. (cherry picked from commit fdc2ac51128a30ddfbaa3a5e9e133fbc54204353) --- noir-projects/aztec-nr/CLAUDE.md | 30 ++++++++++ .../src/messages/processing/offchain/mod.nr | 57 ++++++++++++++++++- .../messages/processing/offchain/reception.nr | 12 ++++ .../txe/esbuild/plugins/size_guard.mjs | 4 +- 4 files changed, 99 insertions(+), 4 deletions(-) diff --git a/noir-projects/aztec-nr/CLAUDE.md b/noir-projects/aztec-nr/CLAUDE.md index a16864f2677e..975f709899d1 100644 --- a/noir-projects/aztec-nr/CLAUDE.md +++ b/noir-projects/aztec-nr/CLAUDE.md @@ -34,6 +34,36 @@ Follow the Rust stdlib style roughly. See `PublicImmutable` in `state_vars/publi - **Show practical patterns in examples.** Don't just show the API call — show it in context (e.g. inside a `#[external("public")]` function with realistic variable names). - **Document cost for methods.** When relevant, note which AVM opcodes are invoked and how many times (e.g. "`SLOAD` is invoked a number of times equal to `T`'s packed length"). +### Brevity and attitude + +Write for a caller using the API, not for an implementor or an auditor. State what the item does and any restriction a caller must respect, plus the alternative to reach for — then stop. Leave out the internal mechanism and the security rationale behind a restriction: those belong in code comments messages or design docs, not the rustdoc. A reader who hits a restriction needs to know it exists and where to go next, not the argument for why it holds. + +- **State the restriction, not the reason it's sound.** Say "this can only be used for X", not the multi-sentence explanation of what goes wrong otherwise. +- **Don't narrate the mechanism.** Key siloing, tree layouts, hash preimages, and similar internals are implementation detail; mention them only when a caller cannot use the API correctly without knowing (and then put them under `## Implementation Details`). +- **Point to the alternative instead of enumerating trade-offs.** A single "use `[other_fn]` for the other case" beats a paragraph weighing options. + +Example — documenting a helper that only works on the executing contract's own notes: + +```rust +// Too much: explains the key-siloing mechanism and the failure the guard prevents. +/// The note's nullifier is recomputed using the executing contract's app-siloed nullifier key +/// and then siloed with `confirmed_note.contract_address`. These only agree for the executing +/// contract's own notes; otherwise the non-inclusion proof passes unconditionally and wrongly +/// reports a nullified note as not nullified, so this asserts that +/// `confirmed_note.contract_address == context.this_address()`. + +// Right: states the restriction and the alternative, nothing more. +/// ## Local notes only +/// +/// This can only be used for notes of the executing contract. Use [`assert_note_existed_by`] to +/// prove existence of another contract's notes. +``` + +## Testing + +- **No messages on assertions inside tests.** Write `assert_eq(actual, expected)` and `assert(cond)`, not `assert_eq(actual, expected, "values should match")`. The test's name and body already make clear what is being checked, so an assertion message is redundant noise. (Library code is the opposite: an `assert`/`panic` there must carry a message, since it explains a runtime failure to a contract developer.) +- `#[test(should_fail_with = "...")]` is not an assertion message and is encouraged. The string matches a substring of the expected failure, pinning down *which* error the failing case must produce so the test can't pass for the wrong reason. + ## Logging - **Always use the prefixed logging functions** from `crate::logging` (e.g. `logging::aztecnr_debug_log!`, `logging::aztecnr_debug_log_format!`). These automatically prepend `[aztec-nr] ` to all messages at compile time. diff --git a/noir-projects/aztec-nr/aztec/src/messages/processing/offchain/mod.nr b/noir-projects/aztec-nr/aztec/src/messages/processing/offchain/mod.nr index 5bd712009923..5308ac9a830e 100644 --- a/noir-projects/aztec-nr/aztec/src/messages/processing/offchain/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/messages/processing/offchain/mod.nr @@ -7,7 +7,7 @@ use crate::{ }; mod reception; -use reception::OffchainReception; +use reception::{MAX_ANCHOR_FUTURE_SKEW, OffchainReception}; /// Maximum number of offchain messages accepted by `offchain_receive` in a single call. pub global MAX_OFFCHAIN_MESSAGES_PER_RECEIVE_CALL: u32 = 16; @@ -50,6 +50,19 @@ pub unconstrained fn receive( // Offchain reception facts are scoped to the recipient. Note that because offchain messages have no integrity or // authenticity checks it is not possible to verify that this is indeed the intended recipient. In this case we're // assuming a cooperative environment, which is on par with other offchain delivery expectations. + // + // The sender-supplied anchor timestamp is the one field we bound: a message claiming to originate implausibly far + // in our future (see MAX_ANCHOR_FUTURE_SKEW) cannot be genuine, and accepting it would let a malicious anchor evade + // the TTL-based inbox eviction. We reject such a batch by panicking instead of silently dropping the message, so + // the calling wallet can react to it (e.g. surface the error or distrust the sender). + let now = UtilityContext::new().timestamp(); + messages.for_each(|msg| { + assert( + msg.anchor_block_timestamp <= now + MAX_ANCHOR_FUTURE_SKEW, + "offchain message anchor timestamp is implausibly far in the future", + ); + }); + messages.for_each(|msg| OffchainReception::init(contract_address, msg)); // Clear cache for message recipients so the next sync runs. @@ -97,10 +110,12 @@ pub struct OffchainMessage { mod test { use crate::{ - oracle::random::random, protocol::address::AztecAddress, test::helpers::test_environment::TestEnvironment, + oracle::random::random, + protocol::{address::AztecAddress, constants::MAX_TX_LIFETIME}, + test::helpers::test_environment::TestEnvironment, }; use super::{MAX_OFFCHAIN_MESSAGES_PER_RECEIVE_CALL, OffchainMessage, receive, sync_inbox}; - use super::reception::{MAX_MSG_TTL, OffchainReception}; + use super::reception::{MAX_ANCHOR_FUTURE_SKEW, MAX_MSG_TTL, OffchainReception}; unconstrained fn setup() -> (TestEnvironment, AztecAddress) { let mut env = TestEnvironment::new(); @@ -177,6 +192,42 @@ mod test { }); } + #[test] + unconstrained fn accepts_messages_within_the_tolerated_future_skew() { + let (env, scope) = setup(); + let now = advance_by(env, 10); + + // A present-dated anchor and a 24h-future one both fall within the tolerated skew. The latter models a genuine + // message whose sender anchored ahead of our lagging PXE, and must still be accepted. + let present = make_msg(scope, Option::some(random()), now); + let lagged = make_msg(scope, Option::some(random()), now + MAX_TX_LIFETIME); + + env.utility_context(|context| { + let address = context.this_address(); + receive(address, BoundedVec::from_array([present, lagged])); + + assert(OffchainReception::is_active(address, scope, present)); + assert(OffchainReception::is_active(address, scope, lagged)); + assert_eq(OffchainReception::load_all(address, scope).len(), 2); + }); + } + + #[test(should_fail_with = "offchain message anchor timestamp is implausibly far in the future")] + unconstrained fn rejects_batch_with_implausible_future_anchor() { + let (env, scope) = setup(); + let now = advance_by(env, 10); + + // Beyond `now + MAX_ANCHOR_FUTURE_SKEW`: our view would trail the tip by more than a tx lifetime, so this + // cannot be genuine. Reception panics so the wallet can react rather than accept it. + let too_future = make_msg( + scope, + Option::some(random()), + now + MAX_ANCHOR_FUTURE_SKEW + 7200, + ); + + env.utility_context(|context| { receive(context.this_address(), BoundedVec::from_array([too_future])); }); + } + // -- Idempotent re-delivery (first-write-wins fact recording) --------- #[test] diff --git a/noir-projects/aztec-nr/aztec/src/messages/processing/offchain/reception.nr b/noir-projects/aztec-nr/aztec/src/messages/processing/offchain/reception.nr index 36bafe540a1e..99c0e15a9ba3 100644 --- a/noir-projects/aztec-nr/aztec/src/messages/processing/offchain/reception.nr +++ b/noir-projects/aztec-nr/aztec/src/messages/processing/offchain/reception.nr @@ -117,6 +117,18 @@ use super::OffchainMessage; /// (7200 == 2 hours) pub(crate) global MAX_MSG_TTL: u64 = MAX_TX_LIFETIME + 7200; +/// Maximum amount by which a received message's `anchor_block_timestamp` may exceed our current timestamp. +/// +/// A message's anchor is the timestamp of the block the sender anchored its originating tx to, so for a genuine +/// message it is at or before our own anchor timestamp. Our PXE may lag the chain tip though, which can make a +/// legitimate anchor appear to be in our future, so we tolerate a forward skew of `MAX_TX_LIFETIME` (the longest a tx +/// can wait to be mined after its anchor) plus a 2h margin. A larger skew would mean our view trails the tip by more +/// than a tx lifetime -- so far behind that we could not even build a tx that wouldn't be immediately expired -- so +/// such a message cannot be genuine and is rejected on reception. Bounding the anchor this way also keeps accepted +/// values near the present, which prevents the `anchor_block_timestamp + MAX_MSG_TTL` eviction check from overflowing. +/// (7200 == 2 hours) +pub(crate) global MAX_ANCHOR_FUTURE_SKEW: u64 = MAX_TX_LIFETIME + 7200; + /// Fact type id of the fact that stores the offchain message body inside a reception collection. global OFFCHAIN_MESSAGE_RECEIVED: Field = sha256_to_field("AZTEC_NR::OFFCHAIN_MESSAGE_RECEIVED".as_bytes()); diff --git a/yarn-project/txe/esbuild/plugins/size_guard.mjs b/yarn-project/txe/esbuild/plugins/size_guard.mjs index 556801f4e1e2..40787feb2a57 100644 --- a/yarn-project/txe/esbuild/plugins/size_guard.mjs +++ b/yarn-project/txe/esbuild/plugins/size_guard.mjs @@ -11,6 +11,8 @@ // Bump log: // - 2026-05-27: initial limits. +// - 2026-07-10: total 14 -> 14.5 MiB. Stopgap: the total was already ~13.9995 MiB (mostly sourcemaps, which this +// guard counts), so a tiny per-contract bytecode change tipped it over. Pending a fix to exclude .map from the total. export const sizeLimits = [ // Shared chunks emitted by code-splitting; carry the simulator + PXE + world-state graph. // Spikes here usually mean a heavy dep crept into the eager import path. @@ -22,7 +24,7 @@ export const sizeLimits = [ { pattern: /^dest\/bin\/index\.js$/, maxKB: 8, description: 'CLI entrypoint stub' }, ]; -export const totalLimitMiB = 14; +export const totalLimitMiB = 14.5; /** * Validates a built esbuild `metafile` against the configured limits. Logs all violations then From 7976ace156807bfe1ec88fb5c0ad7927b9167370 Mon Sep 17 00:00:00 2001 From: Nicolas Chamo Date: Mon, 13 Jul 2026 23:53:54 -0300 Subject: [PATCH 24/35] fix(aztec-nr): reject infinity ephemeral key in message encryption (#24665) ## Problem When encrypting a message, the ephemeral secret key comes from an unconstrained routine, so a malicious sender can substitute any value while proving. Substituting `eph_sk = 0` yields the point at infinity as the ephemeral public key, which passed the y-sign check: its y-coordinate is 0, which counts as positive. Its x-coordinate (0) is then broadcast, but 0 is not a valid x-coordinate on the curve, so the recipient can never reconstruct the key and the message is permanently undecryptable. This breaks the constrained-delivery guarantee that a note delivered by an untrusted sender remains decryptable by the recipient. ## Fix `generate_positive_ephemeral_key_pair` now asserts the ephemeral public key is not the point at infinity, alongside the existing sign check. A test emulates the substitution by mocking the randomness oracle to return 0. Fixes F-799 (cherry picked from commit 47c30a1f39b54904f123485277825a05e1ff19ae) --- noir-projects/aztec-nr/aztec/src/keys/ephemeral.nr | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/noir-projects/aztec-nr/aztec/src/keys/ephemeral.nr b/noir-projects/aztec-nr/aztec/src/keys/ephemeral.nr index d0947c61d281..3afaced71e51 100644 --- a/noir-projects/aztec-nr/aztec/src/keys/ephemeral.nr +++ b/noir-projects/aztec-nr/aztec/src/keys/ephemeral.nr @@ -37,6 +37,9 @@ pub fn generate_positive_ephemeral_key_pair() -> (Scalar, EmbeddedCurvePoint) { let eph_sk = unsafe { generate_secret_key_for_positive_public_key() }; let eph_pk = fixed_base_scalar_mul(eph_sk); + // The point at infinity has x = 0, which is not a valid x-coordinate on the curve, so the recipient could + // never reconstruct the key from it and the message would be undecryptable. + assert(!eph_pk.is_infinite(), "Ephemeral public key is the point at infinity"); assert(get_sign_of_point(eph_pk), "Got an ephemeral public key with a negative y coordinate"); (eph_sk, eph_pk) @@ -63,6 +66,7 @@ unconstrained fn generate_secret_key_for_positive_public_key() -> EmbeddedCurveS mod test { use crate::utils::point::get_sign_of_point; use super::generate_positive_ephemeral_key_pair; + use std::test::OracleMock; #[test] fn generate_positive_ephemeral_key_pair_produces_positive_keys() { @@ -73,4 +77,11 @@ mod test { assert(get_sign_of_point(pk)); } } + + #[test(should_fail_with = "point at infinity")] + unconstrained fn generate_positive_ephemeral_key_pair_rejects_zero_randomness() { + // Making the randomness oracle return 0 emulates a malicious sender substituting eph_sk = 0. + let _ = OracleMock::mock("aztec_misc_getRandomField").returns(0); + let _ = generate_positive_ephemeral_key_pair(); + } } From 217a85acc6ad59392ecec53c39dd2e7dac2d6f1e Mon Sep 17 00:00:00 2001 From: Gregorio Juliana Date: Tue, 14 Jul 2026 15:55:05 +0200 Subject: [PATCH 25/35] feat: exported in-process testing network (#24629) Upstreaming the utilities build for `aztec-kit` so external projects can benefit from our e2e scaffolding. Essentially the same thing we already use, but pure ts rather than the unpublishable .sh script. Verified with `ci-full-no-test-cache` (cherry picked from commit 7e090083057aa2d34b5cd646dd4e69c28303e345) --- l1-contracts/scripts/forge_broadcast.js | 8 +- yarn-project/aztec/bootstrap.sh | 2 + .../aztec/src/local-network/local-network.ts | 7 +- yarn-project/aztec/src/testing/index.ts | 1 + .../aztec/src/testing/local-network.test.ts | 52 ++++++++++ .../aztec/src/testing/local-network.ts | 97 +++++++++++++++++++ .../end-to-end/src/fixtures/fixtures.ts | 8 +- .../ethereum/scripts/anvil_kill_wrapper.sh | 51 ---------- .../ethereum/src/deploy_aztec_l1_contracts.ts | 20 +++- yarn-project/ethereum/src/foundry_binary.ts | 57 +++++++++++ yarn-project/ethereum/src/test/start_anvil.ts | 44 +++++++-- 11 files changed, 278 insertions(+), 69 deletions(-) create mode 100644 yarn-project/aztec/src/testing/local-network.test.ts create mode 100644 yarn-project/aztec/src/testing/local-network.ts delete mode 100755 yarn-project/ethereum/scripts/anvil_kill_wrapper.sh create mode 100644 yarn-project/ethereum/src/foundry_binary.ts diff --git a/l1-contracts/scripts/forge_broadcast.js b/l1-contracts/scripts/forge_broadcast.js index fb7b809e4a5a..1d61effab83e 100755 --- a/l1-contracts/scripts/forge_broadcast.js +++ b/l1-contracts/scripts/forge_broadcast.js @@ -59,11 +59,11 @@ const timeoutMs = (isAnvil ? 120_000 : 1_200_000); const proc = spawn( - "forge", + process.env.FORGE_BIN || "forge", ["script", ...args, "--broadcast", "--batch-size", batchSize], { stdio: ["ignore", "pipe", "inherit"], - }, + } ); const stdout = []; @@ -85,14 +85,14 @@ const exitCode = await new Promise((resolve) => { }); proc.on("close", (code) => { clearTimeout(timeout); - resolve(timedOut ? 1 : (code ?? 1)); + resolve(timedOut ? 1 : code ?? 1); }); }); log( exitCode === 0 ? "Broadcast succeeded." - : `Broadcast failed (exit ${exitCode}).`, + : `Broadcast failed (exit ${exitCode}).` ); const data = Buffer.concat(stdout); if (data.length > 0) writeSync(1, data); diff --git a/yarn-project/aztec/bootstrap.sh b/yarn-project/aztec/bootstrap.sh index c27fba277781..fadce4199f06 100755 --- a/yarn-project/aztec/bootstrap.sh +++ b/yarn-project/aztec/bootstrap.sh @@ -12,6 +12,8 @@ function test_cmds { # All CLI tests share test/mixed-workspace/target so they must run sequentially # in a single jest invocation (--runInBand is set by run_test.sh). echo "$hash:ISOLATE=1:NAME=aztec/cli NARGO=$NARGO BB=$BB PROFILER_PATH=$PROFILER_PATH yarn-project/scripts/run_test.sh aztec/src/cli" + # setupLocalNetwork smoke test: spins up anvil + an in-process node (network usage ⇒ ISOLATE). + echo "$hash:ISOLATE=1:NAME=aztec/testing yarn-project/scripts/run_test.sh aztec/src/testing/local-network.test.ts" } case "$cmd" in diff --git a/yarn-project/aztec/src/local-network/local-network.ts b/yarn-project/aztec/src/local-network/local-network.ts index eefbd37ed8df..9c0fb10eec23 100644 --- a/yarn-project/aztec/src/local-network/local-network.ts +++ b/yarn-project/aztec/src/local-network/local-network.ts @@ -88,6 +88,8 @@ export type LocalNetworkConfig = AztecNodeConfig & { l1Mnemonic: string; /** Whether to deploy test accounts on local network start.*/ testAccounts: boolean; + /** Override the default per-address fee juice granted at genesis to funded addresses. */ + initialAccountFeeJuice?: Fr; }; /** @@ -176,7 +178,10 @@ export async function createLocalNetwork(config: Partial = { ...(initialAccounts.length ? [bananaFPC, sponsoredFPC] : []), ...prefundAddresses, ]; - const { genesisArchiveRoot, genesis, fundingNeeded } = await getGenesisValues(fundedAddresses); + const { genesisArchiveRoot, genesis, fundingNeeded } = await getGenesisValues( + fundedAddresses, + config.initialAccountFeeJuice, + ); const dateProvider = new TestDateProvider(); diff --git a/yarn-project/aztec/src/testing/index.ts b/yarn-project/aztec/src/testing/index.ts index 0155a48144fd..6459e351146b 100644 --- a/yarn-project/aztec/src/testing/index.ts +++ b/yarn-project/aztec/src/testing/index.ts @@ -1,4 +1,5 @@ export { EthCheatCodes, RollupCheatCodes } from '@aztec/ethereum/test'; export { CheatCodes } from './cheat_codes.js'; export { EpochTestSettler } from './epoch_test_settler.js'; +export { setupLocalNetwork, TEST_FEE_PADDING, type LocalNetwork, type LocalNetworkOptions } from './local-network.js'; export { getTokenAllowedSetupFunctions } from './token_allowed_setup.js'; diff --git a/yarn-project/aztec/src/testing/local-network.test.ts b/yarn-project/aztec/src/testing/local-network.test.ts new file mode 100644 index 000000000000..648a748f2b51 --- /dev/null +++ b/yarn-project/aztec/src/testing/local-network.test.ts @@ -0,0 +1,52 @@ +import { getInitialTestAccountsData } from '@aztec/accounts/testing'; +import { TokenContract } from '@aztec/noir-contracts.js/Token'; +import { EmbeddedWallet } from '@aztec/wallets/embedded'; + +import { TEST_FEE_PADDING, setupLocalNetwork } from './local-network.js'; + +describe('setupLocalNetwork', () => { + it('serves a live node on a random L1 port and tears down cleanly', async () => { + await using net = await setupLocalNetwork(); + const info = await net.node.getNodeInfo(); + expect(info.l1ContractAddresses.rollupAddress).toBeDefined(); + expect(await net.node.getBlockNumber()).toBeGreaterThanOrEqual(0); + expect(net.l1ChainId).toBe(31337); + // OS-assigned ephemeral port, never the fixed default 8545. + expect(net.l1RpcUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(net.l1RpcUrl).not.toContain(':8545'); + }, 300_000); + + it('runs two networks in parallel on distinct ports', async () => { + const [a, b] = await Promise.all([setupLocalNetwork(), setupLocalNetwork()]); + try { + expect(a.l1RpcUrl).not.toEqual(b.l1RpcUrl); + expect(await a.node.getBlockNumber()).toBeGreaterThanOrEqual(0); + expect(await b.node.getBlockNumber()).toBeGreaterThanOrEqual(0); + } finally { + await Promise.all([a.stop(), b.stop()]); + } + }, 300_000); + + it('pre-funds addresses at genesis so they can pay for their own txs', async () => { + const [alice] = await getInitialTestAccountsData(); + const net = await setupLocalNetwork({ fundedAddresses: [alice.address] }); + try { + const wallet = await EmbeddedWallet.create(net.node, { + ephemeral: true, + pxeConfig: { proverEnabled: false }, + }); + await wallet.createSchnorrInitializerlessAccount(alice.secret, alice.salt, alice.signingKey); + wallet.setMinFeePadding(TEST_FEE_PADDING); + + const { contract } = await TokenContract.deploy(wallet, alice.address, 'TokenName', 'TKN', 18).send({ + from: alice.address, + }); + expect(contract.address).toBeDefined(); + expect(await net.node.getBlockNumber()).toBeGreaterThan(0); + + await wallet.stop(); + } finally { + await net.stop(); + } + }, 300_000); +}); diff --git a/yarn-project/aztec/src/testing/local-network.ts b/yarn-project/aztec/src/testing/local-network.ts new file mode 100644 index 000000000000..39c3a97e0167 --- /dev/null +++ b/yarn-project/aztec/src/testing/local-network.ts @@ -0,0 +1,97 @@ +import type { AztecNodeService } from '@aztec/aztec-node'; +import type { AztecNodeConfig } from '@aztec/aztec-node/config'; +import type { Fr } from '@aztec/aztec.js/fields'; +import { startAnvil } from '@aztec/ethereum/test'; +import type { AztecAddress } from '@aztec/stdlib/aztec-address'; + +import { foundry } from 'viem/chains'; + +import { createLocalNetwork } from '../local-network/local-network.js'; + +/** A running in-process local network: an inline Aztec node backed by its own anvil L1. */ +export interface LocalNetwork extends AsyncDisposable { + /** Fully-synced Aztec node, ready to serve client requests. */ + node: AztecNodeService; + /** RPC URL of the spawned anvil instance. */ + l1RpcUrl: string; + /** Chain id used on L1 (foundry's default 31337). */ + l1ChainId: number; + /** Stops every process started by the fixture: node and anvil. Also invoked by `await using`. */ + stop: () => Promise; +} + +/** Options for {@link setupLocalNetwork}. */ +export interface LocalNetworkOptions { + /** + * Addresses that should hold fee juice at genesis. Saves each of these the round-trip of bridging + * + claiming fee juice before they can pay for gas. + */ + fundedAddresses?: AztecAddress[]; + /** Override the default per-address genesis fee juice granted to {@link fundedAddresses}. */ + initialAccountFeeJuice?: Fr; + /** Node config overrides, e.g. `realProofs`, `aztecEpochDuration`, `p2pEnabled`. */ + config?: Partial; +} + +/** + * Spin up an in-process local network with the given addresses pre-funded. + * + * Each call spawns its own anvil on an OS-assigned random port and runs the Aztec node inline via + * the same {@link createLocalNetwork} codepath that backs `aztec start --local-network` (with the + * sandbox account/FPC/token setup skipped). Distinct ports let independent suites run in parallel. + * The caller must `await result.stop()` in its teardown (or hold the result with `await using`). + * + * Requires a Foundry toolchain (`anvil`/`forge`), installed via `aztec-up` or `foundryup`. Binaries + * are located in the standard install directories or on `PATH`; set `$ANVIL_BIN` / `$FORGE_BIN` to + * pin specific ones. + */ +export async function setupLocalNetwork(opts: LocalNetworkOptions = {}): Promise { + // `--port 0` → anvil binds an ephemeral port that `startAnvil` reads back, so parallel suites + // never collide on a fixed port. + const { rpcUrl, stop: stopAnvil } = await startAnvil({ port: 0 }); + + try { + const { node, stop: stopNode } = await createLocalNetwork( + { + ...opts.config, + l1RpcUrls: [rpcUrl], + testAccounts: false, + prefundAddresses: (opts.fundedAddresses ?? []).map(a => a.toString()), + initialAccountFeeJuice: opts.initialAccountFeeJuice, + }, + () => {}, + ); + + // Stop the node before anvil (its teardown still talks to L1); the finally guarantees anvil is + // reaped even if node shutdown throws. + const stop = async () => { + try { + await stopNode(); + } finally { + await stopAnvil(); + } + }; + + return { + node, + l1RpcUrl: rpcUrl, + l1ChainId: foundry.id, + stop, + [Symbol.asyncDispose]: stop, + }; + } catch (err) { + await stopAnvil(); + throw err; + } +} + +/** + * Min-fee padding multiplier for test wallets whose txs may mine well after their fee estimate. + * The automine sequencer builds one block per tx and advances L1 time in big jumps, and proposer + * pipelining evolves the fee-asset price across the build/publish gap (~20x observed in CI), so the + * network's congestion base fee can swing sharply between the wallet's fee estimate and the block + * the tx actually lands in. The default wallet padding isn't enough and trips + * `maxFeesPerGas.feePerL2Gas must be >= gasFees.feePerL2Gas`. Apply via + * `wallet.setMinFeePadding(TEST_FEE_PADDING)` on every test wallet that sends txs. + */ +export const TEST_FEE_PADDING = 30; diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts index 586c6d3ad474..217bc1ccf645 100644 --- a/yarn-project/end-to-end/src/fixtures/fixtures.ts +++ b/yarn-project/end-to-end/src/fixtures/fixtures.ts @@ -1,4 +1,5 @@ import type { AztecNode } from '@aztec/aztec.js/node'; +import { TEST_FEE_PADDING } from '@aztec/aztec/testing'; import type { GasFees } from '@aztec/stdlib/gas'; export const METRICS_PORT = 4318; @@ -17,9 +18,10 @@ export const LARGE_MIN_FEE_PADDING = 15; * price modifier evolves faster across the build/publish gap, so client-set maxFeesPerGas (sized * for the default 5x padding) was getting bumped past by the time the tx mined a few slots later. * Observed worst case in CI: fee evolved ~20x between PXE snapshot and inclusion, exceeding even - * LARGE_MIN_FEE_PADDING (15x). + * LARGE_MIN_FEE_PADDING (15x). Same multiplier and same class of problem as the published + * {@link TEST_FEE_PADDING}, re-exported under a name that captures the pipelining rationale. */ -export const PIPELINED_FEE_PADDING = 30; +export const PIPELINED_FEE_PADDING = TEST_FEE_PADDING; /** * Setup option preset that opts a test into proposer pipelining. Use with `setup()`: @@ -77,7 +79,7 @@ export const AUTOMINE_E2E_OPTS = { minTxsPerBlock: 0, aztecSlotDuration: 12, ethereumSlotDuration: 4, - walletMinFeePadding: PIPELINED_FEE_PADDING, + walletMinFeePadding: TEST_FEE_PADDING, } as const; /** Returns worst-case predicted min fees with padding applied, mirroring the BaseWallet pattern. */ diff --git a/yarn-project/ethereum/scripts/anvil_kill_wrapper.sh b/yarn-project/ethereum/scripts/anvil_kill_wrapper.sh deleted file mode 100755 index 629750efda21..000000000000 --- a/yarn-project/ethereum/scripts/anvil_kill_wrapper.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# Function to get the PPID in macOS -get_ppid_macos() { - ps -j $$ | awk 'NR==2 {print $3}' -} - -# Function to get the PPID in Linux -get_ppid_linux() { - awk '{print $4}' /proc/$$/stat -} - -# Function to check if a process is alive in macOS -is_process_alive_macos() { - ps -p $1 > /dev/null 2>&1 -} - -# Function to check if a process is alive in Linux -is_process_alive_linux() { - [ -d /proc/$1 ] -} - - -# Determine the operating system and call the appropriate function -if [[ "$OSTYPE" == "darwin"* ]]; then - PARENT_PID=$(get_ppid_macos) - check_process_alive() { is_process_alive_macos $1; } -elif [[ "$OSTYPE" == "linux-gnu"* ]]; then - PARENT_PID=$(get_ppid_linux) - check_process_alive() { is_process_alive_linux $1; } -else - echo "Unsupported OS" - exit 1 -fi - -# echo "Parent PID: $PARENT_PID" - -# Start anvil in the background. -RAYON_NUM_THREADS=1 anvil $@ & -CHILD_PID=$! - -cleanup() { - kill $CHILD_PID -} - -trap cleanup EXIT - -# Continuously check if the parent process is still alive. -while check_process_alive $PARENT_PID; do - sleep 1 -done diff --git a/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts b/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts index 71e0189bc071..f62c005c0949 100644 --- a/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts +++ b/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts @@ -22,6 +22,7 @@ import { createExtendedL1Client } from './client.js'; import { type L1ContractsConfig, assertValidSlotDurations } from './config.js'; import { deployMulticall3 } from './contracts/multicall.js'; import { RollupContract } from './contracts/rollup.js'; +import { resolveFoundryBinary } from './foundry_binary.js'; import type { L1ContractAddresses } from './l1_contract_addresses.js'; import type { ExtendedViemWalletClient } from './types.js'; @@ -95,11 +96,16 @@ function runProcess( // Covers an edge where where we may have a cached BlobLib that is not meant for production. // Despite the profile apparently sometimes cached code remains (so says Lasse after his ignition-monorepo arc). -async function maybeForgeForceProductionBuild(l1ContractsPath: string, script: string, chainId: number) { +async function maybeForgeForceProductionBuild( + forgeBin: string, + l1ContractsPath: string, + script: string, + chainId: number, +) { if (chainId === mainnet.id) { logger.info(`Recompiling ${script} with production profile for mainnet deployment`); logger.info('This may take a minute but ensures production BlobLib is used.'); - await runProcess('forge', ['build', script, '--force'], { FOUNDRY_PROFILE: 'production' }, l1ContractsPath); + await runProcess(forgeBin, ['build', script, '--force'], { FOUNDRY_PROFILE: 'production' }, l1ContractsPath); } } @@ -320,8 +326,9 @@ export async function deployAztecL1Contracts( // Use foundry-artifacts from l1-artifacts package const l1ContractsPath = prepareL1ContractsForDeployment(); + const forgeBin = resolveFoundryBinary('forge'); const FORGE_SCRIPT = 'script/deploy/DeployAztecL1Contracts.s.sol'; - await maybeForgeForceProductionBuild(l1ContractsPath, FORGE_SCRIPT, chainId); + await maybeForgeForceProductionBuild(forgeBin, l1ContractsPath, FORGE_SCRIPT, chainId); // Verify contracts on Etherscan when on mainnet/sepolia and ETHERSCAN_API_KEY is available. const isVerifiableChain = chainId === mainnet.id || chainId === sepolia.id; @@ -346,6 +353,8 @@ export async function deployAztecL1Contracts( ...(shouldVerify ? ['--verify'] : []), ]; const forgeEnv = { + // Resolved forge binary picked up by forge_broadcast.js, so it works without forge on PATH. + FORGE_BIN: forgeBin, // Env vars required by l1-contracts/script/deploy/DeploymentConfiguration.sol. NETWORK: getActiveNetworkName(), FOUNDRY_PROFILE: chainId === mainnet.id ? 'production' : undefined, @@ -618,12 +627,15 @@ export const deployRollupForUpgrade = async ( // Use foundry-artifacts from l1-artifacts package const l1ContractsPath = prepareL1ContractsForDeployment(); + const forgeBin = resolveFoundryBinary('forge'); const FORGE_SCRIPT = 'script/deploy/DeployRollupForUpgrade.s.sol'; - await maybeForgeForceProductionBuild(l1ContractsPath, FORGE_SCRIPT, chainId); + await maybeForgeForceProductionBuild(forgeBin, l1ContractsPath, FORGE_SCRIPT, chainId); const scriptPath = join(getL1ContractsPath(), 'scripts', 'forge_broadcast.js'); const forgeArgs = [FORGE_SCRIPT, '--sig', 'run()', '--private-key', privateKey, '--rpc-url', rpcUrl]; const forgeEnv = { + // Resolved forge binary picked up by forge_broadcast.js, so it works without forge on PATH. + FORGE_BIN: forgeBin, FOUNDRY_PROFILE: chainId === mainnet.id ? 'production' : undefined, // Env vars required by l1-contracts/script/deploy/RollupConfiguration.sol. REGISTRY_ADDRESS: registryAddress.toString(), diff --git a/yarn-project/ethereum/src/foundry_binary.ts b/yarn-project/ethereum/src/foundry_binary.ts new file mode 100644 index 000000000000..3e4aeeabf0ac --- /dev/null +++ b/yarn-project/ethereum/src/foundry_binary.ts @@ -0,0 +1,57 @@ +import { spawnSync } from 'child_process'; +import { accessSync, constants } from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; + +function isExecutable(path: string): boolean { + try { + accessSync(path, constants.X_OK); + return true; + } catch { + return false; + } +} + +/** + * Locate a Foundry binary (`anvil`, `forge`, ...) without relying on the caller's PATH. Order: + * 1. `$_BIN` (e.g. `$ANVIL_BIN`, `$FORGE_BIN`) — explicit override, e.g. for CI with a pinned + * version. Throws if set but not pointing at an executable, instead of silently falling back. + * 2. `~/.aztec/current/internal-bin/` — where aztec-up installs it. + * 3. `~/.aztec/current/bin/aztec-` — the publicly-exposed symlink. + * 4. `~/.foundry/bin/` — standalone foundryup install. + * 5. `command -v ` — anything else on PATH. + * + * Throws with a directive message if none work. + */ +export function resolveFoundryBinary(name: string): string { + const envVar = `${name.toUpperCase()}_BIN`; + const envBin = process.env[envVar]; + if (envBin) { + if (!isExecutable(envBin)) { + throw new Error(`$${envVar} is set to ${envBin}, which does not exist or is not executable.`); + } + return envBin; + } + + const candidates = [ + join(homedir(), '.aztec', 'current', 'internal-bin', name), + join(homedir(), '.aztec', 'current', 'bin', `aztec-${name}`), + join(homedir(), '.foundry', 'bin', name), + ]; + for (const path of candidates) { + if (isExecutable(path)) { + return path; + } + } + + const which = spawnSync('sh', ['-c', `command -v ${name}`], { encoding: 'utf8' }); + if (which.status === 0 && which.stdout.trim()) { + return which.stdout.trim(); + } + + throw new Error( + `${name} binary not found. Tried $${envVar}, ~/.aztec/current/internal-bin/${name}, ` + + `~/.aztec/current/bin/aztec-${name}, ~/.foundry/bin/${name}, and $PATH. ` + + `Install via \`aztec-up\` or set ${envVar} to a working binary.`, + ); +} diff --git a/yarn-project/ethereum/src/test/start_anvil.ts b/yarn-project/ethereum/src/test/start_anvil.ts index f5ba8609e15f..c1de2f673409 100644 --- a/yarn-project/ethereum/src/test/start_anvil.ts +++ b/yarn-project/ethereum/src/test/start_anvil.ts @@ -1,10 +1,10 @@ import { createLogger } from '@aztec/foundation/log'; import { makeBackoff, retry } from '@aztec/foundation/retry'; import type { TestDateProvider } from '@aztec/foundation/timer'; -import { fileURLToPath } from '@aztec/foundation/url'; import { type ChildProcess, spawn } from 'child_process'; -import { dirname, resolve } from 'path'; + +import { resolveFoundryBinary } from '../foundry_binary.js'; /** Minimal interface matching the @viem/anvil Anvil shape used by callers. */ export interface Anvil { @@ -14,6 +14,30 @@ export interface Anvil { stop(): Promise; } +// Watchdog wrapper: instead of spawning anvil directly, we spawn a small bash supervisor that runs +// anvil as a background child and polls its own parent (this node process). If the parent dies for +// ANY reason — including SIGKILL / crash / OOM, where node's own exit handlers never run — the poll +// loop ends and the EXIT trap reaps anvil. The script is inlined (rather than shipped as a `.sh`) so +// it works from the published npm tarball too, and the resolved anvil binary is passed via +// `$ANVIL_BIN` so it works without `anvil` on PATH. +// +// `$@` is the anvil argv; `bash -c