chore: Accumulated backports to v5-next (#24497)

BEGIN_COMMIT_OVERRIDE
fix: improve valkeys password handling (#24459)
feat: allow separate valkeys passwords. (#24460)
feat: speed up valkeys generation (#24467)
END_COMMIT_OVERRIDE

Author: AztecBot

docs/docs-operate/operators/keystore/creating-keystores.md

@@ -394,6 +394,28 @@ This creates keystores at:
 - **Directory**: `~/.aztec/keystore/`
 - **Filename**: `key1.json`, `key2.json`, etc. (auto-increments)
 
+### Encrypted Keystore Passwords
+
+Use `--password`, `--password-file`, or `AZTEC_KEYSTORE_PASSWORD` to write encrypted ETH JSON V3 and BLS
+EIP-2335 keystore files with a shared password. To use different passwords, provide role-specific sources such as
+`--eth-password` and `--bls-password`.
+
+```bash
+export AZTEC_KEYSTORE_PASSWORD='your-secure-password'
+aztec validator-keys new \
+  --fee-recipient 0x0000000000000000000000000000000000000000000000000000000000000000 \
+  --count 5 \
+  --publishers 0x7988a4a779f058a0 \
+  --encrypted-keystore-dir ~/.aztec/keystore/encrypted
+```
+
+For each key type, role-specific sources take precedence over shared sources: `--eth-password`,
+`--eth-password-file`, and `AZTEC_ETH_KEYSTORE_PASSWORD` for ETH keystores; `--bls-password`,
+`--bls-password-file`, and `AZTEC_BLS_KEYSTORE_PASSWORD` for BLS keystores. Shared sources use `--password`,
+`--password-file`, then `AZTEC_KEYSTORE_PASSWORD`. Passwords cannot be empty. When the command generates a mnemonic
+while writing encrypted keystores, it does not print the mnemonic to stdout; provide `--mnemonic` if you need deterministic
+recovery.
+
 ## Verifying Your Keystore
 
 Verify the keystore is valid JSON:

docs/docs-operate/operators/keystore/troubleshooting.md

@@ -370,6 +370,13 @@ aztec validator-keys new [options]
 | `--gse-address <address>` | GSE contract address (required with --staker-output) | None |
 | `--l1-rpc-urls <urls>` | L1 RPC endpoints (required with --staker-output) | None |
 | `--legacy` | Use 2.1.4 BLS derivation path (only for regenerating old keys) | `false` |
+| `--password <str>` | Shared password for encrypted ETH and BLS keystore files | `AZTEC_KEYSTORE_PASSWORD` if set |
+| `--password-file <path>` | File containing the shared encrypted-keystore password | None |
+| `--eth-password <str>` | Password for encrypted ETH keystore files | `AZTEC_ETH_KEYSTORE_PASSWORD`, then shared password |
+| `--eth-password-file <path>` | File containing the ETH encrypted-keystore password | None |
+| `--bls-password <str>` | Password for encrypted BLS keystore files | `AZTEC_BLS_KEYSTORE_PASSWORD`, then shared password |
+| `--bls-password-file <path>` | File containing the BLS encrypted-keystore password | None |
+| `--encrypted-keystore-dir <path>` | Directory for encrypted ETH and BLS keystore files | Keystore output directory |
 | `--data-dir <path>` | Output directory | `~/.aztec/keystore` |
 | `--file <name>` | Keystore filename | `key1.json` |
 

yarn-project/cli/src/cmds/validator_keys/eth_json_v3_worker.ts

@@ -0,0 +1,30 @@
+import { Wallet } from '@ethersproject/wallet';
+import { parentPort, workerData } from 'worker_threads';
+
+type EthJsonV3WorkerData = {
+  privateKeyHex: string;
+  password: string;
+};
+
+type SerializedError = {
+  message: string;
+  name?: string;
+  stack?: string;
+};
+
+function serializeError(error: unknown): SerializedError {
+  if (error instanceof Error) {
+    return { message: error.message, name: error.name, stack: error.stack };
+  }
+  return { message: String(error) };
+}
+
+void (async () => {
+  try {
+    const { privateKeyHex, password } = workerData as EthJsonV3WorkerData;
+    const json = await new Wallet(privateKeyHex).encrypt(password);
+    parentPort?.postMessage({ json });
+  } catch (error) {
+    parentPort?.postMessage({ error: serializeError(error) });
+  }
+})();

yarn-project/cli/src/cmds/validator_keys/index.ts

@@ -1,6 +1,6 @@
 import type { LogFn } from '@aztec/foundation/log';
 
-import { Command } from 'commander';
+import { Command, Option } from 'commander';
 
 import { parseAztecAddress, parseEthereumAddress, parseHex, parseOptionalInteger } from '../../utils/commands.js';
 import { defaultBlsPath } from './utils.js';
@@ -38,10 +38,24 @@ export function injectCommands(program: Command, log: LogFn) {
     .option('--remote-signer <url>', 'Default remote signer URL for accounts in this file')
     .option('--ikm <hex>', 'Initial keying material for BLS (alternative to mnemonic)', value => parseHex(value, 32))
     .option('--bls-path <path>', `EIP-2334 path (default ${defaultBlsPath})`)
-    .option(
-      '--password <str>',
-      'Password for writing keystore files (ETH JSON V3 and BLS EIP-2335). Empty string allowed',
+    .addOption(
+      new Option('--password <str>', 'Shared password for writing ETH JSON V3 and BLS EIP-2335 keystore files').env(
+        'AZTEC_KEYSTORE_PASSWORD',
+      ),
+    )
+    .option('--password-file <path>', 'File containing the shared password for writing keystore files')
+    .addOption(
+      new Option('--eth-password <str>', 'Password for writing ETH JSON V3 keystore files').env(
+        'AZTEC_ETH_KEYSTORE_PASSWORD',
+      ),
+    )
+    .option('--eth-password-file <path>', 'File containing the password for writing ETH JSON V3 keystore files')
+    .addOption(
+      new Option('--bls-password <str>', 'Password for writing BLS EIP-2335 keystore files').env(
+        'AZTEC_BLS_KEYSTORE_PASSWORD',
+      ),
     )
+    .option('--bls-password-file <path>', 'File containing the password for writing BLS EIP-2335 keystore files')
     .option('--encrypted-keystore-dir <dir>', 'Output directory for encrypted keystore file(s)')
     .option('--json', 'Echo resulting JSON to stdout')
     .option('--staker-output', 'Generate a single staker output JSON file with an array of validator entries')

yarn-project/cli/src/cmds/validator_keys/new.ts

@@ -6,7 +6,7 @@ import type { LogFn } from '@aztec/foundation/log';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 
 import { wordlist } from '@scure/bip39/wordlists/english.js';
-import { writeFile } from 'fs/promises';
+import { readFile, writeFile } from 'fs/promises';
 import { basename, dirname, join } from 'path';
 import { createPublicClient, fallback, http } from 'viem';
 import { generateMnemonic, mnemonicToAccount } from 'viem/accounts';
@@ -42,6 +42,11 @@ export type NewValidatorKeystoreOptions = {
   ikm?: string;
   blsPath?: string;
   password?: string;
+  passwordFile?: string;
+  ethPassword?: string;
+  ethPasswordFile?: string;
+  blsPassword?: string;
+  blsPasswordFile?: string;
   encryptedKeystoreDir?: string;
   json?: boolean;
   feeRecipient: AztecAddress;
@@ -53,6 +58,86 @@ export type NewValidatorKeystoreOptions = {
   l1ChainId?: number;
 };
 
+type PasswordSourceOptions = Pick<
+  NewValidatorKeystoreOptions,
+  'password' | 'passwordFile' | 'ethPassword' | 'ethPasswordFile' | 'blsPassword' | 'blsPasswordFile'
+>;
+
+type PasswordSource = {
+  password?: string;
+  passwordFile?: string;
+  passwordOption: string;
+  passwordFileOption: string;
+};
+
+function validatePassword(password: string, source: string): string {
+  if (password.length === 0) {
+    throw new Error(`${source} cannot be empty`);
+  }
+  return password;
+}
+
+function stripOneTrailingNewline(password: string): string {
+  if (password.endsWith('\n')) {
+    password = password.slice(0, -1);
+  }
+  if (password.endsWith('\r')) {
+    password = password.slice(0, -1);
+  }
+  return password;
+}
+
+async function resolvePasswordSource(source: PasswordSource): Promise<string | undefined> {
+  if (source.password !== undefined && source.passwordFile !== undefined) {
+    throw new Error(`${source.passwordOption} and ${source.passwordFileOption} cannot be used together`);
+  }
+  if (source.password !== undefined) {
+    return validatePassword(source.password, source.passwordOption);
+  }
+  if (source.passwordFile !== undefined) {
+    if (source.passwordFile.length === 0) {
+      throw new Error(`${source.passwordFileOption} cannot be empty`);
+    }
+    const password = stripOneTrailingNewline(await readFile(source.passwordFile, 'utf-8'));
+    return validatePassword(password, source.passwordFileOption);
+  }
+
+  return undefined;
+}
+
+async function resolvePasswords(options: PasswordSourceOptions) {
+  const sharedPassword = await resolvePasswordSource({
+    password: options.password,
+    passwordFile: options.passwordFile,
+    passwordOption: '--password',
+    passwordFileOption: '--password-file',
+  });
+  const ethPassword =
+    (await resolvePasswordSource({
+      password: options.ethPassword,
+      passwordFile: options.ethPasswordFile,
+      passwordOption: '--eth-password',
+      passwordFileOption: '--eth-password-file',
+    })) ?? sharedPassword;
+  const blsPassword =
+    (await resolvePasswordSource({
+      password: options.blsPassword,
+      passwordFile: options.blsPasswordFile,
+      passwordOption: '--bls-password',
+      passwordFileOption: '--bls-password-file',
+    })) ?? sharedPassword;
+
+  if (ethPassword === undefined && blsPassword === undefined) {
+    return {};
+  }
+  if (ethPassword === undefined || blsPassword === undefined) {
+    throw new Error(
+      'Both ETH and BLS passwords are required when writing encrypted keystores. Provide --password to use one password for both, or provide both --eth-password and --bls-password.',
+    );
+  }
+  return { ethPassword, blsPassword };
+}
+
 export async function newValidatorKeystore(options: NewValidatorKeystoreOptions, log: LogFn) {
   // validate bls-path inputs before proceeding with key generation
   validateBlsPathOptions(options);
@@ -78,17 +163,31 @@ export async function newValidatorKeystore(options: NewValidatorKeystoreOptions,
     blsPath,
     ikm,
     mnemonic: _mnemonic,
-    password,
+    password: passwordOption,
+    passwordFile,
+    ethPassword: ethPasswordOption,
+    ethPasswordFile,
+    blsPassword: blsPasswordOption,
+    blsPasswordFile,
     encryptedKeystoreDir,
     stakerOutput,
     gseAddress,
     l1RpcUrls,
     l1ChainId,
   } = options;
 
+  const { ethPassword, blsPassword } = await resolvePasswords({
+    password: passwordOption,
+    passwordFile,
+    ethPassword: ethPasswordOption,
+    ethPasswordFile,
+    blsPassword: blsPasswordOption,
+    blsPasswordFile,
+  });
+  const shouldEncryptKeystores = ethPassword !== undefined && blsPassword !== undefined;
   const mnemonic = _mnemonic ?? generateMnemonic(wordlist);
 
-  if (!_mnemonic && !json) {
+  if (!_mnemonic && !json && !shouldEncryptKeystores) {
     log('No mnemonic provided, generating new one...');
     log(`Using new mnemonic:`);
     log('');
@@ -115,11 +214,11 @@ export async function newValidatorKeystore(options: NewValidatorKeystoreOptions,
   });
 
   // If password provided, write ETH JSON V3 and BLS BN254 keystores and replace plaintext
-  if (password !== undefined) {
+  if (shouldEncryptKeystores) {
     const encryptedKeystoreOutDir =
       encryptedKeystoreDir && encryptedKeystoreDir.length > 0 ? encryptedKeystoreDir : keystoreOutDir;
-    await writeEthJsonV3ToFile(validators, { outDir: encryptedKeystoreOutDir, password });
-    await writeBlsBn254ToFile(validators, { outDir: encryptedKeystoreOutDir, password });
+    await writeEthJsonV3ToFile(validators, { outDir: encryptedKeystoreOutDir, password: ethPassword });
+    await writeBlsBn254ToFile(validators, { outDir: encryptedKeystoreOutDir, password: blsPassword });
   }
 
   const keystore = {
@@ -145,7 +244,7 @@ export async function newValidatorKeystore(options: NewValidatorKeystoreOptions,
     // Process each validator
     for (let i = 0; i < validators.length; i++) {
       const validator = validators[i];
-      const outputs = await processAttesterAccounts(validator.attester, gse, password);
+      const outputs = await processAttesterAccounts(validator.attester, gse);
 
       // Collect all staker outputs
       for (let j = 0; j < outputs.length; j++) {
@@ -160,7 +259,7 @@ export async function newValidatorKeystore(options: NewValidatorKeystoreOptions,
     }
   }
 
-  const outputData = !_mnemonic ? { ...keystore, generatedMnemonic: mnemonic } : keystore;
+  const outputData = !_mnemonic && !shouldEncryptKeystores ? { ...keystore, generatedMnemonic: mnemonic } : keystore;
 
   // Handle JSON output
   if (json) {