Skip to content

feat: speed up valkeys generation #24461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alexghr wants to merge 1 commit into from
Closed

feat: speed up valkeys generation #24461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 44 additions & 44 deletions yarn-project/cli/src/cmds/validator_keys/shared.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ export async function writeBn254BlsKeystore(
): Promise<string> {
mkdirSync(outDir, { recursive: true });

const keystore = createBn254Keystore(password, privateKeyHex, pubkeyHex, derivationPath);
const keystore = await createBn254Keystore(password, privateKeyHex, pubkeyHex, derivationPath);

const safeBase = fileNameBase.replace(/[^a-zA-Z0-9_-]/g, '_');
const outPath = join(outDir, `keystore-${safeBase}.json`);
Expand All @@ -241,28 +241,29 @@ export async function writeBlsBn254ToFile(
validators: ValidatorKeyStore[],
options: { outDir: string; password: string; blsPath?: string },
): Promise<void> {
for (let i = 0; i < validators.length; i++) {
const v = validators[i];
if (!v || typeof v !== 'object' || !('attester' in v)) {
continue;
}
const att = (v as any).attester;
await Promise.all(
validators.map(async (v, i) => {
if (!v || typeof v !== 'object' || !('attester' in v)) {
return;
}
const att = (v as any).attester;

// Shapes: { bls: <hex> } or { eth: <ethAccount>, bls?: <hex> } or plain EthAccount
const blsKey: string | undefined = typeof att === 'object' && 'bls' in att ? (att as any).bls : undefined;
if (!blsKey || typeof blsKey !== 'string') {
continue;
}
// Shapes: { bls: <hex> } or { eth: <ethAccount>, bls?: <hex> } or plain EthAccount
const blsKey: string | undefined = typeof att === 'object' && 'bls' in att ? (att as any).bls : undefined;
if (!blsKey || typeof blsKey !== 'string') {
return;
}

const pub = await computeBlsPublicKeyCompressed(blsKey);
const path = options.blsPath ?? defaultBlsPath;
const fileBase = `${String(i + 1)}_${pub.slice(2, 18)}`;
const keystorePath = await writeBn254BlsKeystore(options.outDir, fileBase, options.password, blsKey, pub, path);
const pub = await computeBlsPublicKeyCompressed(blsKey);
const path = options.blsPath ?? defaultBlsPath;
const fileBase = `${String(i + 1)}_${pub.slice(2, 18)}`;
const keystorePath = await writeBn254BlsKeystore(options.outDir, fileBase, options.password, blsKey, pub, path);

if (typeof att === 'object') {
(att as any).bls = { path: keystorePath, password: options.password };
}
}
if (typeof att === 'object') {
(att as any).bls = { path: keystorePath, password: options.password };
}
}),
);
}

/** Writes an Ethereum JSON V3 keystore using ethers, returns absolute path */
Expand Down Expand Up @@ -295,32 +296,31 @@ export async function writeEthJsonV3ToFile(
return account;
};

for (let i = 0; i < validators.length; i++) {
const v = validators[i];
if (!v || typeof v !== 'object') {
continue;
}
await Promise.all(
validators.map(async (v, i) => {
if (!v || typeof v !== 'object') {
return;
}

// attester may be string (eth), object with eth, or remote signer
const att = (v as any).attester;
if (typeof att === 'string') {
(v as any).attester = await maybeEncryptEth(att, `attester_${i + 1}`);
} else if (att && typeof att === 'object' && 'eth' in att) {
(att as any).eth = await maybeEncryptEth((att as any).eth, `attester_${i + 1}`);
}
// attester may be string (eth), object with eth, or remote signer
const att = (v as any).attester;
if (typeof att === 'string') {
(v as any).attester = await maybeEncryptEth(att, `attester_${i + 1}`);
} else if (att && typeof att === 'object' && 'eth' in att) {
(att as any).eth = await maybeEncryptEth((att as any).eth, `attester_${i + 1}`);
}

// publisher can be single or array
if ('publisher' in v) {
const pub = (v as any).publisher;
if (Array.isArray(pub)) {
const out: any[] = [];
for (let j = 0; j < pub.length; j++) {
out.push(await maybeEncryptEth(pub[j], `publisher_${i + 1}_${j + 1}`));
// publisher can be single or array
if ('publisher' in v) {
const pub = (v as any).publisher;
if (Array.isArray(pub)) {
(v as any).publisher = await Promise.all(
pub.map((account, j) => maybeEncryptEth(account, `publisher_${i + 1}_${j + 1}`)),
);
} else if (pub !== undefined) {
(v as any).publisher = await maybeEncryptEth(pub, `publisher_${i + 1}`);
}
(v as any).publisher = out;
} else if (pub !== undefined) {
(v as any).publisher = await maybeEncryptEth(pub, `publisher_${i + 1}`);
}
}
}
}),
);
}
88 changes: 44 additions & 44 deletions yarn-project/foundation/src/crypto/bls/bn254_keystore.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ describe('BN254 Keystore', () => {
});

describe('createBn254Keystore', () => {
it('creates a valid BN254 keystore structure', () => {
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('creates a valid BN254 keystore structure', async () => {
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);

expect(keystore.version).toBe(4);
expect(keystore.path).toBe(testPath);
Expand All @@ -50,8 +50,8 @@ describe('BN254 Keystore', () => {
expect(keystore.crypto.checksum.message).toMatch(/^[0-9a-f]{64}$/);
});

it('encrypts the private key so it can be decrypted', () => {
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('encrypts the private key so it can be decrypted', async () => {
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);

// Derive the decryption key using the same KDF
const salt = Buffer.from(keystore.crypto.kdf.params.salt, 'hex');
Expand All @@ -74,87 +74,87 @@ describe('BN254 Keystore', () => {
expect('0x' + decrypted.toString('hex')).toBe(testPrivateKey);
});

it('produces different ciphertexts for the same key with different passwords', () => {
const keystore1 = createBn254Keystore('password1', testPrivateKey, testPublicKey, testPath);
const keystore2 = createBn254Keystore('password2', testPrivateKey, testPublicKey, testPath);
it('produces different ciphertexts for the same key with different passwords', async () => {
const keystore1 = await createBn254Keystore('password1', testPrivateKey, testPublicKey, testPath);
const keystore2 = await createBn254Keystore('password2', testPrivateKey, testPublicKey, testPath);

expect(keystore1.crypto.cipher.message).not.toBe(keystore2.crypto.cipher.message);
expect(keystore1.crypto.checksum.message).not.toBe(keystore2.crypto.checksum.message);
});

it('produces different ciphertexts on each call due to random salt and IV', () => {
const keystore1 = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
const keystore2 = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('produces different ciphertexts on each call due to random salt and IV', async () => {
const keystore1 = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
const keystore2 = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);

expect(keystore1.crypto.kdf.params.salt).not.toBe(keystore2.crypto.kdf.params.salt);
expect(keystore1.crypto.cipher.params.iv).not.toBe(keystore2.crypto.cipher.params.iv);
expect(keystore1.crypto.cipher.message).not.toBe(keystore2.crypto.cipher.message);
});

it('accepts private keys with or without 0x prefix', () => {
const withPrefix = createBn254Keystore(testPassword, '0x' + '42'.repeat(32), testPublicKey, testPath);
const withoutPrefix = createBn254Keystore(testPassword, '42'.repeat(32), testPublicKey, testPath);
it('accepts private keys with or without 0x prefix', async () => {
const withPrefix = await createBn254Keystore(testPassword, '0x' + '42'.repeat(32), testPublicKey, testPath);
const withoutPrefix = await createBn254Keystore(testPassword, '42'.repeat(32), testPublicKey, testPath);

// Both should produce valid keystores with same length ciphertext
expect(withPrefix.crypto.cipher.message.length).toBe(64);
expect(withoutPrefix.crypto.cipher.message.length).toBe(64);
});

it('stores public key without 0x prefix in description field', () => {
it('stores public key without 0x prefix in description field', async () => {
const pubkeyWithPrefix = '0x' + 'ab'.repeat(33);
const keystore = createBn254Keystore(testPassword, testPrivateKey, pubkeyWithPrefix, testPath);
const keystore = await createBn254Keystore(testPassword, testPrivateKey, pubkeyWithPrefix, testPath);

expect(keystore.description).toBe('ab'.repeat(33));
expect(keystore.pubkey).toBe(pubkeyWithPrefix);
});

it('throws error for invalid private key length', () => {
it('throws error for invalid private key length', async () => {
const shortKey = '0x1234';
expect(() => createBn254Keystore(testPassword, shortKey, testPublicKey, testPath)).toThrow(
await expect(createBn254Keystore(testPassword, shortKey, testPublicKey, testPath)).rejects.toThrow(
'BLS private key must be 32-byte hex',
);

const longKey = '0x' + '42'.repeat(33);
expect(() => createBn254Keystore(testPassword, longKey, testPublicKey, testPath)).toThrow(
await expect(createBn254Keystore(testPassword, longKey, testPublicKey, testPath)).rejects.toThrow(
'BLS private key must be 32-byte hex',
);
});

it('throws error for non-hex private key', () => {
it('throws error for non-hex private key', async () => {
const invalidKey = '0xGGGG' + '42'.repeat(30);
expect(() => createBn254Keystore(testPassword, invalidKey, testPublicKey, testPath)).toThrow(
await expect(createBn254Keystore(testPassword, invalidKey, testPublicKey, testPath)).rejects.toThrow(
'BLS private key must be 32-byte hex',
);
});

it('normalizes password using NFKD', () => {
it('normalizes password using NFKD', async () => {
// Password with combining characters
const password = 'café'; // é can be represented as single char or e + combining accent
const keystore = createBn254Keystore(password, testPrivateKey, testPublicKey, testPath);
const keystore = await createBn254Keystore(password, testPrivateKey, testPublicKey, testPath);

// Should successfully create keystore (normalization happens internally)
expect(keystore).toBeDefined();
expect(keystore.version).toBe(4);
});

it('handles empty password', () => {
const keystore = createBn254Keystore('', testPrivateKey, testPublicKey, testPath);
it('handles empty password', async () => {
const keystore = await createBn254Keystore('', testPrivateKey, testPublicKey, testPath);

expect(keystore).toBeDefined();
expect(keystore.crypto.cipher.message).toMatch(/^[0-9a-f]{64}$/);
});

it('preserves derivation path exactly as provided', () => {
it('preserves derivation path exactly as provided', async () => {
const customPath = 'm/12381/3600/99/88/77';
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, customPath);
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, customPath);

expect(keystore.path).toBe(customPath);
});
});

describe('loadBn254Keystore', () => {
it('loads and validates a valid BN254 keystore file', () => {
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('loads and validates a valid BN254 keystore file', async () => {
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
const filePath = join(tempDir, 'valid-keystore.json');
writeFileSync(filePath, JSON.stringify(keystore));

Expand Down Expand Up @@ -191,8 +191,8 @@ describe('BN254 Keystore', () => {
});

describe('decryptBn254Keystore', () => {
it('successfully decrypts a keystore with correct password', () => {
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('successfully decrypts a keystore with correct password', async () => {
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
const filePath = join(tempDir, 'decrypt-test.json');
writeFileSync(filePath, JSON.stringify(keystore));

Expand All @@ -201,18 +201,18 @@ describe('BN254 Keystore', () => {
expect(decrypted).toBe(testPrivateKey);
});

it('throws on incorrect password', () => {
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('throws on incorrect password', async () => {
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
const filePath = join(tempDir, 'wrong-password-test.json');
writeFileSync(filePath, JSON.stringify(keystore));

expect(() => decryptBn254Keystore(filePath, 'wrong-password')).toThrow(Bn254KeystoreError);
expect(() => decryptBn254Keystore(filePath, 'wrong-password')).toThrow(/Checksum verification failed/);
});

it('works with empty password if keystore was created with empty password', () => {
it('works with empty password if keystore was created with empty password', async () => {
const emptyPassword = '';
const keystore = createBn254Keystore(emptyPassword, testPrivateKey, testPublicKey, testPath);
const keystore = await createBn254Keystore(emptyPassword, testPrivateKey, testPublicKey, testPath);
const filePath = join(tempDir, 'empty-password-test.json');
writeFileSync(filePath, JSON.stringify(keystore));

Expand All @@ -223,25 +223,25 @@ describe('BN254 Keystore', () => {
});

describe('decryptBn254KeystoreFromObject', () => {
it('decrypts from an in-memory keystore object', () => {
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('decrypts from an in-memory keystore object', async () => {
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);

const decrypted = decryptBn254KeystoreFromObject(keystore, testPassword);

expect(decrypted).toBe(testPrivateKey);
});

it('throws on unsupported KDF function', () => {
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('throws on unsupported KDF function', async () => {
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
// Manually modify to unsupported KDF
(keystore.crypto.kdf as any).function = 'scrypt';

expect(() => decryptBn254KeystoreFromObject(keystore, testPassword)).toThrow(Bn254KeystoreError);
expect(() => decryptBn254KeystoreFromObject(keystore, testPassword)).toThrow(/Unsupported KDF function/);
});

it('throws on unsupported cipher function', () => {
const keystore = createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
it('throws on unsupported cipher function', async () => {
const keystore = await createBn254Keystore(testPassword, testPrivateKey, testPublicKey, testPath);
// Manually modify to unsupported cipher
(keystore.crypto.cipher as any).function = 'aes-256-gcm';

Expand All @@ -251,30 +251,30 @@ describe('BN254 Keystore', () => {
});

describe('round-trip encryption and decryption', () => {
it('encrypts and then decrypts to get original key', () => {
it('encrypts and then decrypts to get original key', async () => {
const originalKey = '0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef';
const password = 'super-secret';
const pubkey = '0x' + 'ff'.repeat(33);
const path = 'm/12381/3600/5/0/0';

// Create encrypted keystore
const encrypted = createBn254Keystore(password, originalKey, pubkey, path);
const encrypted = await createBn254Keystore(password, originalKey, pubkey, path);

// Decrypt it
const decrypted = decryptBn254KeystoreFromObject(encrypted, password);

expect(decrypted).toBe(originalKey);
});

it('round-trips multiple different keys', () => {
it('round-trips multiple different keys', async () => {
const testCases = [
{ key: '0x' + '11'.repeat(32), password: 'pass1' },
{ key: '0x' + '22'.repeat(32), password: 'pass2' },
{ key: '0x' + 'ab'.repeat(32), password: 'pass3' },
];

for (const { key, password } of testCases) {
const encrypted = createBn254Keystore(password, key, testPublicKey, testPath);
const encrypted = await createBn254Keystore(password, key, testPublicKey, testPath);
const decrypted = decryptBn254KeystoreFromObject(encrypted, password);
expect(decrypted).toBe(key);
}
Expand Down
Loading
Loading