From 63908d176ad3ce6ded3e0b8a794b2605e2e247e4 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Tue, 3 Jun 2025 12:29:09 +0900 Subject: [PATCH 01/11] perform export in transaction --- .../source/snapshot/generator.ts | 162 +++++++++--------- 1 file changed, 85 insertions(+), 77 deletions(-) diff --git a/packages/snapshot-legacy-exporter/source/snapshot/generator.ts b/packages/snapshot-legacy-exporter/source/snapshot/generator.ts index c05c55c7ec..636fa8dd97 100644 --- a/packages/snapshot-legacy-exporter/source/snapshot/generator.ts +++ b/packages/snapshot-legacy-exporter/source/snapshot/generator.ts @@ -7,7 +7,7 @@ import { inject, injectable } from "@mainsail/container"; import { Contracts, Identifiers } from "@mainsail/contracts"; import { Application, Providers } from "@mainsail/kernel"; import { assert } from "@mainsail/utils"; -import { DataSource } from "typeorm"; +import { DataSource, EntityManager } from "typeorm"; import { PostgresConnectionOptions } from "typeorm/driver/postgres/PostgresConnectionOptions.js"; import { Identifiers as InternalIdentifiers } from "../identifiers.js"; @@ -15,6 +15,8 @@ import { LegacyChainTip, LegacySnapshot, LegacyWallet } from "../interfaces.js"; interface DatabaseOptions extends PostgresConnectionOptions { readonly v3: { + readonly host: string; + readonly port: number; readonly user: string; readonly password: string; readonly database: string; @@ -26,33 +28,6 @@ export class Generator { @inject(InternalIdentifiers.Application) private app!: Application; - async #connect(): Promise { - const pluginConfig = await this.app - .resolve(Providers.PluginConfiguration) - .discover("@mainsail/snapshot-legacy-exporter", process.cwd()); - - const options = pluginConfig.get("database"); - assert.defined(options); - - const dataSource = new DataSource({ - ...options, - entities: [], - migrations: [], - migrationsRun: false, - synchronize: false, - }); - - await dataSource.initialize(); - - // Link v3 database - await dataSource.query(` - CREATE EXTENSION IF NOT EXISTS dblink; - SELECT dblink_connect('v3_db', 'dbname=${options.v3.database} user=${options.v3.user} password=${options.v3.password}'); - `); - - return dataSource; - } - public async generateStatic(chainTip: LegacyChainTip, wallets: LegacyWallet[]): Promise { const addressFactory = this.app.get( Identifiers.Cryptography.Identity.Address.Factory, @@ -75,64 +50,97 @@ export class Generator { } public async generate(): Promise { - // Connect to V3 database - const dataSource = await this.#connect(); - console.log("connected!"); - - const [chainTip] = await dataSource.query( - "SELECT * FROM dblink('v3_db', 'SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1') AS blocks(hash varchar, number bigint);", - ); + await this.#runInTransaction(async (entityManager) => { + console.log("connected!"); + + const [chainTip] = await entityManager.query( + "SELECT * FROM dblink('v3_db', 'SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1') AS blocks(hash varchar, number bigint);", + ); + + const addressFactory = this.app.get( + Identifiers.Cryptography.Identity.Address.Factory, + ); + + // Loop all wallets + const limit = 1000; + let offset = 0; + + const wallets: LegacyWallet[] = []; + for (;;) { + console.log(`Fetching wallets (offset: ${offset}, limit: ${limit})`); + + const chunk: LegacyWallet[] = await entityManager.query(` + SELECT * FROM dblink( + 'v3_db', + ' + SELECT address, public_key, balance, attributes FROM wallets + ORDER BY balance DESC, address ASC LIMIT ${limit} OFFSET ${offset} + ' + ) AS wallets("arkAddress" varchar, "publicKey" varchar, balance bigint, attributes jsonb); + `); + + for (const wallet of chunk) { + // sanitize + if (wallet.attributes?.["delegate"]) { + delete wallet.attributes?.["delegate"]["lastBlock"]; + } + + delete wallet.attributes?.["ipfs"]; // ? + delete wallet.attributes?.["business"]; // ? + delete wallet.attributes?.["htlc"]; // ? + delete wallet.attributes?.["entities"]; // ? + + wallets.push({ + ...wallet, + ...(wallet.publicKey + ? { + ethAddress: await addressFactory.fromPublicKey(wallet.publicKey), + } + : {}), + }); + } - const addressFactory = this.app.get( - Identifiers.Cryptography.Identity.Address.Factory, - ); + offset += limit; - // Loop all wallets - const limit = 1000; - let offset = 0; - - const wallets: LegacyWallet[] = []; - for (;;) { - const chunk: LegacyWallet[] = await dataSource.query(` - SELECT * FROM dblink( - 'v3_db', - ' - SELECT address, public_key, balance, attributes FROM wallets -- WHERE attributes ?| array[''vote'', ''delegate'', ''username''] - ORDER BY balance DESC, address ASC LIMIT ${limit} OFFSET ${offset} - ' - ) AS wallets("arkAddress" varchar, "publicKey" varchar, balance bigint, attributes jsonb); - `); - - for (const wallet of chunk) { - // sanitize - if (wallet.attributes?.["delegate"]) { - delete wallet.attributes?.["delegate"]["lastBlock"]; + if (chunk.length === 0) { + break; } + } - delete wallet.attributes?.["ipfs"]; // ? - delete wallet.attributes?.["business"]; // ? - delete wallet.attributes?.["htlc"]; // ? - delete wallet.attributes?.["entities"]; // ? + await this.#writeSnapshot(chainTip, wallets); + }); + } - wallets.push({ - ...wallet, - ...(wallet.publicKey - ? { - ethAddress: await addressFactory.fromPublicKey(wallet.publicKey), - } - : {}), - }); - } + async #runInTransaction(callback: (entityManager: EntityManager) => Promise): Promise { + const pluginConfig = await this.app + .resolve(Providers.PluginConfiguration) + .discover("@mainsail/snapshot-legacy-exporter", process.cwd()); - offset += limit; + const options = pluginConfig.get("database"); + assert.defined(options); - if (chunk.length === 0) { - break; - } - } + const dataSource = new DataSource({ + ...options, + entities: [], + migrations: [], + migrationsRun: false, + synchronize: false, + }); + + await dataSource.initialize(); try { - await this.#writeSnapshot(chainTip, wallets); + await dataSource.transaction("REPEATABLE READ", async (entityManager) => { + // Link v3 database + await entityManager.query(` + CREATE EXTENSION IF NOT EXISTS dblink; + SELECT dblink_connect('v3_db', 'host=${options.v3.host} port=${options.v3.port} dbname=${options.v3.database} user=${options.v3.user} password=${options.v3.password}'); + `); + + await callback(entityManager); + }); + } catch (ex) { + console.log(ex); } finally { await dataSource.destroy(); } From 07316c9a9c227abd909a7c508d10258fcf28cba8 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Tue, 3 Jun 2025 12:32:10 +0900 Subject: [PATCH 02/11] defaults --- packages/snapshot-legacy-exporter/source/defaults.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/snapshot-legacy-exporter/source/defaults.ts b/packages/snapshot-legacy-exporter/source/defaults.ts index 167afabd1f..f956cc3c16 100644 --- a/packages/snapshot-legacy-exporter/source/defaults.ts +++ b/packages/snapshot-legacy-exporter/source/defaults.ts @@ -29,6 +29,10 @@ export const defaults = { Environment.get(Constants.EnvironmentVariables.MAINSAIL_TOKEN), v3: { + // when using podman + // host: "host.containers.internal", + host: "localhost", + port: 5432, database: "ark_devnet", password: "test_db", user: "test_db", From 6e4c0594ebed10b7f9df34241ff9a7f86b96d3cd Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Tue, 3 Jun 2025 12:33:37 +0900 Subject: [PATCH 03/11] more import stats --- .../contracts/source/contracts/snapshot.ts | 5 ++ .../source/importer.ts | 72 ++++++++++++++----- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/packages/contracts/source/contracts/snapshot.ts b/packages/contracts/source/contracts/snapshot.ts index 30c3e7d76e..d8d7520ed3 100644 --- a/packages/contracts/source/contracts/snapshot.ts +++ b/packages/contracts/source/contracts/snapshot.ts @@ -24,6 +24,11 @@ export interface LegacyImportOptions { export interface LegacyImportResult { readonly initialTotalSupply: bigint; + readonly importedValidators: number; + readonly importedUsernames: number; + readonly importedVoters: number; + readonly skippedVoters: number; + readonly skippedValidators: number; } export interface ImportedLegacyWallet { diff --git a/packages/snapshot-legacy-importer/source/importer.ts b/packages/snapshot-legacy-importer/source/importer.ts index 69ab896940..506268ecbb 100644 --- a/packages/snapshot-legacy-importer/source/importer.ts +++ b/packages/snapshot-legacy-importer/source/importer.ts @@ -293,13 +293,13 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { const totalSupply = await this.#seedWallets(options); // 2) Seed validators - await this.#seedValidators(options); + const { importedValidators, skippedValidators } = await this.#seedValidators(options); // 3) Seed voters - await this.#seedVoters(options); + const { importedVoters, skippedVoters } = await this.#seedVoters(options); // 4) Seed usernames - await this.#seedUsernames(options); + const { importedUsernames } = await this.#seedUsernames(options); if (totalSupply !== this.totalSupply) { throw new Error("totalSupply mismatch"); @@ -307,6 +307,11 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { return { initialTotalSupply: totalSupply, + importedValidators, + importedVoters, + skippedVoters, + importedUsernames, + skippedValidators, }; } @@ -348,21 +353,28 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { return totalSupply; } - async #seedValidators(options: Contracts.Snapshot.LegacyImportOptions): Promise { + async #seedValidators( + options: Contracts.Snapshot.LegacyImportOptions, + ): Promise<{ importedValidators: number; skippedValidators: number }> { const iface = new ethers.Interface(ConsensusAbi.abi); this.logger.info(`seeding ${this.#data.validators.length} validators`); + const stats = { + importedValidators: 0, + skippedValidators: 0, + }; + for (const validator of this.#data.validators) { assert.defined(validator.ethAddress); - let blsPublicKey: string | undefined = validator.blsPublicKey; - - if (!blsPublicKey) { + if (!validator.blsPublicKey) { if (!options.mockFakeValidatorBlsKeys) { this.logger.info( `skipping legacy delegate ${validator.arkAddress} (${validator.username}) without registered blsPublicKey`, ); + stats.skippedValidators++; + continue; } @@ -370,22 +382,23 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { const mnemonic = entropyToMnemonic(Buffer.from(entropy, "hex")); const consensusKeyPair = await this.consensusKeyPairFactory.fromMnemonic(mnemonic); - blsPublicKey = consensusKeyPair.publicKey; + validator.blsPublicKey = consensusKeyPair.publicKey; } else { - if (!(await this.consensusPublicKeyFactory.verify(blsPublicKey))) { + if (!(await this.consensusPublicKeyFactory.verify(validator.blsPublicKey))) { this.logger.info( - `skipping legacy delegate ${validator.arkAddress} (${validator.username}) with invalid blsPublicKey ${blsPublicKey}`, + `skipping legacy delegate ${validator.arkAddress} (${validator.username}) with invalid blsPublicKey ${validator.blsPublicKey}`, ); + stats.skippedValidators++; continue; } } - assert.defined(blsPublicKey); + assert.defined(validator.blsPublicKey); const data = iface .encodeFunctionData("addValidator", [ validator.ethAddress, - Buffer.from(blsPublicKey, "hex"), + Buffer.from(validator.blsPublicKey, "hex"), validator.isResigned, ]) .slice(2); @@ -401,16 +414,28 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { if (!result.receipt.status) { throw new Error("failed to add validator"); } + + stats.importedValidators++; } + + return stats; } - async #seedVoters(options: Contracts.Snapshot.LegacyImportOptions): Promise { + async #seedVoters( + options: Contracts.Snapshot.LegacyImportOptions, + ): Promise<{ importedVoters: number; skippedVoters: number }> { const iface = new ethers.Interface(ConsensusAbi.abi); - const validatorLookup = this.#data.validators.reduce((accumulator, current) => { - accumulator[current.ethAddress!] = accumulator; - return accumulator; - }, {}); + const stats = { + importedVoters: 0, + skippedVoters: 0, + }; + + const validatorLookup: Record = + this.#data.validators.reduce((accumulator, current) => { + accumulator[current.ethAddress!] = current; + return accumulator; + }, {}); this.logger.info(`seeding ${this.#data.voters.length} voters`); @@ -446,14 +471,21 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { console.log(result.receipt); throw new Error("failed to add votes"); } + + stats.importedVoters += voterAddresses.length; } + return stats; } - async #seedUsernames(options: Contracts.Snapshot.LegacyImportOptions): Promise { + async #seedUsernames(options: Contracts.Snapshot.LegacyImportOptions): Promise<{ importedUsernames: number }> { const iface = new ethers.Interface(UsernamesAbi.abi); this.logger.info(`seeding ${this.#data.validators.length} usernames`); + const stats = { + importedUsernames: 0, + }; + for (const validator of this.#data.validators) { if (!validator.username) { continue; @@ -472,7 +504,11 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { if (!result.receipt.status) { throw new Error("failed to add username"); } + + stats.importedUsernames += 1; } + + return stats; } #getTransactionContext( From 4f82eb4da224c8cea8c7bb441d2bbe5b85e16a3d Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Tue, 3 Jun 2025 12:33:58 +0900 Subject: [PATCH 04/11] skip voters for missing bls public keys --- .../source/importer.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/snapshot-legacy-importer/source/importer.ts b/packages/snapshot-legacy-importer/source/importer.ts index 506268ecbb..51189e80b8 100644 --- a/packages/snapshot-legacy-importer/source/importer.ts +++ b/packages/snapshot-legacy-importer/source/importer.ts @@ -446,10 +446,22 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { for (const voter of voters) { assert.defined(voter.ethAddress); - if (!validatorLookup[voter.vote]) { - this.logger.warning( + const validator = validatorLookup[voter.vote]; + if (!validator) { + this.logger.debug( `!!! skipping voter ${voter.arkAddress} for non-existent validator: ${voter.vote}`, ); + + stats.skippedVoters++; + continue; + } + + if (!validator.blsPublicKey) { + this.logger.debug( + `!!! skipping voter ${voter.arkAddress} for validator without registered BlsPublicKey: ${validator.arkAddress}`, + ); + + stats.skippedVoters++; continue; } @@ -468,7 +480,7 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { ); if (!result.receipt.status) { - console.log(result.receipt); + console.log(result.receipt, result.receipt.output?.toString("hex")); throw new Error("failed to add votes"); } From 9b74533f92af082bb74f278e5600c5a59c5bbe88 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Tue, 3 Jun 2025 12:34:27 +0900 Subject: [PATCH 05/11] adjust path --- packages/configuration-generator/bin/create-genesis-block.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/configuration-generator/bin/create-genesis-block.js b/packages/configuration-generator/bin/create-genesis-block.js index d7338af440..43bf239d0d 100644 --- a/packages/configuration-generator/bin/create-genesis-block.js +++ b/packages/configuration-generator/bin/create-genesis-block.js @@ -21,7 +21,7 @@ async function run() { chainId: 10000, initialHeight: 0, // snapshot: { - // path: "../../snapshot-19a87c96dbe8ad1be06d33e97cd17f5662eb952c29efd3d8bb00c9c75e7582bc.json", + // path: "../../f07a7068c50e2e5591beaa572070933008744425d727c792d328d1d5e2fac306.compressed", // }, }); From b7788c418e01d82ea6f8b503982fc348d9dd792a Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Tue, 3 Jun 2025 03:40:23 +0000 Subject: [PATCH 06/11] style: resolve style guide violations --- packages/snapshot-legacy-exporter/source/defaults.ts | 4 ++-- packages/snapshot-legacy-importer/source/importer.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/snapshot-legacy-exporter/source/defaults.ts b/packages/snapshot-legacy-exporter/source/defaults.ts index f956cc3c16..7757976b45 100644 --- a/packages/snapshot-legacy-exporter/source/defaults.ts +++ b/packages/snapshot-legacy-exporter/source/defaults.ts @@ -29,12 +29,12 @@ export const defaults = { Environment.get(Constants.EnvironmentVariables.MAINSAIL_TOKEN), v3: { + database: "ark_devnet", // when using podman // host: "host.containers.internal", host: "localhost", - port: 5432, - database: "ark_devnet", password: "test_db", + port: 5432, user: "test_db", }, }, diff --git a/packages/snapshot-legacy-importer/source/importer.ts b/packages/snapshot-legacy-importer/source/importer.ts index 51189e80b8..766fe84ec9 100644 --- a/packages/snapshot-legacy-importer/source/importer.ts +++ b/packages/snapshot-legacy-importer/source/importer.ts @@ -306,12 +306,12 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { } return { - initialTotalSupply: totalSupply, + importedUsernames, importedValidators, importedVoters, - skippedVoters, - importedUsernames, + initialTotalSupply: totalSupply, skippedValidators, + skippedVoters, }; } From deb301ebee016e0d07a0bf067860b6034f72c4e5 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Thu, 5 Jun 2025 12:45:15 +0900 Subject: [PATCH 07/11] refactor --- packages/bootstrap/source/bootstrapper.ts | 8 +++- .../contracts/source/contracts/snapshot.ts | 3 +- .../source/importer.ts | 41 ++++++++++--------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/packages/bootstrap/source/bootstrapper.ts b/packages/bootstrap/source/bootstrapper.ts index 037ea4699e..73c4f3d11c 100644 --- a/packages/bootstrap/source/bootstrapper.ts +++ b/packages/bootstrap/source/bootstrapper.ts @@ -56,6 +56,9 @@ export class Bootstrapper { @inject(Identifiers.Evm.Worker) private readonly evmWorker!: Contracts.Evm.Worker; + @inject(Identifiers.Services.Log.Service) + private readonly logger!: Contracts.Kernel.Logger; + public async bootstrap(): Promise { await this.#setGenesisCommit(); await this.#checkStoredGenesisCommit(); @@ -174,6 +177,9 @@ export class Bootstrapper { throw new Error("snapshot importer not loaded"); } - await this.snapshotImporter.run(genesisBlock); + const result = await this.snapshotImporter.run(genesisBlock); + this.logger.info( + `snapshot import result: ${JSON.stringify({ ...result, initialTotalSupply: result.initialTotalSupply.toString() })}`, + ); } } diff --git a/packages/contracts/source/contracts/snapshot.ts b/packages/contracts/source/contracts/snapshot.ts index d8d7520ed3..842711497a 100644 --- a/packages/contracts/source/contracts/snapshot.ts +++ b/packages/contracts/source/contracts/snapshot.ts @@ -24,7 +24,8 @@ export interface LegacyImportOptions { export interface LegacyImportResult { readonly initialTotalSupply: bigint; - readonly importedValidators: number; + readonly importedValidatorsWithBlsKey: number; + readonly importedValidatorsWithoutBlsKey: number; readonly importedUsernames: number; readonly importedVoters: number; readonly skippedVoters: number; diff --git a/packages/snapshot-legacy-importer/source/importer.ts b/packages/snapshot-legacy-importer/source/importer.ts index 766fe84ec9..bdd0174bb9 100644 --- a/packages/snapshot-legacy-importer/source/importer.ts +++ b/packages/snapshot-legacy-importer/source/importer.ts @@ -293,7 +293,8 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { const totalSupply = await this.#seedWallets(options); // 2) Seed validators - const { importedValidators, skippedValidators } = await this.#seedValidators(options); + const { importedValidatorsWithBlsKey, importedValidatorsWithoutBlsKey, skippedValidators } = + await this.#seedValidators(options); // 3) Seed voters const { importedVoters, skippedVoters } = await this.#seedVoters(options); @@ -307,7 +308,8 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { return { importedUsernames, - importedValidators, + importedValidatorsWithBlsKey, + importedValidatorsWithoutBlsKey, importedVoters, initialTotalSupply: totalSupply, skippedValidators, @@ -353,15 +355,18 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { return totalSupply; } - async #seedValidators( - options: Contracts.Snapshot.LegacyImportOptions, - ): Promise<{ importedValidators: number; skippedValidators: number }> { + async #seedValidators(options: Contracts.Snapshot.LegacyImportOptions): Promise<{ + importedValidatorsWithBlsKey: number; + importedValidatorsWithoutBlsKey: number; + skippedValidators: number; + }> { const iface = new ethers.Interface(ConsensusAbi.abi); this.logger.info(`seeding ${this.#data.validators.length} validators`); const stats = { - importedValidators: 0, + importedValidatorsWithBlsKey: 0, + importedValidatorsWithoutBlsKey: 0, skippedValidators: 0, }; @@ -371,18 +376,16 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { if (!validator.blsPublicKey) { if (!options.mockFakeValidatorBlsKeys) { this.logger.info( - `skipping legacy delegate ${validator.arkAddress} (${validator.username}) without registered blsPublicKey`, + `importing legacy delegate ${validator.arkAddress} (${validator.username}) without registered blsPublicKey`, ); - stats.skippedValidators++; + stats.importedValidatorsWithoutBlsKey++; + } else { + const entropy = sha256(Buffer.from(validator.username, "utf8")).slice(2, 34); + const mnemonic = entropyToMnemonic(Buffer.from(entropy, "hex")); - continue; + const consensusKeyPair = await this.consensusKeyPairFactory.fromMnemonic(mnemonic); + validator.blsPublicKey = consensusKeyPair.publicKey; } - - const entropy = sha256(Buffer.from(validator.username, "utf8")).slice(2, 34); - const mnemonic = entropyToMnemonic(Buffer.from(entropy, "hex")); - - const consensusKeyPair = await this.consensusKeyPairFactory.fromMnemonic(mnemonic); - validator.blsPublicKey = consensusKeyPair.publicKey; } else { if (!(await this.consensusPublicKeyFactory.verify(validator.blsPublicKey))) { this.logger.info( @@ -391,14 +394,14 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { stats.skippedValidators++; continue; } - } - assert.defined(validator.blsPublicKey); + stats.importedValidatorsWithBlsKey++; + } const data = iface .encodeFunctionData("addValidator", [ validator.ethAddress, - Buffer.from(validator.blsPublicKey, "hex"), + validator.blsPublicKey ? Buffer.from(validator.blsPublicKey, "hex") : Buffer.alloc(0), validator.isResigned, ]) .slice(2); @@ -414,8 +417,6 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { if (!result.receipt.status) { throw new Error("failed to add validator"); } - - stats.importedValidators++; } return stats; From 98efc693a65e3ddfe2940de2cb02dd49082223e2 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 6 Jun 2025 09:03:38 +0900 Subject: [PATCH 08/11] use logger --- packages/bootstrap/source/bootstrapper.ts | 8 +------- packages/snapshot-legacy-exporter/package.json | 1 + .../source/application-factory.ts | 2 ++ .../source/snapshot/generator.ts | 11 +++++++---- packages/snapshot-legacy-importer/source/importer.ts | 4 ++++ pnpm-lock.yaml | 3 +++ 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/bootstrap/source/bootstrapper.ts b/packages/bootstrap/source/bootstrapper.ts index 73c4f3d11c..037ea4699e 100644 --- a/packages/bootstrap/source/bootstrapper.ts +++ b/packages/bootstrap/source/bootstrapper.ts @@ -56,9 +56,6 @@ export class Bootstrapper { @inject(Identifiers.Evm.Worker) private readonly evmWorker!: Contracts.Evm.Worker; - @inject(Identifiers.Services.Log.Service) - private readonly logger!: Contracts.Kernel.Logger; - public async bootstrap(): Promise { await this.#setGenesisCommit(); await this.#checkStoredGenesisCommit(); @@ -177,9 +174,6 @@ export class Bootstrapper { throw new Error("snapshot importer not loaded"); } - const result = await this.snapshotImporter.run(genesisBlock); - this.logger.info( - `snapshot import result: ${JSON.stringify({ ...result, initialTotalSupply: result.initialTotalSupply.toString() })}`, - ); + await this.snapshotImporter.run(genesisBlock); } } diff --git a/packages/snapshot-legacy-exporter/package.json b/packages/snapshot-legacy-exporter/package.json index a647fb5490..837886f2d6 100644 --- a/packages/snapshot-legacy-exporter/package.json +++ b/packages/snapshot-legacy-exporter/package.json @@ -28,6 +28,7 @@ "@mainsail/crypto-key-pair-ecdsa": "workspace:*", "@mainsail/crypto-validation": "workspace:*", "@mainsail/kernel": "workspace:*", + "@mainsail/logger-pino": "workspace:*", "@mainsail/utils": "workspace:*", "@mainsail/validation": "workspace:*", "pg": "8.14.0", diff --git a/packages/snapshot-legacy-exporter/source/application-factory.ts b/packages/snapshot-legacy-exporter/source/application-factory.ts index 67713d9043..45fe56539d 100644 --- a/packages/snapshot-legacy-exporter/source/application-factory.ts +++ b/packages/snapshot-legacy-exporter/source/application-factory.ts @@ -3,6 +3,7 @@ import { Identifiers } from "@mainsail/contracts"; import { ServiceProvider as CryptoAddressKeccak256 } from "@mainsail/crypto-address-keccak256"; import { ServiceProvider as CryptoKeyPairEcdsa } from "@mainsail/crypto-key-pair-ecdsa"; import { ServiceProvider as CryptoValidation } from "@mainsail/crypto-validation"; +import { ServiceProvider as Logger } from "@mainsail/logger-pino"; import { Application } from "@mainsail/kernel"; import { ServiceProvider as Validation } from "@mainsail/validation"; import { dirSync, setGracefulCleanup } from "tmp"; @@ -29,6 +30,7 @@ export const makeApplication = async (configurationPath: string, options: Record await app.resolve(CryptoValidation).register(); await app.resolve(CryptoKeyPairEcdsa).register(); await app.resolve(CryptoAddressKeccak256).register(); + await app.resolve(Logger).register(); // app.bind(InternalIdentifiers.Snapshot.Generator).to(Generator); diff --git a/packages/snapshot-legacy-exporter/source/snapshot/generator.ts b/packages/snapshot-legacy-exporter/source/snapshot/generator.ts index 636fa8dd97..d70d723864 100644 --- a/packages/snapshot-legacy-exporter/source/snapshot/generator.ts +++ b/packages/snapshot-legacy-exporter/source/snapshot/generator.ts @@ -28,6 +28,9 @@ export class Generator { @inject(InternalIdentifiers.Application) private app!: Application; + @inject(Identifiers.Services.Log.Service) + private readonly logger!: Contracts.Kernel.Logger; + public async generateStatic(chainTip: LegacyChainTip, wallets: LegacyWallet[]): Promise { const addressFactory = this.app.get( Identifiers.Cryptography.Identity.Address.Factory, @@ -51,7 +54,7 @@ export class Generator { public async generate(): Promise { await this.#runInTransaction(async (entityManager) => { - console.log("connected!"); + this.logger.info("connected!"); const [chainTip] = await entityManager.query( "SELECT * FROM dblink('v3_db', 'SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1') AS blocks(hash varchar, number bigint);", @@ -67,7 +70,7 @@ export class Generator { const wallets: LegacyWallet[] = []; for (;;) { - console.log(`Fetching wallets (offset: ${offset}, limit: ${limit})`); + this.logger.info(`Fetching wallets (offset: ${offset}, limit: ${limit})`); const chunk: LegacyWallet[] = await entityManager.query(` SELECT * FROM dblink( @@ -140,7 +143,7 @@ export class Generator { await callback(entityManager); }); } catch (ex) { - console.log(ex); + this.logger.error(ex); } finally { await dataSource.destroy(); } @@ -167,7 +170,7 @@ export class Generator { const jsonString = JSON.stringify(snapshot); const compressedBuffer = await promisify(brotliCompress)(jsonString); await writeFile(path, compressedBuffer); - console.log(`Wrote ${snapshot.wallets.length} wallets to '${path}'`); + this.logger.info(`Wrote ${snapshot.wallets.length} wallets to '${path}'`); } } diff --git a/packages/snapshot-legacy-importer/source/importer.ts b/packages/snapshot-legacy-importer/source/importer.ts index bdd0174bb9..ec4927d531 100644 --- a/packages/snapshot-legacy-importer/source/importer.ts +++ b/packages/snapshot-legacy-importer/source/importer.ts @@ -134,6 +134,10 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { this.#data.result = result; + this.logger.info( + `snapshot import result: ${JSON.stringify({ ...result, initialTotalSupply: result.initialTotalSupply.toString() })}`, + ); + return result; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 33468c6103..71cec0a169 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2374,6 +2374,9 @@ importers: '@mainsail/kernel': specifier: workspace:* version: link:../kernel + '@mainsail/logger-pino': + specifier: workspace:* + version: link:../logger-pino '@mainsail/utils': specifier: workspace:* version: link:../utils From dfcab1c6173eb86fabe0df28beb266b846fc411c Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 6 Jun 2025 09:04:01 +0900 Subject: [PATCH 09/11] dont skip any voters / validators --- .../contracts/source/contracts/snapshot.ts | 2 - .../source/importer.ts | 38 ++++++------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/packages/contracts/source/contracts/snapshot.ts b/packages/contracts/source/contracts/snapshot.ts index 842711497a..32483f301d 100644 --- a/packages/contracts/source/contracts/snapshot.ts +++ b/packages/contracts/source/contracts/snapshot.ts @@ -28,8 +28,6 @@ export interface LegacyImportResult { readonly importedValidatorsWithoutBlsKey: number; readonly importedUsernames: number; readonly importedVoters: number; - readonly skippedVoters: number; - readonly skippedValidators: number; } export interface ImportedLegacyWallet { diff --git a/packages/snapshot-legacy-importer/source/importer.ts b/packages/snapshot-legacy-importer/source/importer.ts index ec4927d531..0188b06d9a 100644 --- a/packages/snapshot-legacy-importer/source/importer.ts +++ b/packages/snapshot-legacy-importer/source/importer.ts @@ -297,11 +297,10 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { const totalSupply = await this.#seedWallets(options); // 2) Seed validators - const { importedValidatorsWithBlsKey, importedValidatorsWithoutBlsKey, skippedValidators } = - await this.#seedValidators(options); + const { importedValidatorsWithBlsKey, importedValidatorsWithoutBlsKey } = await this.#seedValidators(options); // 3) Seed voters - const { importedVoters, skippedVoters } = await this.#seedVoters(options); + const { importedVoters } = await this.#seedVoters(options); // 4) Seed usernames const { importedUsernames } = await this.#seedUsernames(options); @@ -316,8 +315,6 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { importedValidatorsWithoutBlsKey, importedVoters, initialTotalSupply: totalSupply, - skippedValidators, - skippedVoters, }; } @@ -362,7 +359,6 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { async #seedValidators(options: Contracts.Snapshot.LegacyImportOptions): Promise<{ importedValidatorsWithBlsKey: number; importedValidatorsWithoutBlsKey: number; - skippedValidators: number; }> { const iface = new ethers.Interface(ConsensusAbi.abi); @@ -371,7 +367,6 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { const stats = { importedValidatorsWithBlsKey: 0, importedValidatorsWithoutBlsKey: 0, - skippedValidators: 0, }; for (const validator of this.#data.validators) { @@ -379,7 +374,7 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { if (!validator.blsPublicKey) { if (!options.mockFakeValidatorBlsKeys) { - this.logger.info( + this.logger.debug( `importing legacy delegate ${validator.arkAddress} (${validator.username}) without registered blsPublicKey`, ); stats.importedValidatorsWithoutBlsKey++; @@ -391,12 +386,14 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { validator.blsPublicKey = consensusKeyPair.publicKey; } } else { - if (!(await this.consensusPublicKeyFactory.verify(validator.blsPublicKey))) { + if (await this.consensusPublicKeyFactory.verify(validator.blsPublicKey)) { this.logger.info( - `skipping legacy delegate ${validator.arkAddress} (${validator.username}) with invalid blsPublicKey ${validator.blsPublicKey}`, + `importing legacy delegate ${validator.arkAddress} (${validator.username}) with valid blsPublicKey '${validator.blsPublicKey}'`, + ); + } else { + this.logger.warning( + `importing legacy delegate ${validator.arkAddress} (${validator.username}) with invalid blsPublicKey '${validator.blsPublicKey}'`, ); - stats.skippedValidators++; - continue; } stats.importedValidatorsWithBlsKey++; @@ -426,14 +423,11 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { return stats; } - async #seedVoters( - options: Contracts.Snapshot.LegacyImportOptions, - ): Promise<{ importedVoters: number; skippedVoters: number }> { + async #seedVoters(options: Contracts.Snapshot.LegacyImportOptions): Promise<{ importedVoters: number }> { const iface = new ethers.Interface(ConsensusAbi.abi); const stats = { importedVoters: 0, - skippedVoters: 0, }; const validatorLookup: Record = @@ -453,21 +447,13 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { const validator = validatorLookup[voter.vote]; if (!validator) { - this.logger.debug( - `!!! skipping voter ${voter.arkAddress} for non-existent validator: ${voter.vote}`, - ); - - stats.skippedVoters++; - continue; + throw new Error(`encountered voter ${voter.arkAddress} for non-existent validator: ${voter.vote}`); } if (!validator.blsPublicKey) { this.logger.debug( - `!!! skipping voter ${voter.arkAddress} for validator without registered BlsPublicKey: ${validator.arkAddress}`, + `!!! adding voter ${voter.arkAddress} for validator without registered BlsPublicKey: ${validator.arkAddress}`, ); - - stats.skippedVoters++; - continue; } voterAddresses.push(voter.ethAddress); From 97bc6193600425457ccb96b046924145abfbcd18 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 6 Jun 2025 00:05:34 +0000 Subject: [PATCH 10/11] style: resolve style guide violations --- packages/snapshot-legacy-exporter/source/application-factory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/snapshot-legacy-exporter/source/application-factory.ts b/packages/snapshot-legacy-exporter/source/application-factory.ts index 45fe56539d..0fbd45289c 100644 --- a/packages/snapshot-legacy-exporter/source/application-factory.ts +++ b/packages/snapshot-legacy-exporter/source/application-factory.ts @@ -3,8 +3,8 @@ import { Identifiers } from "@mainsail/contracts"; import { ServiceProvider as CryptoAddressKeccak256 } from "@mainsail/crypto-address-keccak256"; import { ServiceProvider as CryptoKeyPairEcdsa } from "@mainsail/crypto-key-pair-ecdsa"; import { ServiceProvider as CryptoValidation } from "@mainsail/crypto-validation"; -import { ServiceProvider as Logger } from "@mainsail/logger-pino"; import { Application } from "@mainsail/kernel"; +import { ServiceProvider as Logger } from "@mainsail/logger-pino"; import { ServiceProvider as Validation } from "@mainsail/validation"; import { dirSync, setGracefulCleanup } from "tmp"; From 6d0f0908025f64ee9580f256597bfe4349b6b325 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Mon, 9 Jun 2025 10:05:45 +0900 Subject: [PATCH 11/11] cleanup --- .../source/importer.ts | 41 +++++-------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/packages/snapshot-legacy-importer/source/importer.ts b/packages/snapshot-legacy-importer/source/importer.ts index 0188b06d9a..86d9410b76 100644 --- a/packages/snapshot-legacy-importer/source/importer.ts +++ b/packages/snapshot-legacy-importer/source/importer.ts @@ -300,10 +300,10 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { const { importedValidatorsWithBlsKey, importedValidatorsWithoutBlsKey } = await this.#seedValidators(options); // 3) Seed voters - const { importedVoters } = await this.#seedVoters(options); + const importedVoters = await this.#seedVoters(options); // 4) Seed usernames - const { importedUsernames } = await this.#seedUsernames(options); + const importedUsernames = await this.#seedUsernames(options); if (totalSupply !== this.totalSupply) { throw new Error("totalSupply mismatch"); @@ -423,18 +423,10 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { return stats; } - async #seedVoters(options: Contracts.Snapshot.LegacyImportOptions): Promise<{ importedVoters: number }> { + async #seedVoters(options: Contracts.Snapshot.LegacyImportOptions): Promise { const iface = new ethers.Interface(ConsensusAbi.abi); - const stats = { - importedVoters: 0, - }; - - const validatorLookup: Record = - this.#data.validators.reduce((accumulator, current) => { - accumulator[current.ethAddress!] = current; - return accumulator; - }, {}); + let importedVoters = 0; this.logger.info(`seeding ${this.#data.voters.length} voters`); @@ -445,17 +437,6 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { for (const voter of voters) { assert.defined(voter.ethAddress); - const validator = validatorLookup[voter.vote]; - if (!validator) { - throw new Error(`encountered voter ${voter.arkAddress} for non-existent validator: ${voter.vote}`); - } - - if (!validator.blsPublicKey) { - this.logger.debug( - `!!! adding voter ${voter.arkAddress} for validator without registered BlsPublicKey: ${validator.arkAddress}`, - ); - } - voterAddresses.push(voter.ethAddress); validatorAddresses.push(voter.vote); } @@ -475,19 +456,17 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { throw new Error("failed to add votes"); } - stats.importedVoters += voterAddresses.length; + importedVoters += voterAddresses.length; } - return stats; + return importedVoters; } - async #seedUsernames(options: Contracts.Snapshot.LegacyImportOptions): Promise<{ importedUsernames: number }> { + async #seedUsernames(options: Contracts.Snapshot.LegacyImportOptions): Promise { const iface = new ethers.Interface(UsernamesAbi.abi); this.logger.info(`seeding ${this.#data.validators.length} usernames`); - const stats = { - importedUsernames: 0, - }; + let importedUsernames = 0; for (const validator of this.#data.validators) { if (!validator.username) { @@ -508,10 +487,10 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { throw new Error("failed to add username"); } - stats.importedUsernames += 1; + importedUsernames++; } - return stats; + return importedUsernames; } #getTransactionContext(