diff --git a/packages/configuration-generator/bin/create-genesis-block.js b/packages/configuration-generator/bin/create-genesis-block.js index d7338af440..43bf239d0d 100644 --- a/packages/configuration-generator/bin/create-genesis-block.js +++ b/packages/configuration-generator/bin/create-genesis-block.js @@ -21,7 +21,7 @@ async function run() { chainId: 10000, initialHeight: 0, // snapshot: { - // path: "../../snapshot-19a87c96dbe8ad1be06d33e97cd17f5662eb952c29efd3d8bb00c9c75e7582bc.json", + // path: "../../f07a7068c50e2e5591beaa572070933008744425d727c792d328d1d5e2fac306.compressed", // }, }); diff --git a/packages/contracts/source/contracts/snapshot.ts b/packages/contracts/source/contracts/snapshot.ts index 30c3e7d76e..32483f301d 100644 --- a/packages/contracts/source/contracts/snapshot.ts +++ b/packages/contracts/source/contracts/snapshot.ts @@ -24,6 +24,10 @@ export interface LegacyImportOptions { export interface LegacyImportResult { readonly initialTotalSupply: bigint; + readonly importedValidatorsWithBlsKey: number; + readonly importedValidatorsWithoutBlsKey: number; + readonly importedUsernames: number; + readonly importedVoters: number; } export interface ImportedLegacyWallet { diff --git a/packages/snapshot-legacy-exporter/package.json b/packages/snapshot-legacy-exporter/package.json index a647fb5490..837886f2d6 100644 --- a/packages/snapshot-legacy-exporter/package.json +++ b/packages/snapshot-legacy-exporter/package.json @@ -28,6 +28,7 @@ "@mainsail/crypto-key-pair-ecdsa": "workspace:*", "@mainsail/crypto-validation": "workspace:*", "@mainsail/kernel": "workspace:*", + "@mainsail/logger-pino": "workspace:*", "@mainsail/utils": "workspace:*", "@mainsail/validation": "workspace:*", "pg": "8.14.0", diff --git a/packages/snapshot-legacy-exporter/source/application-factory.ts b/packages/snapshot-legacy-exporter/source/application-factory.ts index 67713d9043..0fbd45289c 100644 --- a/packages/snapshot-legacy-exporter/source/application-factory.ts +++ b/packages/snapshot-legacy-exporter/source/application-factory.ts @@ -4,6 +4,7 @@ import { ServiceProvider as CryptoAddressKeccak256 } from "@mainsail/crypto-addr import { ServiceProvider as CryptoKeyPairEcdsa } from "@mainsail/crypto-key-pair-ecdsa"; import { ServiceProvider as CryptoValidation } from "@mainsail/crypto-validation"; import { Application } from "@mainsail/kernel"; +import { ServiceProvider as Logger } from "@mainsail/logger-pino"; import { ServiceProvider as Validation } from "@mainsail/validation"; import { dirSync, setGracefulCleanup } from "tmp"; @@ -29,6 +30,7 @@ export const makeApplication = async (configurationPath: string, options: Record await app.resolve(CryptoValidation).register(); await app.resolve(CryptoKeyPairEcdsa).register(); await app.resolve(CryptoAddressKeccak256).register(); + await app.resolve(Logger).register(); // app.bind(InternalIdentifiers.Snapshot.Generator).to(Generator); diff --git a/packages/snapshot-legacy-exporter/source/defaults.ts b/packages/snapshot-legacy-exporter/source/defaults.ts index 167afabd1f..7757976b45 100644 --- a/packages/snapshot-legacy-exporter/source/defaults.ts +++ b/packages/snapshot-legacy-exporter/source/defaults.ts @@ -30,7 +30,11 @@ export const defaults = { v3: { database: "ark_devnet", + // when using podman + // host: "host.containers.internal", + host: "localhost", password: "test_db", + port: 5432, user: "test_db", }, }, diff --git a/packages/snapshot-legacy-exporter/source/snapshot/generator.ts b/packages/snapshot-legacy-exporter/source/snapshot/generator.ts index c05c55c7ec..d70d723864 100644 --- a/packages/snapshot-legacy-exporter/source/snapshot/generator.ts +++ b/packages/snapshot-legacy-exporter/source/snapshot/generator.ts @@ -7,7 +7,7 @@ import { inject, injectable } from "@mainsail/container"; import { Contracts, Identifiers } from "@mainsail/contracts"; import { Application, Providers } from "@mainsail/kernel"; import { assert } from "@mainsail/utils"; -import { DataSource } from "typeorm"; +import { DataSource, EntityManager } from "typeorm"; import { PostgresConnectionOptions } from "typeorm/driver/postgres/PostgresConnectionOptions.js"; import { Identifiers as InternalIdentifiers } from "../identifiers.js"; @@ -15,6 +15,8 @@ import { LegacyChainTip, LegacySnapshot, LegacyWallet } from "../interfaces.js"; interface DatabaseOptions extends PostgresConnectionOptions { readonly v3: { + readonly host: string; + readonly port: number; readonly user: string; readonly password: string; readonly database: string; @@ -26,32 +28,8 @@ export class Generator { @inject(InternalIdentifiers.Application) private app!: Application; - async #connect(): Promise { - const pluginConfig = await this.app - .resolve(Providers.PluginConfiguration) - .discover("@mainsail/snapshot-legacy-exporter", process.cwd()); - - const options = pluginConfig.get("database"); - assert.defined(options); - - const dataSource = new DataSource({ - ...options, - entities: [], - migrations: [], - migrationsRun: false, - synchronize: false, - }); - - await dataSource.initialize(); - - // Link v3 database - await dataSource.query(` - CREATE EXTENSION IF NOT EXISTS dblink; - SELECT dblink_connect('v3_db', 'dbname=${options.v3.database} user=${options.v3.user} password=${options.v3.password}'); - `); - - return dataSource; - } + @inject(Identifiers.Services.Log.Service) + private readonly logger!: Contracts.Kernel.Logger; public async generateStatic(chainTip: LegacyChainTip, wallets: LegacyWallet[]): Promise { const addressFactory = this.app.get( @@ -75,64 +53,97 @@ export class Generator { } public async generate(): Promise { - // Connect to V3 database - const dataSource = await this.#connect(); - console.log("connected!"); - - const [chainTip] = await dataSource.query( - "SELECT * FROM dblink('v3_db', 'SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1') AS blocks(hash varchar, number bigint);", - ); + await this.#runInTransaction(async (entityManager) => { + this.logger.info("connected!"); + + const [chainTip] = await entityManager.query( + "SELECT * FROM dblink('v3_db', 'SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1') AS blocks(hash varchar, number bigint);", + ); + + const addressFactory = this.app.get( + Identifiers.Cryptography.Identity.Address.Factory, + ); + + // Loop all wallets + const limit = 1000; + let offset = 0; + + const wallets: LegacyWallet[] = []; + for (;;) { + this.logger.info(`Fetching wallets (offset: ${offset}, limit: ${limit})`); + + const chunk: LegacyWallet[] = await entityManager.query(` + SELECT * FROM dblink( + 'v3_db', + ' + SELECT address, public_key, balance, attributes FROM wallets + ORDER BY balance DESC, address ASC LIMIT ${limit} OFFSET ${offset} + ' + ) AS wallets("arkAddress" varchar, "publicKey" varchar, balance bigint, attributes jsonb); + `); + + for (const wallet of chunk) { + // sanitize + if (wallet.attributes?.["delegate"]) { + delete wallet.attributes?.["delegate"]["lastBlock"]; + } + + delete wallet.attributes?.["ipfs"]; // ? + delete wallet.attributes?.["business"]; // ? + delete wallet.attributes?.["htlc"]; // ? + delete wallet.attributes?.["entities"]; // ? + + wallets.push({ + ...wallet, + ...(wallet.publicKey + ? { + ethAddress: await addressFactory.fromPublicKey(wallet.publicKey), + } + : {}), + }); + } - const addressFactory = this.app.get( - Identifiers.Cryptography.Identity.Address.Factory, - ); + offset += limit; - // Loop all wallets - const limit = 1000; - let offset = 0; - - const wallets: LegacyWallet[] = []; - for (;;) { - const chunk: LegacyWallet[] = await dataSource.query(` - SELECT * FROM dblink( - 'v3_db', - ' - SELECT address, public_key, balance, attributes FROM wallets -- WHERE attributes ?| array[''vote'', ''delegate'', ''username''] - ORDER BY balance DESC, address ASC LIMIT ${limit} OFFSET ${offset} - ' - ) AS wallets("arkAddress" varchar, "publicKey" varchar, balance bigint, attributes jsonb); - `); - - for (const wallet of chunk) { - // sanitize - if (wallet.attributes?.["delegate"]) { - delete wallet.attributes?.["delegate"]["lastBlock"]; + if (chunk.length === 0) { + break; } + } - delete wallet.attributes?.["ipfs"]; // ? - delete wallet.attributes?.["business"]; // ? - delete wallet.attributes?.["htlc"]; // ? - delete wallet.attributes?.["entities"]; // ? + await this.#writeSnapshot(chainTip, wallets); + }); + } - wallets.push({ - ...wallet, - ...(wallet.publicKey - ? { - ethAddress: await addressFactory.fromPublicKey(wallet.publicKey), - } - : {}), - }); - } + async #runInTransaction(callback: (entityManager: EntityManager) => Promise): Promise { + const pluginConfig = await this.app + .resolve(Providers.PluginConfiguration) + .discover("@mainsail/snapshot-legacy-exporter", process.cwd()); + + const options = pluginConfig.get("database"); + assert.defined(options); - offset += limit; + const dataSource = new DataSource({ + ...options, + entities: [], + migrations: [], + migrationsRun: false, + synchronize: false, + }); - if (chunk.length === 0) { - break; - } - } + await dataSource.initialize(); try { - await this.#writeSnapshot(chainTip, wallets); + await dataSource.transaction("REPEATABLE READ", async (entityManager) => { + // Link v3 database + await entityManager.query(` + CREATE EXTENSION IF NOT EXISTS dblink; + SELECT dblink_connect('v3_db', 'host=${options.v3.host} port=${options.v3.port} dbname=${options.v3.database} user=${options.v3.user} password=${options.v3.password}'); + `); + + await callback(entityManager); + }); + } catch (ex) { + this.logger.error(ex); } finally { await dataSource.destroy(); } @@ -159,7 +170,7 @@ export class Generator { const jsonString = JSON.stringify(snapshot); const compressedBuffer = await promisify(brotliCompress)(jsonString); await writeFile(path, compressedBuffer); - console.log(`Wrote ${snapshot.wallets.length} wallets to '${path}'`); + this.logger.info(`Wrote ${snapshot.wallets.length} wallets to '${path}'`); } } diff --git a/packages/snapshot-legacy-importer/source/importer.ts b/packages/snapshot-legacy-importer/source/importer.ts index 69ab896940..86d9410b76 100644 --- a/packages/snapshot-legacy-importer/source/importer.ts +++ b/packages/snapshot-legacy-importer/source/importer.ts @@ -134,6 +134,10 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { this.#data.result = result; + this.logger.info( + `snapshot import result: ${JSON.stringify({ ...result, initialTotalSupply: result.initialTotalSupply.toString() })}`, + ); + return result; } @@ -293,19 +297,23 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { const totalSupply = await this.#seedWallets(options); // 2) Seed validators - await this.#seedValidators(options); + const { importedValidatorsWithBlsKey, importedValidatorsWithoutBlsKey } = await this.#seedValidators(options); // 3) Seed voters - await this.#seedVoters(options); + const importedVoters = await this.#seedVoters(options); // 4) Seed usernames - await this.#seedUsernames(options); + const importedUsernames = await this.#seedUsernames(options); if (totalSupply !== this.totalSupply) { throw new Error("totalSupply mismatch"); } return { + importedUsernames, + importedValidatorsWithBlsKey, + importedValidatorsWithoutBlsKey, + importedVoters, initialTotalSupply: totalSupply, }; } @@ -348,44 +356,53 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { return totalSupply; } - async #seedValidators(options: Contracts.Snapshot.LegacyImportOptions): Promise { + async #seedValidators(options: Contracts.Snapshot.LegacyImportOptions): Promise<{ + importedValidatorsWithBlsKey: number; + importedValidatorsWithoutBlsKey: number; + }> { const iface = new ethers.Interface(ConsensusAbi.abi); this.logger.info(`seeding ${this.#data.validators.length} validators`); + const stats = { + importedValidatorsWithBlsKey: 0, + importedValidatorsWithoutBlsKey: 0, + }; + for (const validator of this.#data.validators) { assert.defined(validator.ethAddress); - let blsPublicKey: string | undefined = validator.blsPublicKey; - - if (!blsPublicKey) { + if (!validator.blsPublicKey) { if (!options.mockFakeValidatorBlsKeys) { - this.logger.info( - `skipping legacy delegate ${validator.arkAddress} (${validator.username}) without registered blsPublicKey`, + this.logger.debug( + `importing legacy delegate ${validator.arkAddress} (${validator.username}) without registered blsPublicKey`, ); - continue; - } - - const entropy = sha256(Buffer.from(validator.username, "utf8")).slice(2, 34); - const mnemonic = entropyToMnemonic(Buffer.from(entropy, "hex")); + stats.importedValidatorsWithoutBlsKey++; + } else { + const entropy = sha256(Buffer.from(validator.username, "utf8")).slice(2, 34); + const mnemonic = entropyToMnemonic(Buffer.from(entropy, "hex")); - const consensusKeyPair = await this.consensusKeyPairFactory.fromMnemonic(mnemonic); - blsPublicKey = consensusKeyPair.publicKey; + const consensusKeyPair = await this.consensusKeyPairFactory.fromMnemonic(mnemonic); + validator.blsPublicKey = consensusKeyPair.publicKey; + } } else { - if (!(await this.consensusPublicKeyFactory.verify(blsPublicKey))) { + if (await this.consensusPublicKeyFactory.verify(validator.blsPublicKey)) { this.logger.info( - `skipping legacy delegate ${validator.arkAddress} (${validator.username}) with invalid blsPublicKey ${blsPublicKey}`, + `importing legacy delegate ${validator.arkAddress} (${validator.username}) with valid blsPublicKey '${validator.blsPublicKey}'`, + ); + } else { + this.logger.warning( + `importing legacy delegate ${validator.arkAddress} (${validator.username}) with invalid blsPublicKey '${validator.blsPublicKey}'`, ); - continue; } - } - assert.defined(blsPublicKey); + stats.importedValidatorsWithBlsKey++; + } const data = iface .encodeFunctionData("addValidator", [ validator.ethAddress, - Buffer.from(blsPublicKey, "hex"), + validator.blsPublicKey ? Buffer.from(validator.blsPublicKey, "hex") : Buffer.alloc(0), validator.isResigned, ]) .slice(2); @@ -402,15 +419,14 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { throw new Error("failed to add validator"); } } + + return stats; } - async #seedVoters(options: Contracts.Snapshot.LegacyImportOptions): Promise { + async #seedVoters(options: Contracts.Snapshot.LegacyImportOptions): Promise { const iface = new ethers.Interface(ConsensusAbi.abi); - const validatorLookup = this.#data.validators.reduce((accumulator, current) => { - accumulator[current.ethAddress!] = accumulator; - return accumulator; - }, {}); + let importedVoters = 0; this.logger.info(`seeding ${this.#data.voters.length} voters`); @@ -421,13 +437,6 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { for (const voter of voters) { assert.defined(voter.ethAddress); - if (!validatorLookup[voter.vote]) { - this.logger.warning( - `!!! skipping voter ${voter.arkAddress} for non-existent validator: ${voter.vote}`, - ); - continue; - } - voterAddresses.push(voter.ethAddress); validatorAddresses.push(voter.vote); } @@ -443,17 +452,22 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { ); if (!result.receipt.status) { - console.log(result.receipt); + console.log(result.receipt, result.receipt.output?.toString("hex")); throw new Error("failed to add votes"); } + + importedVoters += voterAddresses.length; } + return importedVoters; } - async #seedUsernames(options: Contracts.Snapshot.LegacyImportOptions): Promise { + async #seedUsernames(options: Contracts.Snapshot.LegacyImportOptions): Promise { const iface = new ethers.Interface(UsernamesAbi.abi); this.logger.info(`seeding ${this.#data.validators.length} usernames`); + let importedUsernames = 0; + for (const validator of this.#data.validators) { if (!validator.username) { continue; @@ -472,7 +486,11 @@ export class Importer implements Contracts.Snapshot.LegacyImporter { if (!result.receipt.status) { throw new Error("failed to add username"); } + + importedUsernames++; } + + return importedUsernames; } #getTransactionContext( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 33468c6103..71cec0a169 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2374,6 +2374,9 @@ importers: '@mainsail/kernel': specifier: workspace:* version: link:../kernel + '@mainsail/logger-pino': + specifier: workspace:* + version: link:../logger-pino '@mainsail/utils': specifier: workspace:* version: link:../utils