fix(snapshot): allow export from remote db and other enhancements

packages/configuration-generator/bin/create-genesis-block.js

@@ -21,7 +21,7 @@ async function run() {
 		chainId: 10000,
 		initialHeight: 0,
 		// snapshot: {
-		// 	path: "../../snapshot-19a87c96dbe8ad1be06d33e97cd17f5662eb952c29efd3d8bb00c9c75e7582bc.json",
+		// 	path: "../../f07a7068c50e2e5591beaa572070933008744425d727c792d328d1d5e2fac306.compressed",
 		// },
 	});
 

packages/contracts/source/contracts/snapshot.ts

@@ -24,6 +24,10 @@ export interface LegacyImportOptions {
 
 export interface LegacyImportResult {
 	readonly initialTotalSupply: bigint;
+	readonly importedValidatorsWithBlsKey: number;
+	readonly importedValidatorsWithoutBlsKey: number;
+	readonly importedUsernames: number;
+	readonly importedVoters: number;
 }
 
 export interface ImportedLegacyWallet {

packages/snapshot-legacy-exporter/package.json

@@ -28,6 +28,7 @@
 		"@mainsail/crypto-key-pair-ecdsa": "workspace:*",
 		"@mainsail/crypto-validation": "workspace:*",
 		"@mainsail/kernel": "workspace:*",
+		"@mainsail/logger-pino": "workspace:*",
 		"@mainsail/utils": "workspace:*",
 		"@mainsail/validation": "workspace:*",
 		"pg": "8.14.0",

packages/snapshot-legacy-exporter/source/application-factory.ts

@@ -4,6 +4,7 @@
 import { ServiceProvider as CryptoKeyPairEcdsa } from "@mainsail/crypto-key-pair-ecdsa";
 import { ServiceProvider as CryptoValidation } from "@mainsail/crypto-validation";
 import { Application } from "@mainsail/kernel";
+import { ServiceProvider as Logger } from "@mainsail/logger-pino";
 import { ServiceProvider as Validation } from "@mainsail/validation";
 import { dirSync, setGracefulCleanup } from "tmp";
 
@@ -29,6 +30,7 @@
 	await app.resolve(CryptoValidation).register();
 	await app.resolve(CryptoKeyPairEcdsa).register();
 	await app.resolve(CryptoAddressKeccak256).register();
+	await app.resolve(Logger).register();
 
 	//
 	app.bind(InternalIdentifiers.Snapshot.Generator).to(Generator);

packages/snapshot-legacy-exporter/source/defaults.ts

@@ -30,7 +30,11 @@
 
 		v3: {
 			database: "ark_devnet",
+			// when using podman
+			// host: "host.containers.internal",
+			host: "localhost",
 			password: "test_db",
+			port: 5432,
 			user: "test_db",
 		},
 	},

packages/snapshot-legacy-exporter/source/snapshot/generator.ts

@@ -7,14 +7,16 @@
 import { Contracts, Identifiers } from "@mainsail/contracts";
 import { Application, Providers } from "@mainsail/kernel";
 import { assert } from "@mainsail/utils";
-import { DataSource } from "typeorm";
+import { DataSource, EntityManager } from "typeorm";
 import { PostgresConnectionOptions } from "typeorm/driver/postgres/PostgresConnectionOptions.js";
 
 import { Identifiers as InternalIdentifiers } from "../identifiers.js";
 import { LegacyChainTip, LegacySnapshot, LegacyWallet } from "../interfaces.js";
 
 interface DatabaseOptions extends PostgresConnectionOptions {
 	readonly v3: {
+		readonly host: string;
+		readonly port: number;
 		readonly user: string;
 		readonly password: string;
 		readonly database: string;
@@ -26,32 +28,8 @@
 	@inject(InternalIdentifiers.Application)
 	private app!: Application;
 
-	async #connect(): Promise<DataSource> {
-		const pluginConfig = await this.app
-			.resolve(Providers.PluginConfiguration)
-			.discover("@mainsail/snapshot-legacy-exporter", process.cwd());
-
-		const options = pluginConfig.get<DatabaseOptions>("database");
-		assert.defined(options);
-
-		const dataSource = new DataSource({
-			...options,
-			entities: [],
-			migrations: [],
-			migrationsRun: false,
-			synchronize: false,
-		});
-
-		await dataSource.initialize();
-
-		// Link v3 database
-		await dataSource.query(`
-			CREATE EXTENSION IF NOT EXISTS dblink;
-			SELECT dblink_connect('v3_db', 'dbname=${options.v3.database} user=${options.v3.user} password=${options.v3.password}');
-		`);
-
-		return dataSource;
-	}
+	@inject(Identifiers.Services.Log.Service)
+	private readonly logger!: Contracts.Kernel.Logger;
 
 	public async generateStatic(chainTip: LegacyChainTip, wallets: LegacyWallet[]): Promise<void> {
 		const addressFactory = this.app.get<Contracts.Crypto.AddressFactory>(
@@ -75,64 +53,97 @@
 	}
 
 	public async generate(): Promise<void> {
-		// Connect to V3 database
-		const dataSource = await this.#connect();
-		console.log("connected!");
-
-		const [chainTip] = await dataSource.query(
-			"SELECT * FROM dblink('v3_db', 'SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1') AS blocks(hash varchar, number bigint);",
-		);
+		await this.#runInTransaction(async (entityManager) => {
+			this.logger.info("connected!");
+
+			const [chainTip] = await entityManager.query(
+				"SELECT * FROM dblink('v3_db', 'SELECT id, height FROM blocks ORDER BY height DESC LIMIT 1') AS blocks(hash varchar, number bigint);",
+			);
+
+			const addressFactory = this.app.get<Contracts.Crypto.AddressFactory>(
+				Identifiers.Cryptography.Identity.Address.Factory,
+			);
+
+			// Loop all wallets
+			const limit = 1000;
+			let offset = 0;
+
+			const wallets: LegacyWallet[] = [];
+			for (;;) {
+				this.logger.info(`Fetching wallets (offset: ${offset}, limit: ${limit})`);
+
+				const chunk: LegacyWallet[] = await entityManager.query(`
+					SELECT * FROM dblink(
+						'v3_db',
+						'
+							SELECT address, public_key, balance, attributes FROM wallets
+							ORDER BY balance DESC, address ASC LIMIT ${limit} OFFSET ${offset}
+						'
+					) AS wallets("arkAddress" varchar, "publicKey" varchar, balance bigint, attributes jsonb);
+				`);
+
+				for (const wallet of chunk) {
+					// sanitize
+					if (wallet.attributes?.["delegate"]) {
+						delete wallet.attributes?.["delegate"]["lastBlock"];
+					}
+
+					delete wallet.attributes?.["ipfs"]; // ?
+					delete wallet.attributes?.["business"]; // ?
+					delete wallet.attributes?.["htlc"]; // ?
+					delete wallet.attributes?.["entities"]; // ?
+
+					wallets.push({
+						...wallet,
+						...(wallet.publicKey
+							? {
+									ethAddress: await addressFactory.fromPublicKey(wallet.publicKey),
+								}
+							: {}),
+					});
+				}
 
-		const addressFactory = this.app.get<Contracts.Crypto.AddressFactory>(
-			Identifiers.Cryptography.Identity.Address.Factory,
-		);
+				offset += limit;
 
-		// Loop all wallets
-		const limit = 1000;
-		let offset = 0;
-
-		const wallets: LegacyWallet[] = [];
-		for (;;) {
-			const chunk: LegacyWallet[] = await dataSource.query(`
-				SELECT * FROM dblink(
-					'v3_db',
-					'
-						SELECT address, public_key, balance, attributes FROM wallets -- WHERE attributes ?| array[''vote'', ''delegate'', ''username'']
-						ORDER BY balance DESC, address ASC LIMIT ${limit} OFFSET ${offset}
-					'
-				) AS wallets("arkAddress" varchar, "publicKey" varchar, balance bigint, attributes jsonb);
-			`);
-
-			for (const wallet of chunk) {
-				// sanitize
-				if (wallet.attributes?.["delegate"]) {
-					delete wallet.attributes?.["delegate"]["lastBlock"];
+				if (chunk.length === 0) {
+					break;
 				}
+			}
 
-				delete wallet.attributes?.["ipfs"]; // ?
-				delete wallet.attributes?.["business"]; // ?
-				delete wallet.attributes?.["htlc"]; // ?
-				delete wallet.attributes?.["entities"]; // ?
+			await this.#writeSnapshot(chainTip, wallets);
+		});
+	}
 
-				wallets.push({
-					...wallet,
-					...(wallet.publicKey
-						? {
-								ethAddress: await addressFactory.fromPublicKey(wallet.publicKey),
-							}
-						: {}),
-				});
-			}
+	async #runInTransaction(callback: (entityManager: EntityManager) => Promise<void>): Promise<void> {
+		const pluginConfig = await this.app
+			.resolve(Providers.PluginConfiguration)
+			.discover("@mainsail/snapshot-legacy-exporter", process.cwd());
+
+		const options = pluginConfig.get<DatabaseOptions>("database");
+		assert.defined(options);
 
-			offset += limit;
+		const dataSource = new DataSource({
+			...options,
+			entities: [],
+			migrations: [],
+			migrationsRun: false,
+			synchronize: false,
+		});
 
-			if (chunk.length === 0) {
-				break;
-			}
-		}
+		await dataSource.initialize();
 
 		try {
-			await this.#writeSnapshot(chainTip, wallets);
+			await dataSource.transaction("REPEATABLE READ", async (entityManager) => {
+				// Link v3 database
+				await entityManager.query(`
+							CREATE EXTENSION IF NOT EXISTS dblink;
+							SELECT dblink_connect('v3_db', 'host=${options.v3.host} port=${options.v3.port} dbname=${options.v3.database} user=${options.v3.user} password=${options.v3.password}');
+						`);
+
+				await callback(entityManager);
+			});
+		} catch (ex) {
+			this.logger.error(ex);
 		} finally {
 			await dataSource.destroy();
 		}
@@ -159,7 +170,7 @@
 		const jsonString = JSON.stringify(snapshot);
 		const compressedBuffer = await promisify(brotliCompress)(jsonString);
 		await writeFile(path, compressedBuffer);
-		console.log(`Wrote ${snapshot.wallets.length} wallets to '${path}'`);
+		this.logger.info(`Wrote ${snapshot.wallets.length} wallets to '${path}'`);
 	}
 }