test-runtime: GenesisBuilder runtime API impl + tests (#14310)

* test-runtime: GenesisBuilder runtime API impl + tests

This PR provides implementation of `GenesisBuilder` API for `substrate-test-runtime`, can be considered as reference imiplementation for other runtimes.
The `GenesisBuilder` implementation is gated by `gensis-config` feature.

Tested scenarios:
- default `GenesisConfig` to JSON blob,
- deserialization of `GenesisConfig` from custom JSON, and storing its keys into the Storage (genesis storage creation).
- creation of genesis storage using partial JSON definition,
- checking if invalid/renamed JSON files causes the runtime to panic,

* missing file added

* client: GenesisBuilder helper added

* feature renamed: genesis-config -> genesis-builder

* Update Cargo.toml

* Update Cargo.toml

* Update Cargo.toml

* Update Cargo.toml

* redundant function removed

* genesis builder helper: introduced RuntimeGenesisBuild

* test-runtime: get rid of unused T

* redundant bound removed

* helper: use GenesisBuild again

* tests adjusted for on_genesis

* test-runtime: support for BuildGenesisConfig

* helper: BuildGenesisConfig support

* Update client/genesis-builder/src/lib.rs

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Update test-utils/runtime/src/test_json/readme.md

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Update test-utils/runtime/src/test_json/readme.md

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Update test-utils/runtime/src/genesismap.rs

Co-authored-by: Davide Galassi <davxy@datawok.net>

* jsons are now human-friendly

* fix

* improvements

* jsons fixed

* helper: no_defaults added

* test-runtime: no_defaults added

* test-runtime: patching fn removed

* helper: patching fn removed

* helper: moved to frame_support

* test-runtime: fixes

* Cargo.lock updated

* fmt + naming

* test-runtime: WasmExecutor used

* helper / test-runtime: struct removed

* test-runtime: merge fixes

* Cargo.lock + test-utils/runtime/Cargo.toml updated

* doc fixed

* client/rpc: test fixed (new rt api)

* client/rpc-spec-v2: test fix

* doc fix

* test-runtime: disable-genesis-builder feature

* fix

* fix

* ".git/.scripts/commands/fmt/fmt.sh"

* test-runtime: rerun added to build script

---------

Co-authored-by: Davide Galassi <davxy@datawok.net>
Co-authored-by: parity-processbot <>

Author: michalkucharczyk

Cargo.lock

@@ -194,7 +194,7 @@ async fn follow_with_runtime() {
 		[\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\
 		[\"0xbc9d89904f5b923f\",1],[\"0xc6e9a76309f39b09\",2],[\"0xdd718d5cc53262d4\",1],\
 		[\"0xcbca25e39f142387\",2],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\
-		[\"0xed99c5acb25eedf5\",3]],\"transactionVersion\":1,\"stateVersion\":1}";
+		[\"0xed99c5acb25eedf5\",3],[\"0xfbc577b9d747efd6\",1]],\"transactionVersion\":1,\"stateVersion\":1}";
 
 	let runtime: RuntimeVersion = serde_json::from_str(runtime_str).unwrap();
 

client/rpc-spec-v2/src/chain_head/tests.rs

@@ -518,7 +518,7 @@ async fn should_return_runtime_version() {
 		[\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\
 		[\"0xbc9d89904f5b923f\",1],[\"0xc6e9a76309f39b09\",2],[\"0xdd718d5cc53262d4\",1],\
 		[\"0xcbca25e39f142387\",2],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\
-		[\"0xed99c5acb25eedf5\",3]],\"transactionVersion\":1,\"stateVersion\":1}";
+		[\"0xed99c5acb25eedf5\",3],[\"0xfbc577b9d747efd6\",1]],\"transactionVersion\":1,\"stateVersion\":1}";
 
 	let runtime_version = api.runtime_version(None.into()).unwrap();
 	let serialized = serde_json::to_string(&runtime_version).unwrap();

client/rpc/src/state/tests.rs

@@ -40,11 +40,12 @@ log = { version = "0.4.17", default-features = false }
 sp-core-hashing-proc-macro = { version = "9.0.0", path = "../../primitives/core/hashing/proc-macro" }
 k256 = { version = "0.13.0", default-features = false, features = ["ecdsa"] }
 environmental = { version = "1.1.4", default-features = false }
+sp-genesis-builder = { version = "0.1.0", default-features=false, path = "../../primitives/genesis-builder" }
+serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] }
 
 aquamarine = { version = "0.3.2" }
 
 [dev-dependencies]
-serde_json = "1.0.85"
 assert_matches = "1.3.0"
 pretty_assertions = "1.2.1"
 frame-system = { version = "4.0.0-dev", path = "../system" }
@@ -72,6 +73,7 @@ std = [
 	"frame-support-procedural/std",
 	"log/std",
 	"environmental/std",
+	"sp-genesis-builder/std"
 ]
 runtime-benchmarks = [
 	"frame-system/runtime-benchmarks",

frame/support/Cargo.toml

@@ -0,0 +1,41 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Helper functions for implementing [`sp_genesis_builder::GenesisBuilder`] for runtimes.
+//!
+//! Provides common logic. For more info refer to [`sp_genesis_builder::GenesisBuilder`].
+
+use frame_support::traits::BuildGenesisConfig;
+use sp_genesis_builder::Result as BuildResult;
+use sp_runtime::format_runtime_string;
+
+/// Get the default `GenesisConfig` as a JSON blob. For more info refer to
+/// [`sp_genesis_builder::GenesisBuilder::create_default_config`]
+pub fn create_default_config<GC: BuildGenesisConfig>() -> sp_std::vec::Vec<u8> {
+	serde_json::to_string(&GC::default())
+		.expect("serialization to json is expected to work. qed.")
+		.into_bytes()
+}
+
+/// Build `GenesisConfig` from a JSON blob not using any defaults and store it in the storage. For
+/// more info refer to [`sp_genesis_builder::GenesisBuilder::build_config`].
+pub fn build_config<GC: BuildGenesisConfig>(json: sp_std::vec::Vec<u8>) -> BuildResult {
+	let gc = serde_json::from_slice::<GC>(&json)
+		.map_err(|e| format_runtime_string!("Invalid JSON blob: {}", e))?;
+	<GC as BuildGenesisConfig>::build(&gc);
+	Ok(())
+}

frame/support/src/genesis_builder_helper.rs

@@ -2915,3 +2915,6 @@ pub use frame_support_procedural::register_default_impl;
 
 // Generate a macro that will enable/disable code based on `std` feature being active.
 sp_core::generate_feature_enabled_macro!(std_enabled, feature = "std", $);
+
+// Helper for implementing GenesisBuilder runtime API
+pub mod genesis_builder_helper;

frame/support/src/lib.rs

@@ -13,9 +13,10 @@ publish = false
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-sp-application-crypto = { version = "23.0.0", default-features = false, path = "../../primitives/application-crypto" }
-sp-consensus-aura = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/aura" }
-sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/babe" }
+sp-application-crypto = { version = "23.0.0", default-features = false, path = "../../primitives/application-crypto", features = ["serde"]  }
+sp-consensus-aura = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/aura", features = ["serde"] }
+sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/babe", features = ["serde"] }
+sp-genesis-builder = { version = "0.1.0-dev", default-features = false, path = "../../primitives/genesis-builder" }
 sp-block-builder = { version = "4.0.0-dev", default-features = false, path = "../../primitives/block-builder" }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.5.0", default-features = false, features = ["derive"] }
@@ -29,14 +30,14 @@ frame-support = { version = "4.0.0-dev", default-features = false, path = "../..
 sp-version = { version = "22.0.0", default-features = false, path = "../../primitives/version" }
 sp-session = { version = "4.0.0-dev", default-features = false, path = "../../primitives/session" }
 sp-api = { version = "4.0.0-dev", default-features = false, path = "../../primitives/api" }
-sp-runtime = { version = "24.0.0", default-features = false, path = "../../primitives/runtime" }
+sp-runtime = { version = "24.0.0", default-features = false, path = "../../primitives/runtime", features = ["serde"] }
 pallet-babe = { version = "4.0.0-dev", default-features = false, path = "../../frame/babe" }
 pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../frame/balances" }
 frame-executive = { version = "4.0.0-dev", default-features = false, path = "../../frame/executive" }
 frame-system = { version = "4.0.0-dev", default-features = false, path = "../../frame/system" }
 frame-system-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, path = "../../frame/system/rpc/runtime-api" }
 pallet-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../frame/timestamp" }
-sp-consensus-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../primitives/consensus/grandpa" }
+sp-consensus-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../primitives/consensus/grandpa", features = ["serde"] }
 sp-trie = { version = "22.0.0", default-features = false, path = "../../primitives/trie" }
 sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../primitives/transaction-pool" }
 trie-db = { version = "0.27.0", default-features = false }
@@ -47,14 +48,18 @@ sp-externalities = { version = "0.19.0", default-features = false, path = "../..
 # 3rd party
 array-bytes = { version = "6.1", optional = true }
 log = { version = "0.4.17", default-features = false }
+serde = { version = "1.0.163", features = ["alloc", "derive"], default-features = false }
+serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] }
 
 [dev-dependencies]
 futures = "0.3.21"
 sc-block-builder = { version = "0.10.0-dev", path = "../../client/block-builder" }
 sc-executor = { version = "0.10.0-dev", path = "../../client/executor" }
+sc-executor-common = { version = "0.10.0-dev", path = "../../client/executor/common" }
 sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" }
 substrate-test-runtime-client = { version = "2.0.0", path = "./client" }
 sp-tracing = { version = "10.0.0", path = "../../primitives/tracing" }
+json-patch = { version = "1.0.0", default-features = false }
 
 [build-dependencies]
 substrate-wasm-builder = { version = "5.0.0-dev", path = "../../utils/wasm-builder", optional = true }
@@ -63,8 +68,10 @@ substrate-wasm-builder = { version = "5.0.0-dev", path = "../../utils/wasm-build
 default = [
 	"std",
 ]
+
 std = [
 	"array-bytes",
+	"sp-genesis-builder/std",
 	"sp-application-crypto/std",
 	"sp-consensus-aura/std",
 	"sp-consensus-babe/std",
@@ -100,3 +107,6 @@ std = [
 ]
 # Special feature to disable logging
 disable-logging = [ "sp-api/disable-logging" ]
+
+#Enabling this flag will disable GenesisBuilder API implementation in runtime.
+disable-genesis-builder = []

test-utils/runtime/Cargo.toml

@@ -15,6 +15,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+const BUILD_NO_GENESIS_BUILDER_SUPPORT_ENV: &str = "BUILD_NO_GENESIS_BUILDER_SUPPORT";
+
 fn main() {
 	#[cfg(feature = "std")]
 	{
@@ -29,6 +31,19 @@ fn main() {
 			.build();
 	}
 
+	#[cfg(feature = "std")]
+	if std::env::var(BUILD_NO_GENESIS_BUILDER_SUPPORT_ENV).is_ok() {
+		substrate_wasm_builder::WasmBuilder::new()
+			.with_current_project()
+			.export_heap_base()
+			.append_to_rust_flags("-Clink-arg=-zstack-size=1048576")
+			.set_file_name("wasm_binary_no_genesis_builder")
+			.import_memory()
+			.enable_feature("disable-genesis-builder")
+			.build();
+	}
+	println!("cargo:rerun-if-env-changed={}", BUILD_NO_GENESIS_BUILDER_SUPPORT_ENV);
+
 	#[cfg(feature = "std")]
 	{
 		substrate_wasm_builder::WasmBuilder::new()

test-utils/runtime/build.rs

@@ -73,10 +73,10 @@ impl Default for GenesisStorageBuilder {
 }
 
 impl GenesisStorageBuilder {
-	/// Creates a storage builder for genesis config. `substrage test runtime` `GenesisConfig` is
-	/// initialized with provided `authorities`, `endowed_accounts` with given balance. Key-pairs
-	/// from `extra_storage` will be injected into built storage. `HEAP_PAGES` key and value will
-	/// also be placed into storage.
+	/// Creates a storage builder for genesis config. `substrage test runtime`
+	/// [`RuntimeGenesisConfig`] is initialized with provided `authorities`, `endowed_accounts` with
+	/// given balance. Key-value pairs from `extra_storage` will be injected into built storage.
+	/// `HEAP_PAGES` key and value will also be placed into storage.
 	pub fn new(
 		authorities: Vec<AccountId>,
 		endowed_accounts: Vec<AccountId>,
@@ -91,7 +91,7 @@ impl GenesisStorageBuilder {
 		}
 	}
 
-	/// Override default wasm code to be placed into GenesisConfig.
+	/// Override default wasm code to be placed into RuntimeGenesisConfig.
 	pub fn with_wasm_code(mut self, wasm_code: &Option<Vec<u8>>) -> Self {
 		self.wasm_code = wasm_code.clone();
 		self
@@ -107,16 +107,16 @@ impl GenesisStorageBuilder {
 		self
 	}
 
-	/// Builds the `GenesisConfig` and returns its storage.
-	pub fn build(self) -> Storage {
+	/// A `RuntimeGenesisConfig` from internal configuration
+	pub fn genesis_config(&self) -> RuntimeGenesisConfig {
 		let authorities_sr25519: Vec<_> = self
 			.authorities
 			.clone()
 			.into_iter()
 			.map(|id| sr25519::Public::from(id))
 			.collect();
 
-		let genesis_config = RuntimeGenesisConfig {
+		RuntimeGenesisConfig {
 			system: frame_system::GenesisConfig {
 				code: self.wasm_code.clone().unwrap_or(wasm_binary_unwrap().to_vec()),
 				..Default::default()
@@ -135,11 +135,15 @@ impl GenesisStorageBuilder {
 				..Default::default()
 			},
 			balances: pallet_balances::GenesisConfig { balances: self.balances.clone() },
-		};
+		}
+	}
 
-		let mut storage = genesis_config
+	/// Builds the `RuntimeGenesisConfig` and returns its storage.
+	pub fn build(self) -> Storage {
+		let mut storage = self
+			.genesis_config()
 			.build_storage()
-			.expect("Build storage from substrate-test-runtime GenesisConfig");
+			.expect("Build storage from substrate-test-runtime RuntimeGenesisConfig");
 
 		if let Some(heap_pages) = self.heap_pages_override {
 			storage.top.insert(well_known_keys::HEAP_PAGES.into(), heap_pages.encode());