diff --git a/.github/workflows/backward-compatibility.yml b/.github/workflows/backward-compatibility.yml index 40ffacb7cc3..ebfff8af52d 100644 --- a/.github/workflows/backward-compatibility.yml +++ b/.github/workflows/backward-compatibility.yml @@ -155,6 +155,7 @@ jobs: --cardano-node-version ${{ matrix.cardano_node_version }} \ --cardano-slot-length 0.25 \ --cardano-epoch-length 45.0 \ + full \ --signed-entity-types ${{ inputs.signed-entity-types }} EXIT_CODE=$? if [ $EXIT_CODE -eq 0 ]; then diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c380730f339..1d83bf68d60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -345,7 +345,7 @@ jobs: run_id: ["#1"] extra_args: [ - "--aggregate-signature-type=Concatenation --check-client-cli-snapshot-converter", + "--aggregate-signature-type=Concatenation full --check-client-cli-snapshot-converter", ] include: diff --git a/Cargo.lock b/Cargo.lock index 1ba381d72b3..f5670132b8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4444,7 +4444,7 @@ dependencies = [ [[package]] name = "mithril-end-to-end" -version = "0.4.139" +version = "0.5.0" dependencies = [ "anyhow", "async-recursion", diff --git a/mithril-test-lab/cardano-devnet/README.md b/mithril-test-lab/cardano-devnet/README.md index 382a4e04bba..a6cb977c987 100644 --- a/mithril-test-lab/cardano-devnet/README.md +++ b/mithril-test-lab/cardano-devnet/README.md @@ -228,8 +228,8 @@ pool1n6sxl7cfe9j9mf6jv228nluvy3k3xdu62chqk2wfaazrsenz4jz 5.258e-4 # Running Mihtril -Mithril can be run from the E2E test with --run-only argument +Mithril can be run from the E2E test with the `run-only` command -`cargo run -p mithril-end-to-end -- -vvv --bin-directory binaries_location/ --devnet-scripts-directory=mithril-test-lab/cardano-devnet/ --run-only` +`cargo run -p mithril-end-to-end -- -vvv --bin-directory binaries_location/ --devnet-scripts-directory=mithril-test-lab/cardano-devnet/ run-only` See more about Mithril E2E testing in mithril-end-to-end [README.md](../mithril-end-to-end/README.md) diff --git a/mithril-test-lab/mithril-end-to-end/Cargo.toml b/mithril-test-lab/mithril-end-to-end/Cargo.toml index 112e203524c..8bec7dbb81a 100644 --- a/mithril-test-lab/mithril-end-to-end/Cargo.toml +++ b/mithril-test-lab/mithril-end-to-end/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-end-to-end" -version = "0.4.139" +version = "0.5.0" authors = { workspace = true } edition = { workspace = true } documentation = { workspace = true } diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs deleted file mode 100644 index 22f2a57d63a..00000000000 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs +++ /dev/null @@ -1,855 +0,0 @@ -use std::{path::PathBuf, time::Duration}; - -use anyhow::{Context, anyhow}; -use reqwest::StatusCode; -use serde::de::DeserializeOwned; -use slog_scope::{info, warn}; - -use mithril_common::{ - StdResult, - entities::{BlockHash, Epoch, EpochSpecifier, TransactionHash}, - messages::{ - CardanoBlocksTransactionsSnapshotListMessage, CardanoBlocksTransactionsSnapshotMessage, - CardanoDatabaseDigestListMessage, CardanoDatabaseSnapshotListMessage, - CardanoDatabaseSnapshotMessage, CardanoStakeDistributionListMessage, - CardanoStakeDistributionMessage, CardanoTransactionSnapshotListMessage, - CardanoTransactionSnapshotMessage, CertificateMessage, MithrilStakeDistributionListMessage, - MithrilStakeDistributionMessage, - }, -}; - -use crate::{ - Aggregator, CardanoBlockCommand, CardanoDbV2Command, CardanoStakeDistributionCommand, - CardanoTransactionCommand, CardanoTransactionV2Command, Client, ClientCommand, FullNode, - MithrilStakeDistributionCommand, NodeVersion, ToolsCommand, UtxoHdCommand, attempt, - utils::{AttemptResult, file_utils::copy_dir_all}, -}; - -async fn get_json_response(url: String) -> StdResult> { - match reqwest::get(url.clone()).await { - Ok(response) => { - let r = response.status(); - match r { - StatusCode::OK => Ok(response.json::().await), - s => Err(anyhow!("Unexpected status code from Aggregator: {s}")), - } - } - Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), - } -} - -pub async fn assert_node_producing_mithril_stake_distribution( - aggregator: &Aggregator, -) -> StdResult { - let url = format!( - "{}/artifact/mithril-stake-distributions", - aggregator.endpoint() - ); - info!("Waiting for the aggregator to produce a mithril stake distribution"; "aggregator" => &aggregator.name()); - - async fn fetch_last_mithril_stake_distribution_hash(url: String) -> StdResult> { - match get_json_response::(url) - .await? - .as_deref() - { - Ok([stake_distribution, ..]) => Ok(Some(stake_distribution.hash.clone())), - Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!("Invalid mithril stake distribution body: {err}",)), - } - } - - match attempt!(30, Duration::from_secs(3), { - fetch_last_mithril_stake_distribution_hash(url.clone()).await - }) { - AttemptResult::Ok(hash) => { - info!("Aggregator produced a mithril stake distribution"; "hash" => &hash, "aggregator" => &aggregator.name()); - Ok(hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_node_producing_mithril_stake_distribution, no response from `{url}`" - )), - }.with_context(|| { - format!( - "Requesting aggregator `{}`", - aggregator.name() - ) - }) -} - -pub async fn assert_signer_is_signing_mithril_stake_distribution( - aggregator: &Aggregator, - hash: &str, - expected_epoch_min: Epoch, -) -> StdResult { - let url = format!( - "{}/artifact/mithril-stake-distribution/{hash}", - aggregator.endpoint() - ); - info!( - "Asserting the aggregator is signing the mithril stake distribution message `{}` with an expected min epoch of `{}`", - hash, - expected_epoch_min; - "aggregator" => &aggregator.name() - ); - - async fn fetch_mithril_stake_distribution_message( - url: String, - expected_epoch_min: Epoch, - ) -> StdResult> { - match get_json_response::(url.clone()).await? { - Ok(stake_distribution) => match stake_distribution.epoch { - epoch if epoch >= expected_epoch_min => Ok(Some(stake_distribution)), - epoch => Err(anyhow!( - "Minimum expected mithril stake distribution epoch not reached: {epoch} < {expected_epoch_min}" - )), - }, - Err(err) => Err(anyhow!("Invalid mithril stake distribution body: {err}",)), - } - } - - match attempt!(10, Duration::from_millis(1000), { - fetch_mithril_stake_distribution_message(url.clone(), expected_epoch_min).await - }) { - AttemptResult::Ok(stake_distribution) => { - info!("Signer signed a mithril stake distribution"; "certificate_hash" => &stake_distribution.certificate_hash, "aggregator" => &aggregator.name()); - Ok(stake_distribution.certificate_hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_signer_is_signing_mithril_stake_distribution, no response from `{url}`" - )), - }.with_context(|| { - format!( - "Requesting aggregator `{}`", - aggregator.name() - ) - }) -} - -pub async fn assert_node_producing_cardano_database_snapshot( - aggregator: &Aggregator, -) -> StdResult { - let url = format!("{}/artifact/cardano-database", aggregator.endpoint()); - info!("Waiting for the aggregator to produce a Cardano database snapshot"; "aggregator" => &aggregator.name()); - - async fn fetch_last_cardano_database_snapshot_hash(url: String) -> StdResult> { - match get_json_response::(url) - .await? - .as_deref() - { - Ok([cardano_database_snapshot, ..]) => Ok(Some(cardano_database_snapshot.hash.clone())), - Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!("Invalid Cardano database snapshot body: {err}",)), - } - } - - match attempt!(30, Duration::from_millis(2000), { - fetch_last_cardano_database_snapshot_hash(url.clone()).await - }) { - AttemptResult::Ok(hash) => { - info!("Aggregator produced a Cardano database snapshot"; "hash" => &hash, "aggregator" => &aggregator.name()); - Ok(hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_node_producing_snapshot, no response from `{url}`" - )), - } - .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) -} - -pub async fn assert_signer_is_signing_cardano_database_snapshot( - aggregator: &Aggregator, - hash: &str, - expected_epoch_min: Epoch, -) -> StdResult { - let url = format!("{}/artifact/cardano-database/{hash}", aggregator.endpoint()); - info!( - "Asserting the aggregator is signing the Cardano database snapshot message `{}` with an expected min epoch of `{}`", - hash, - expected_epoch_min; - "aggregator" => &aggregator.name() - ); - - async fn fetch_cardano_database_snapshot_message( - url: String, - expected_epoch_min: Epoch, - ) -> StdResult> { - match get_json_response::(url).await? { - Ok(cardano_database_snapshot) => match cardano_database_snapshot.beacon.epoch { - epoch if epoch >= expected_epoch_min => Ok(Some(cardano_database_snapshot)), - epoch => Err(anyhow!( - "Minimum expected Cardano database snapshot epoch not reached: {epoch} < {expected_epoch_min}" - )), - }, - Err(err) => Err(anyhow!(err).context("Invalid Cardano database snapshot body")), - } - } - - match attempt!(10, Duration::from_millis(1000), { - fetch_cardano_database_snapshot_message(url.clone(), expected_epoch_min).await - }) { - AttemptResult::Ok(snapshot) => { - info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash, "aggregator" => &aggregator.name()); - Ok(snapshot.certificate_hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_signer_is_signing_snapshot, no response from `{url}`" - )), - } - .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) -} - -pub async fn assert_node_producing_cardano_database_digests_map( - aggregator: &Aggregator, -) -> StdResult> { - let url = format!( - "{}/artifact/cardano-database/digests", - aggregator.endpoint() - ); - info!("Waiting for the aggregator to produce a Cardano database digests map"; "aggregator" => &aggregator.name()); - - async fn fetch_cardano_database_digests_map( - url: String, - ) -> StdResult>> { - match get_json_response::(url) - .await? - .as_deref() - { - Ok(&[]) => Ok(None), - Ok(cardano_database_digests_map) => Ok(Some( - cardano_database_digests_map - .iter() - .map(|item| (item.immutable_file_name.clone(), item.digest.clone())) - .collect(), - )), - Err(err) => Err(anyhow!("Invalid Cardano database digests map body: {err}",)), - } - } - - match attempt!(30, Duration::from_millis(2000), { - fetch_cardano_database_digests_map(url.clone()).await - }) { - AttemptResult::Ok(cardano_database_digests_map) => { - info!("Aggregator produced a Cardano database digests map"; "total_digests" => &cardano_database_digests_map.len(), "aggregator" => &aggregator.name()); - Ok(cardano_database_digests_map) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_node_producing_cardano_database_digests_map, no response from `{url}`" - )), - }.with_context(|| { - format!( - "Requesting aggregator `{}`", - aggregator.name() - ) - }) -} - -pub async fn assert_node_producing_cardano_transactions( - aggregator: &Aggregator, -) -> StdResult { - let url = format!("{}/artifact/cardano-transactions", aggregator.endpoint()); - info!("Waiting for the aggregator to produce a Cardano transactions artifact"; "aggregator" => &aggregator.name(), "aggregator" => &aggregator.name()); - - async fn fetch_last_cardano_transaction_snapshot_hash( - url: String, - ) -> StdResult> { - match get_json_response::(url) - .await? - .as_deref() - { - Ok([artifact, ..]) => Ok(Some(artifact.hash.clone())), - Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!("Invalid Cardano transactions artifact body: {err}",)), - } - } - - match attempt!(30, Duration::from_millis(2000), { - fetch_last_cardano_transaction_snapshot_hash(url.clone()).await - }) { - AttemptResult::Ok(hash) => { - info!("Aggregator produced a Cardano transactions artifact"; "hash" => &hash, "aggregator" => &aggregator.name()); - Ok(hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_node_producing_cardano_transactions, no response from `{url}`" - )), - } - .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) -} - -pub async fn assert_signer_is_signing_cardano_transactions( - aggregator: &Aggregator, - hash: &str, - expected_epoch_min: Epoch, -) -> StdResult { - let url = format!( - "{}/artifact/cardano-transaction/{hash}", - aggregator.endpoint() - ); - info!( - "Asserting the aggregator is signing the Cardano transactions artifact `{}` with an expected min epoch of `{}`", - hash, - expected_epoch_min; - "aggregator" => &aggregator.name() - ); - - async fn fetch_cardano_transaction_snapshot_message( - url: String, - expected_epoch_min: Epoch, - ) -> StdResult> { - match get_json_response::(url).await? { - Ok(artifact) => match artifact.epoch { - epoch if epoch >= expected_epoch_min => Ok(Some(artifact)), - epoch => Err(anyhow!( - "Minimum expected artifact epoch not reached: {epoch} < {expected_epoch_min}" - )), - }, - Err(err) => Err(anyhow!(err).context("Invalid Cardano transactions artifact body")), - } - } - - match attempt!(10, Duration::from_millis(1000), { - fetch_cardano_transaction_snapshot_message(url.clone(), expected_epoch_min).await - }) { - AttemptResult::Ok(artifact) => { - info!("Signer signed a Cardano transactions artifact"; "certificate_hash" => &artifact.certificate_hash, "aggregator" => &aggregator.name()); - Ok(artifact.certificate_hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_signer_is_signing_cardano_transactions, no response from `{url}`" - )), - }.with_context(|| { - format!( - "Requesting aggregator `{}`", - aggregator.name() - ) - }) -} - -pub async fn assert_node_producing_cardano_blocks_transactions( - aggregator: &Aggregator, -) -> StdResult { - let url = format!( - "{}/artifact/cardano-blocks-transactions", - aggregator.endpoint() - ); - info!("Waiting for the aggregator to produce a Cardano blocks transactions artifact"; "aggregator" => &aggregator.name()); - - async fn fetch_last_cardano_blocks_transactions_snapshot_hash( - url: String, - ) -> StdResult> { - match get_json_response::(url) - .await? - .as_deref() - { - Ok([artifact, ..]) => Ok(Some(artifact.hash.clone())), - Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!( - "Invalid Cardano blocks transactions artifact body: {err}", - )), - } - } - - match attempt!(30, Duration::from_millis(2000), { - fetch_last_cardano_blocks_transactions_snapshot_hash(url.clone()).await - }) { - AttemptResult::Ok(hash) => { - info!("Aggregator produced a Cardano blocks transactions artifact"; "hash" => &hash, "aggregator" => &aggregator.name()); - Ok(hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_node_producing_cardano_blocks_transactions, no response from `{url}`" - )), - } - .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) -} - -pub async fn assert_signer_is_signing_cardano_blocks_transactions( - aggregator: &Aggregator, - hash: &str, - expected_epoch_min: Epoch, -) -> StdResult { - let url = format!( - "{}/artifact/cardano-blocks-transactions/{hash}", - aggregator.endpoint() - ); - info!( - "Asserting the aggregator is signing the Cardano blocks transactions artifact `{}` with an expected min epoch of `{}`", - hash, - expected_epoch_min; - "aggregator" => &aggregator.name() - ); - - async fn fetch_cardano_blocks_transactions_snapshot_message( - url: String, - expected_epoch_min: Epoch, - ) -> StdResult> { - match get_json_response::(url).await? { - Ok(artifact) => match artifact.epoch { - epoch if epoch >= expected_epoch_min => Ok(Some(artifact)), - epoch => Err(anyhow!( - "Minimum expected artifact epoch not reached: {epoch} < {expected_epoch_min}" - )), - }, - Err(err) => { - Err(anyhow!(err).context("Invalid Cardano blocks transactions artifact body")) - } - } - } - - match attempt!(10, Duration::from_millis(1000), { - fetch_cardano_blocks_transactions_snapshot_message(url.clone(), expected_epoch_min).await - }) { - AttemptResult::Ok(artifact) => { - info!("Signer signed a Cardano blocks transactions artifact"; "certificate_hash" => &artifact.certificate_hash, "aggregator" => &aggregator.name()); - Ok(artifact.certificate_hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_signer_is_signing_cardano_blocks_transactions, no response from `{url}`" - )), - }.with_context(|| { - format!( - "Requesting aggregator `{}`", - aggregator.name() - ) - }) -} - -pub async fn assert_node_producing_cardano_stake_distribution( - aggregator: &Aggregator, -) -> StdResult<(String, Epoch)> { - let url = format!( - "{}/artifact/cardano-stake-distributions", - aggregator.endpoint() - ); - info!("Waiting for the aggregator to produce a Cardano stake distribution"; "aggregator" => &aggregator.name()); - - async fn fetch_last_cardano_stake_distribution_message( - url: String, - ) -> StdResult> { - match get_json_response::(url) - .await? - .as_deref() - { - Ok([stake_distribution, ..]) => Ok(Some(( - stake_distribution.hash.clone(), - stake_distribution.epoch, - ))), - Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!("Invalid Cardano stake distribution body: {err}",)), - } - } - - match attempt!(30, Duration::from_millis(2000), { - fetch_last_cardano_stake_distribution_message(url.clone()).await - }) { - AttemptResult::Ok((hash, epoch)) => { - info!("Aggregator produced a Cardano stake distribution"; "hash" => &hash, "epoch" => #?epoch, "aggregator" => &aggregator.name()); - Ok((hash, epoch)) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_node_producing_cardano_stake_distribution, no response from `{url}`" - )), - }.with_context(|| { - format!( - "Requesting aggregator `{}`", - aggregator.name() - ) - }) -} - -pub async fn assert_signer_is_signing_cardano_stake_distribution( - aggregator: &Aggregator, - hash: &str, - expected_epoch_min: Epoch, -) -> StdResult { - let url = format!( - "{}/artifact/cardano-stake-distribution/{hash}", - aggregator.endpoint() - ); - info!( - "Asserting the aggregator is signing the Cardano stake distribution message `{}` with an expected min epoch of `{}`", - hash, - expected_epoch_min; - "aggregator" => &aggregator.name() - ); - - async fn fetch_cardano_stake_distribution_message( - url: String, - expected_epoch_min: Epoch, - ) -> StdResult> { - match get_json_response::(url).await? { - Ok(stake_distribution) => match stake_distribution.epoch { - epoch if epoch >= expected_epoch_min => Ok(Some(stake_distribution)), - epoch => Err(anyhow!( - "Minimum expected Cardano stake distribution epoch not reached: {epoch} < {expected_epoch_min}" - )), - }, - Err(err) => Err(anyhow!(err).context("Invalid Cardano stake distribution body")), - } - } - - match attempt!(10, Duration::from_millis(1000), { - fetch_cardano_stake_distribution_message(url.clone(), expected_epoch_min).await - }) { - AttemptResult::Ok(cardano_stake_distribution) => { - info!("Signer signed a Cardano stake distribution"; "certificate_hash" => &cardano_stake_distribution.certificate_hash, "aggregator" => &aggregator.name()); - Ok(cardano_stake_distribution.certificate_hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_signer_is_signing_cardano_stake_distribution, no response from `{url}`" - )), - }.with_context(|| { - format!( - "Requesting aggregator `{}`", - aggregator.name() - ) - }) -} - -pub async fn assert_is_creating_certificate_with_enough_signers( - aggregator: &Aggregator, - certificate_hash: &str, - total_signers_expected: usize, -) -> StdResult<()> { - let url = format!("{}/certificate/{certificate_hash}", aggregator.endpoint()); - info!("Waiting for the aggregator to create a certificate with enough signers"; "aggregator" => &aggregator.name()); - - async fn fetch_certificate_message(url: String) -> StdResult> { - match get_json_response::(url).await? { - Ok(certificate) => Ok(Some(certificate)), - Err(err) => Err(anyhow!(err).context("Invalid snapshot body")), - } - } - - match attempt!(10, Duration::from_millis(1000), { - fetch_certificate_message(url.clone()).await - }) { - AttemptResult::Ok(certificate) => { - info!("Aggregator produced a certificate"; "certificate" => ?certificate); - if certificate.metadata.signers.len() == total_signers_expected { - info!( - "Certificate is signed by expected number of signers: {} >= {} ", - certificate.metadata.signers.len(), - total_signers_expected ; - "aggregator" => &aggregator.name() - ); - Ok(()) - } else { - Err(anyhow!( - "Certificate is not signed by expected number of signers: {} < {} ", - certificate.metadata.signers.len(), - total_signers_expected - )) - } - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted assert_is_creating_certificate, no response from `{url}`" - )), - } - .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) -} - -pub async fn assert_client_can_verify_cardano_database( - client: &mut Client, - hash: &str, -) -> StdResult<()> { - client - .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::List)) - .await?; - - if client.version().is_above_or_equal("0.12.34") { - client - .run(ClientCommand::CardanoDbV2( - CardanoDbV2Command::ListPerEpoch { - epoch_specifier: EpochSpecifier::LatestMinusOffset(5), - }, - )) - .await?; - } else { - warn!( - "Client version is below 0.12.34, skipping `cardano-db snapshot list --epoch latest-5` check" - ); - } - - client - .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::Show { - hash: hash.to_string(), - })) - .await?; - info!("Client list & show the cardano database snapshot"; "hash" => &hash); - - client - .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::Download { - hash: hash.to_string(), - })) - .await?; - info!("Client downloaded & restored the cardano database snapshot"; "hash" => &hash); - - Ok(()) -} - -pub async fn assert_client_can_verify_mithril_stake_distribution( - client: &mut Client, - hash: &str, -) -> StdResult<()> { - client - .run(ClientCommand::MithrilStakeDistribution( - MithrilStakeDistributionCommand::Download { - hash: hash.to_owned(), - }, - )) - .await?; - info!("Client downloaded the Mithril stake distribution"; "hash" => &hash); - - Ok(()) -} - -pub async fn assert_client_can_verify_transactions( - client: &mut Client, - tx_hashes: Vec, -) -> StdResult<()> { - #[allow(dead_code)] - #[derive(Debug, serde::Deserialize)] - struct ClientCardanoTransactionCertifyResult { - certified_transactions: Vec, - non_certified_transactions: Vec, - } - - let result_file = client - .run(ClientCommand::CardanoTransaction( - CardanoTransactionCommand::Certify { - tx_hashes: tx_hashes.clone(), - }, - )) - .await?; - info!("Client verified the Cardano transactions"; "tx_hashes" => ?tx_hashes); - - let file = std::fs::read_to_string(&result_file).with_context(|| { - format!( - "Failed to read client output from file `{}`", - result_file.display() - ) - })?; - let result: ClientCardanoTransactionCertifyResult = - serde_json::from_str(&file).with_context(|| { - format!( - "Failed to parse client output as json from file `{}`", - result_file.display() - ) - })?; - - info!("Asserting that all Cardano transactions were verified by the Client..."); - if tx_hashes.iter().all(|tx| result.certified_transactions.contains(tx)) { - Ok(()) - } else { - Err(anyhow!( - "Not all transactions were certified:\n'{:#?}'", - result, - )) - } -} - -pub async fn assert_client_can_verify_transactions_v2( - client: &mut Client, - tx_hashes: Vec, -) -> StdResult<()> { - #[allow(dead_code)] - #[derive(Debug, serde::Deserialize)] - struct ClientCardanoTransactionCertifyResult { - certified_transactions: Vec, - non_certified_transactions: Vec, - } - - #[derive(Debug, serde::Deserialize)] - struct CertifiedTransactionV2 { - transaction_hash: String, - } - - if !client.version().is_above_or_equal("0.13.1") { - warn!( - "Client version is below 0.13.1, skipping `cardano-transaction certify --backend v2` check" - ); - return Ok(()); - } - - let result_file = client - .run(ClientCommand::CardanoTransactionV2( - CardanoTransactionV2Command::Certify { - tx_hashes: tx_hashes.clone(), - }, - )) - .await?; - info!("Client verified the Cardano transactions V2"; "tx_hashes" => ?tx_hashes); - - let file = std::fs::read_to_string(&result_file).with_context(|| { - format!( - "Failed to read client output from file `{}`", - result_file.display() - ) - })?; - let result: ClientCardanoTransactionCertifyResult = - serde_json::from_str(&file).with_context(|| { - format!( - "Failed to parse client output as json from file `{}`", - result_file.display() - ) - })?; - - info!("Asserting that all Cardano transactions V2 were verified by the Client..."); - let certified_tx_hashes_result: Vec = result - .certified_transactions - .iter() - .map(|tx| tx.transaction_hash.clone()) - .collect(); - - if tx_hashes.iter().all(|tx| certified_tx_hashes_result.contains(tx)) { - Ok(()) - } else { - Err(anyhow!( - "Not all transactions V2 were certified:\n'{:#?}'", - result, - )) - } -} - -pub async fn assert_client_can_verify_blocks( - client: &mut Client, - block_hashes: Vec, -) -> StdResult<()> { - #[allow(dead_code)] - #[derive(Debug, serde::Deserialize)] - struct ClientCardanoBlockCertifyResult { - certified_blocks: Vec, - non_certified_blocks: Vec, - } - - #[derive(Debug, serde::Deserialize)] - struct CertifiedBlock { - block_hash: String, - } - - if !client.version().is_above_or_equal("0.13.1") { - warn!("Client version is below 0.13.1, skipping `cardano-block certify` check"); - return Ok(()); - } - - let result_file = client - .run(ClientCommand::CardanoBlock(CardanoBlockCommand::Certify { - block_hashes: block_hashes.clone(), - })) - .await?; - info!("Client verified the Cardano blocks"; "block_hashes" => ?block_hashes); - let file = std::fs::read_to_string(&result_file).with_context(|| { - format!( - "Failed to read client output from file `{}`", - result_file.display() - ) - })?; - let result: ClientCardanoBlockCertifyResult = - serde_json::from_str(&file).with_context(|| { - format!( - "Failed to parse client output as json from file `{}`", - result_file.display() - ) - })?; - - info!("Asserting that all Cardano blocks were verified by the Client..."); - let certified_blocks_hashes_result: Vec = result - .certified_blocks - .iter() - .map(|block| block.block_hash.clone()) - .collect(); - - if block_hashes - .iter() - .all(|block| certified_blocks_hashes_result.contains(block)) - { - Ok(()) - } else { - Err(anyhow!("Not all blocks were certified:\n'{:#?}'", result,)) - } -} - -pub async fn assert_client_can_verify_cardano_stake_distribution( - client: &mut Client, - hash: &str, - epoch: Epoch, -) -> StdResult<()> { - client - .run(ClientCommand::CardanoStakeDistribution( - CardanoStakeDistributionCommand::Download { - unique_identifier: epoch.to_string(), - }, - )) - .await?; - info!("Client downloaded the Cardano stake distribution by epoch"; "epoch" => epoch.to_string()); - - client - .run(ClientCommand::CardanoStakeDistribution( - CardanoStakeDistributionCommand::Download { - unique_identifier: hash.to_string(), - }, - )) - .await?; - info!("Client downloaded the Cardano stake distribution by hash"; "hash" => hash.to_string()); - - Ok(()) -} - -pub async fn assert_client_can_convert_the_ledger_snapshot( - client: &mut Client, - full_node: &FullNode, - artifacts_dir: PathBuf, - cardano_node_version: NodeVersion, -) -> StdResult<()> { - if client.version().is_below("0.13.10") { - warn!("Client version is below 0.13.10, skipping snapshot conversion check"); - return Ok(()); - } - - let utxo_hd_flavor = if cardano_node_version.is_below("10.7.0") { - "LMDB" - } else { - "LSM" - }; - - let binary_path = artifacts_dir.join("bin").join("snapshot-converter"); - - //copy the db to another temporary location to avoid any risk of modifying the original one during the conversion process - let db_to_convert = artifacts_dir.join("db_to_convert"); - copy_dir_all(&full_node.db_path, &db_to_convert).with_context(|| { - format!( - "Failed to copy the ledger state database from `{}` to `{}` for the snapshot conversion process", - full_node.db_path.display(), - db_to_convert.display() - ) - })?; - - client - .run(ClientCommand::Tools(ToolsCommand::UtxoHd( - UtxoHdCommand::SnapshotConverter { - db_directory: db_to_convert.to_string_lossy().to_string(), - cardano_node_version: cardano_node_version.to_string(), - binary_path: binary_path.to_string_lossy().to_string(), - config_path: full_node.snapshot_converter_config_path.to_string_lossy().to_string(), - utxo_hd_flavor: utxo_hd_flavor.to_string(), - commit: true, - }, - ))) - .await?; - info!("Client converted the ledger state into {utxo_hd_flavor} format"); - - Ok(()) -} diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs deleted file mode 100644 index d30ea487099..00000000000 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::path::PathBuf; - -use crate::{AggregateSignatureType, Aggregator, Devnet}; -use anyhow::Context; -use mithril_common::StdResult; -use mithril_common::entities::{Epoch, ProtocolParameters}; -use mithril_common::messages::AggregatorStatusMessage; -use slog_scope::info; - -/// Retrieve the current Mithril era from a running aggregator by querying its `/status` route. -pub async fn retrieve_current_era(aggregator: &Aggregator) -> StdResult { - let url = format!("{}/status", aggregator.endpoint()); - let response = reqwest::get(&url) - .await - .with_context(|| format!("Failed to query aggregator status at `{url}`"))?; - let status_message: AggregatorStatusMessage = response - .json() - .await - .with_context(|| "Failed to parse aggregator status response")?; - - Ok(status_message.mithril_era.to_string()) -} - -pub async fn bootstrap_genesis_certificate(aggregator: &Aggregator) -> StdResult<()> { - info!("Bootstrap genesis certificate"; "aggregator" => &aggregator.name()); - info!("> retrieving current era from aggregator"; "aggregator" => &aggregator.name()); - let mithril_era = retrieve_current_era(aggregator).await?; - info!("> stopping aggregator"; "aggregator" => &aggregator.name()); - aggregator.stop().await?; - info!("> bootstrapping genesis using signers registered two epochs ago..."; "aggregator" => &aggregator.name()); - aggregator.bootstrap_genesis(&mithril_era).await?; - info!("> done, restarting aggregator"; "aggregator" => &aggregator.name()); - aggregator.serve().await?; - - Ok(()) -} - -pub async fn register_era_marker( - aggregator: &Aggregator, - devnet: &Devnet, - mithril_era: &str, - era_epoch: Epoch, -) -> StdResult<()> { - info!("Register '{mithril_era}' era marker"; "aggregator" => &aggregator.name()); - - info!("> generating era marker tx datum..."; "aggregator" => &aggregator.name()); - let tx_datum_file_path = devnet - .artifacts_dir() - .join(PathBuf::from("era-tx-datum.txt".to_string())); - aggregator - .era_generate_tx_datum(&tx_datum_file_path, mithril_era, era_epoch) - .await?; - - info!("> writing '{mithril_era}' era marker on the Cardano chain..."; "aggregator" => &aggregator.name()); - devnet.write_era_marker(&tx_datum_file_path).await?; - - Ok(()) -} - -pub async fn delegate_stakes_to_pools(devnet: &Devnet, delegation_round: u16) -> StdResult<()> { - info!("Delegate stakes to the cardano pools"); - - devnet.delegate_stakes(delegation_round).await?; - - Ok(()) -} - -pub async fn transfer_funds(devnet: &Devnet) -> StdResult<()> { - info!("Transfer funds on the devnet"); - - devnet.transfer_funds().await?; - - Ok(()) -} - -pub async fn update_protocol_parameters( - aggregator: &Aggregator, - aggregate_signature_type: AggregateSignatureType, -) -> StdResult<()> { - info!("Update protocol parameters"; "aggregator" => &aggregator.name()); - - info!("> stopping aggregator"); - aggregator.stop().await?; - let protocol_parameters_new = match aggregate_signature_type { - AggregateSignatureType::Concatenation => ProtocolParameters { - k: 145, - m: 210, - phi_f: 0.80, - }, - AggregateSignatureType::Snark => ProtocolParameters { - k: 7, - m: 10, - phi_f: 0.95, - }, - }; - - info!( - "> updating protocol parameters to {protocol_parameters_new:?}..."; "aggregator" => &aggregator.name() - ); - aggregator.set_protocol_parameters(&protocol_parameters_new).await; - info!("> done, restarting aggregator"; "aggregator" => &aggregator.name()); - aggregator.serve().await?; - - Ok(()) -} diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/mod.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/mod.rs deleted file mode 100644 index aed51aabe55..00000000000 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod check; -mod exec; -mod wait; - -pub use check::*; -pub use exec::*; -pub use wait::*; diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs deleted file mode 100644 index e45b896e57f..00000000000 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs +++ /dev/null @@ -1,110 +0,0 @@ -use anyhow::{Context, anyhow}; -use reqwest::StatusCode; -use slog_scope::{info, warn}; -use std::time::Duration; - -use mithril_cardano_node_internal_database::entities::ImmutableFile; -use mithril_common::{StdResult, entities::Epoch, messages::EpochSettingsMessage}; - -use crate::{Aggregator, attempt, utils::AttemptResult}; - -pub async fn wait_for_enough_immutable(aggregator: &Aggregator) -> StdResult<()> { - info!("Waiting that enough immutable have been written in the devnet"; "aggregator" => aggregator.name()); - - let db_directory = aggregator.db_directory(); - match attempt!(24, Duration::from_secs(5), { - match ImmutableFile::list_completed_in_dir(db_directory) - .with_context(|| { - format!( - "Immutable file listing failed in dir `{}`", - db_directory.display(), - ) - })? - .last() - { - Some(_) => Ok(Some(())), - None => Ok(None), - } - }) { - AttemptResult::Ok(_) => Ok(()), - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted for enough immutable to be written in `{}`", - db_directory.display() - )), - } -} - -pub async fn wait_for_epoch_settings(aggregator: &Aggregator) -> StdResult { - let aggregator_endpoint = aggregator.endpoint(); - let url = format!("{aggregator_endpoint}/epoch-settings"); - info!("Waiting for the aggregator to expose epoch settings"; "aggregator" => aggregator.name()); - - match attempt!(20, Duration::from_millis(1000), { - match reqwest::get(url.clone()).await { - Ok(response) => match response.status() { - StatusCode::OK => { - let epoch_settings = response - .json::() - .await - .with_context(|| "Invalid EpochSettings body")?; - info!("Aggregator ready"; "epoch_settings" => ?epoch_settings); - Ok(Some(epoch_settings)) - } - s if s.is_server_error() => { - warn!( "Server error while waiting for the Aggregator, http code: {s}"; "aggregator" => aggregator.name()); - Ok(None) - } - _ => Ok(None), - }, - Err(_) => Ok(None), - } - }) { - AttemptResult::Ok(epoch_settings) => Ok(epoch_settings), - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(anyhow!( - "Timeout exhausted for aggregator to be up, no response from `{url}`" - )), - } -} - -pub async fn wait_for_aggregator_at_target_epoch( - aggregator: &Aggregator, - target_epoch: Epoch, - wait_reason: String, -) -> StdResult<()> { - info!( - "Waiting for the cardano network to be at the target epoch: {}", wait_reason; - "aggregator" => aggregator.name(), - "target_epoch" => ?target_epoch - ); - - match attempt!(90, Duration::from_millis(1000), { - match aggregator - .chain_observer() - .get_current_epoch() - .await - .with_context(|| "Could not query current epoch")? - { - Some(epoch) => { - if epoch >= target_epoch { - Ok(Some(())) - } else { - Ok(None) - } - } - None => Ok(None), - } - }) { - AttemptResult::Ok(_) => { - info!("Target epoch reached!"; "aggregator" => aggregator.name(), "target_epoch" => ?target_epoch); - Ok(()) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => { - Err(anyhow!("Timeout exhausted for target epoch to be reached")) - } - }?; - - Ok(()) -} diff --git a/mithril-test-lab/mithril-end-to-end/src/lib.rs b/mithril-test-lab/mithril-end-to-end/src/lib.rs index 68f63c6ec68..8a74c334439 100644 --- a/mithril-test-lab/mithril-end-to-end/src/lib.rs +++ b/mithril-test-lab/mithril-end-to-end/src/lib.rs @@ -1,15 +1,12 @@ -pub mod assertions; mod devnet; -mod end_to_end_spec; mod mithril; -mod run_only; +pub mod scenario; pub mod stress_test; +pub mod toolkit; mod utils; pub use devnet::*; -pub use end_to_end_spec::Spec; pub use mithril::*; -pub use run_only::RunOnly; pub use utils::{CompatibilityChecker, CompatibilityCheckerError, NodeVersion}; use clap::ValueEnum; diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index fcf7c8b8fec..ffddbc612b9 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -1,5 +1,5 @@ use anyhow::{Context, anyhow}; -use clap::{CommandFactory, Parser, Subcommand}; +use clap::{Args, CommandFactory, Parser, Subcommand}; use slog::{Drain, Level, Logger}; use slog_scope::{error, info}; use std::{ @@ -19,18 +19,24 @@ use tokio::{ use mithril_common::StdResult; use mithril_doc::GenerateDocCommands; +use mithril_end_to_end::scenario::{FullScenario, RunOnlyScenario}; +use mithril_end_to_end::toolkit::{ScenarioToolkit, ScenarioToolkitContext}; use mithril_end_to_end::{ AggregateSignatureType, Aggregator, Client, CompatibilityChecker, CompatibilityCheckerError, Devnet, DevnetBootstrapArgs, DmqNodeFlavor, MithrilInfrastructure, MithrilInfrastructureConfig, - NodeVersion, RelaySigner, RetryableDevnetError, RunOnly, Signer, Spec, + NodeVersion, RelaySigner, RetryableDevnetError, Signer, }; +/// Default signed entity types used by scenarios that support multiple entities, such as Full and RunOnly. +const DEFAULT_SIGNED_ENTITY_TYPES: &str = + "CardanoTransactions,CardanoBlocksTransactions,CardanoStakeDistribution,CardanoDatabase"; + /// Tests args #[derive(Parser, Debug, Clone)] -pub struct Args { - /// Available commands +pub struct Cli { + /// Test scenario to run #[command(subcommand)] - command: Option, + scenario: Option, /// A directory where all logs, generated devnet artifacts, snapshots and store folder /// will be located. @@ -41,10 +47,6 @@ pub struct Args { #[clap(long)] work_directory: Option, - /// Directory containing scripts to bootstrap a devnet - #[clap(long, default_value = "./devnet")] - devnet_scripts_directory: PathBuf, - /// Directory to the mithril binaries /// /// It must contains the binaries of the aggregator, signer and client. @@ -53,6 +55,83 @@ pub struct Args { #[clap(long, default_value = ".")] bin_directory: PathBuf, + #[command(flatten)] + cardano_devnet: CardanoDevnetArgs, + + #[command(flatten)] + mithril: MithrilArgs, + + #[command(flatten)] + network_topology: NetworkTopologyArgs, + + /// Verbosity level + #[clap( + short, + long, + action = clap::ArgAction::Count, + help = "Verbosity level, add more v to increase" + )] + verbose: u8, +} + +#[derive(Subcommand, Debug, Clone, PartialEq)] +enum ScenarioArgs { + /// Run the full scenario (bootstrap, run, shutdown) [default] + Full { + /// Will check the ledger snapshot conversion step using utxo-hd snapshot-converter + #[clap(long)] + check_client_cli_snapshot_converter: bool, + + /// Signed entity types parameters (discriminants names in an ordered comma separated list). + #[clap( + long, + value_delimiter = ',', + default_value = DEFAULT_SIGNED_ENTITY_TYPES + )] + signed_entity_types: Vec, + }, + /// Bootstrap a Cardano devnet, Mithril Aggregators and Signers, and run continuously + RunOnly { + /// Signed entity types parameters (discriminants names in an ordered comma separated list). + #[clap( + long, + value_delimiter = ',', + default_value = DEFAULT_SIGNED_ENTITY_TYPES + )] + signed_entity_types: Vec, + }, + // Note: not a scenario, but clap doesn't support more than one subcommand per command + #[clap(alias("doc"), hide(true))] + GenerateDoc(GenerateDocCommands), +} + +impl Default for ScenarioArgs { + fn default() -> Self { + Self::Full { + check_client_cli_snapshot_converter: false, + signed_entity_types: DEFAULT_SIGNED_ENTITY_TYPES.split(',').map(String::from).collect(), + } + } +} + +impl ScenarioArgs { + fn signed_entity_types(&self) -> Vec { + match self { + ScenarioArgs::Full { + signed_entity_types, + .. + } => signed_entity_types.clone(), + ScenarioArgs::RunOnly { + signed_entity_types, + .. + } => signed_entity_types.clone(), + ScenarioArgs::GenerateDoc(..) => vec![], + } + } +} + +#[derive(Args, Debug, Clone)] +struct NetworkTopologyArgs { /// Number of aggregators #[clap(long, default_value_t = 1, value_parser = clap::value_parser!(u8).range(1..))] number_of_aggregators: u8, @@ -61,22 +140,40 @@ pub struct Args { #[clap(long, default_value_t = 2, value_parser = clap::value_parser!(u8).range(1..))] number_of_signers: u8, - /// Length of a Cardano slot in the devnet (in s) - #[clap(long, default_value_t = 0.10)] - cardano_slot_length: f64, + /// Use Mithril relays + #[clap(long)] + use_relays: bool, - /// Length of a Cardano epoch in the devnet (in s) - #[clap(long, default_value_t = 30.0)] - cardano_epoch_length: f64, + /// Signer registration relay mode (used only when 'use_relays' is set, can be 'passthrough' or 'p2p') + #[clap(long, default_value = "passthrough")] + relay_signer_registration_mode: String, - /// Cardano node version, must be a valid semver version - #[clap(long, default_value = "11.0.1")] - cardano_node_version: semver::Version, + /// Signature registration relay mode (used only when 'use_relays' is set, can be 'passthrough' or 'p2p') + #[clap(long, default_value = "p2p")] + relay_signature_registration_mode: String, - /// Epoch at which hard fork to the latest Cardano era will be made (starts with the latest era by default) - #[clap(long, default_value_t = 0)] - cardano_hard_fork_latest_era_at_epoch: u16, + /// Enable P2P passive relays in P2P mode (used only when 'use_relays' is set) + #[clap(long, default_value = "false")] + use_p2p_passive_relays: bool, + /// Use DMQ protocol (used to broadcast signatures) + #[clap(long)] + use_dmq: bool, + + /// DMQ node flavor (used only when 'use_dmq' is set, can be 'haskell' or 'fake') + /// + /// 'haskell': will use the DMQ network created within the 'cardano-devnet' + /// 'fake': will use a fake DMQ network within created with the Mithril relay + #[arg(long, value_enum, default_value = "haskell")] + dmq_node_flavor: Option, + + /// Haskell DMQ node version + #[clap(long)] + dmq_node_version: Option, +} + +#[derive(Args, Debug, Clone)] +struct MithrilArgs { /// Mithril run interval for nodes (in ms) #[clap(long, default_value_t = 125)] mithril_run_interval: u32, @@ -97,60 +194,36 @@ pub struct Args { #[clap(long, default_value = "cardano-chain")] mithril_era_reader_adapter: String, - /// Signed entity types parameters (discriminants names in an ordered comma separated list). - #[clap( - long, - value_delimiter = ',', - default_value = "CardanoTransactions,CardanoBlocksTransactions,CardanoStakeDistribution,CardanoDatabase" - )] - signed_entity_types: Vec, - /// Aggregate signature type used to create the certificates #[clap(long, value_enum, default_value = "Concatenation")] aggregate_signature_type: AggregateSignatureType, - /// Enable run only mode + /// Skip the signature delayer in mithril-signer #[clap(long)] - run_only: bool, - - /// Will check the ledger snapshot conversion step using utxo-hd snapshot-converter - #[clap(long)] - check_client_cli_snapshot_converter: bool, - - /// Use Mithril relays - #[clap(long)] - use_relays: bool, - - /// Signer registration relay mode (used only when 'use_relays' is set, can be 'passthrough' or 'p2p') - #[clap(long, default_value = "passthrough")] - relay_signer_registration_mode: String, - - /// Signature registration relay mode (used only when 'use_relays' is set, can be 'passthrough' or 'p2p') - #[clap(long, default_value = "p2p")] - relay_signature_registration_mode: String, + skip_signature_delayer: bool, +} - /// Enable P2P passive relays in P2P mode (used only when 'use_relays' is set) - #[clap(long, default_value = "false")] - use_p2p_passive_relays: bool, +#[derive(Args, Debug, Clone)] +struct CardanoDevnetArgs { + /// Directory containing scripts to bootstrap a devnet + #[clap(long, default_value = "./devnet")] + devnet_scripts_directory: PathBuf, - /// Skip the signature delayer - #[clap(long)] - skip_signature_delayer: bool, + /// Length of a Cardano slot in the devnet (in s) + #[clap(long, default_value_t = 0.10)] + cardano_slot_length: f64, - /// Use DMQ protocol (used to broadcast signatures) - #[clap(long)] - use_dmq: bool, + /// Length of a Cardano epoch in the devnet (multiple of the slot length) + #[clap(long, default_value_t = 30.0)] + cardano_epoch_length: f64, - /// DMQ node flavor (used only when 'use_dmq' is set, can be 'haskell' or 'fake') - /// - /// 'haskell': will use the DMQ network created within the 'cardano-devnet' - /// 'fake': will use a fake DMQ network within created with the Mithril relay - #[arg(long, value_enum, default_value = "haskell")] - dmq_node_flavor: Option, + /// Cardano node version, must be a valid semver version + #[clap(long, default_value = "11.0.1")] + cardano_node_version: semver::Version, - /// Haskell DMQ node version - #[clap(long)] - dmq_node_version: Option, + /// Epoch at which hard fork to the latest Cardano era will be made (starts with the latest era by default) + #[clap(long, default_value_t = 0)] + cardano_hard_fork_latest_era_at_epoch: u16, /// Skip cardano binaries download /// (will use the ones in the `bin` folder of the `Args::devnet_scripts_directory`, which defaults to `./devnet/bin`) @@ -160,18 +233,9 @@ pub struct Args { /// URL to download cardano binaries from (if not set, it will default to the official cardano releases) #[clap(long)] cardano_binary_url: Option, - - /// Verbosity level - #[clap( - short, - long, - action = clap::ArgAction::Count, - help = "Verbosity level, add more v to increase" - )] - verbose: u8, } -impl Args { +impl Cli { fn log_level(&self) -> Level { match self.verbose { 0 => Level::Error, @@ -183,7 +247,7 @@ impl Args { } fn validate(&self) -> StdResult<()> { - if !self.use_relays && self.number_of_aggregators >= 2 { + if !self.network_topology.use_relays && self.network_topology.number_of_aggregators >= 2 { return Err(anyhow!( "The 'use_relays' parameter must be activated to run more than one aggregator" )); @@ -193,12 +257,6 @@ impl Args { } } -#[derive(Subcommand, Debug, Clone)] -enum EndToEndCommands { - #[clap(alias("doc"), hide(true))] - GenerateDoc(GenerateDocCommands), -} - fn main() -> AppResult { tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -209,13 +267,15 @@ fn main() -> AppResult { } async fn main_exec() -> StdResult<()> { - let args = Args::parse(); + let args = Cli::parse(); let _guard = slog_scope::set_global_logger(build_logger(&args)); - if let Some(EndToEndCommands::GenerateDoc(cmd)) = &args.command { - return cmd.execute(&mut Args::command()).map_err(|message| anyhow!(message)); + if let Some(ScenarioArgs::GenerateDoc(cmd)) = &args.scenario { + return cmd.execute(&mut Cli::command()).map_err(|message| anyhow!(message)); } + info!("Starting Mithril End-to-End test suite"; "args" => ?args); + let work_dir = { let dir = match &args.work_directory { Some(path) => path.to_owned(), @@ -360,21 +420,21 @@ impl App { pub async fn run( &mut self, - args: Args, + args: Cli, work_dir: PathBuf, store_dir: PathBuf, artifacts_dir: PathBuf, ) -> StdResult<()> { let server_port = 8080; args.validate()?; - let run_only_mode = args.run_only; - let check_client_cli_snapshot_converter = args.check_client_cli_snapshot_converter; - let use_relays = args.use_relays; - let relay_signer_registration_mode = args.relay_signer_registration_mode; - let relay_signature_registration_mode = args.relay_signature_registration_mode; - let use_dmq = args.use_dmq; + let scenario = args.scenario.unwrap_or_default(); + let use_relays = args.network_topology.use_relays; + let relay_signer_registration_mode = args.network_topology.relay_signer_registration_mode; + let relay_signature_registration_mode = + args.network_topology.relay_signature_registration_mode; + let use_dmq = args.network_topology.use_dmq; - let use_p2p_passive_relays = args.use_p2p_passive_relays; + let use_p2p_passive_relays = args.network_topology.use_p2p_passive_relays; CompatibilityChecker::default().check(BTreeMap::from([ ( @@ -393,72 +453,95 @@ impl App { RelaySigner::BIN_NAME, NodeVersion::fetch_semver(RelaySigner::BIN_NAME, &args.bin_directory)?, ), - ("cardano-node", args.cardano_node_version.to_owned()), + ( + "cardano-node", + args.cardano_devnet.cardano_node_version.to_owned(), + ), ]))?; + let toolkit = ScenarioToolkit::new(ScenarioToolkitContext::new_from_cardano_epoch( + args.cardano_devnet.cardano_slot_length, + args.cardano_devnet.cardano_epoch_length, + )); + let devnet = Devnet::bootstrap(&DevnetBootstrapArgs { - devnet_scripts_dir: args.devnet_scripts_directory, + devnet_scripts_dir: args.cardano_devnet.devnet_scripts_directory, artifacts_target_dir: work_dir.join("devnet"), - number_of_pool_nodes: args.number_of_signers, - number_of_full_nodes: args.number_of_aggregators, - cardano_slot_length: args.cardano_slot_length, - cardano_epoch_length: args.cardano_epoch_length, - cardano_node_version: args.cardano_node_version.to_owned(), - dmq_node_version: args.dmq_node_version.clone(), - cardano_hard_fork_latest_era_at_epoch: args.cardano_hard_fork_latest_era_at_epoch, - skip_cardano_bin_download: args.skip_cardano_bin_download, - cardano_binary_url: args.cardano_binary_url.clone(), + number_of_pool_nodes: args.network_topology.number_of_signers, + number_of_full_nodes: args.network_topology.number_of_aggregators, + cardano_slot_length: args.cardano_devnet.cardano_slot_length, + cardano_epoch_length: args.cardano_devnet.cardano_epoch_length, + cardano_node_version: args.cardano_devnet.cardano_node_version.to_owned(), + dmq_node_version: args.network_topology.dmq_node_version.clone(), + cardano_hard_fork_latest_era_at_epoch: args + .cardano_devnet + .cardano_hard_fork_latest_era_at_epoch, + skip_cardano_bin_download: args.cardano_devnet.skip_cardano_bin_download, + cardano_binary_url: args.cardano_devnet.cardano_binary_url.clone(), }) .await?; *self.devnet.lock().await = Some(devnet.clone()); let infrastructure = Arc::new( - MithrilInfrastructure::start(&MithrilInfrastructureConfig { - number_of_aggregators: args.number_of_aggregators, - number_of_signers: args.number_of_signers, - server_port, - devnet: devnet.clone(), - work_dir, - store_dir, - artifacts_dir, - bin_dir: args.bin_directory, - cardano_node_version: args.cardano_node_version, - mithril_run_interval: args.mithril_run_interval, - mithril_era: args.mithril_era, - mithril_era_reader_adapter: args.mithril_era_reader_adapter, - signed_entity_types: args.signed_entity_types.clone(), - aggregate_signature_type: args.aggregate_signature_type, - run_only_mode, - check_client_cli_snapshot_converter, - use_dmq, - dmq_node_flavor: args.dmq_node_flavor, - use_relays, - relay_signer_registration_mode, - relay_signature_registration_mode, - skip_signature_delayer: args.skip_signature_delayer, - use_p2p_passive_relays, - use_era_specific_work_dir: args.mithril_next_era.is_some(), - }) + MithrilInfrastructure::start( + toolkit.clone(), + &MithrilInfrastructureConfig { + number_of_aggregators: args.network_topology.number_of_aggregators, + number_of_signers: args.network_topology.number_of_signers, + server_port, + devnet: devnet.clone(), + work_dir, + store_dir, + artifacts_dir, + bin_dir: args.bin_directory, + cardano_node_version: args.cardano_devnet.cardano_node_version, + mithril_run_interval: args.mithril.mithril_run_interval, + mithril_era: args.mithril.mithril_era, + mithril_era_reader_adapter: args.mithril.mithril_era_reader_adapter, + signed_entity_types: scenario.signed_entity_types(), + aggregate_signature_type: args.mithril.aggregate_signature_type, + use_dmq, + dmq_node_flavor: args.network_topology.dmq_node_flavor, + use_relays, + relay_signer_registration_mode, + relay_signature_registration_mode, + skip_signature_delayer: args.mithril.skip_signature_delayer, + use_p2p_passive_relays, + use_era_specific_work_dir: args.mithril.mithril_next_era.is_some(), + }, + ) .await?, ); *self.infrastructure.lock().await = Some(infrastructure.clone()); - let runner: StdResult<()> = match run_only_mode { - true => RunOnly::new(infrastructure).run().await, - false => { - Spec::new( + let (runner, run_forever): (StdResult<()>, bool) = match scenario { + ScenarioArgs::Full { + check_client_cli_snapshot_converter, + signed_entity_types, + } => { + let runner = FullScenario::new( + toolkit, infrastructure, - args.signed_entity_types, - args.mithril_next_era, - args.mithril_era_regenesis_on_switch, + signed_entity_types, + check_client_cli_snapshot_converter, + args.mithril.mithril_next_era, + args.mithril.mithril_era_regenesis_on_switch, ) .run() - .await + .await; + (runner, false) + } + ScenarioArgs::RunOnly { .. } => ( + RunOnlyScenario::new(toolkit, infrastructure).run().await, + true, + ), + ScenarioArgs::GenerateDoc(_) => { + unreachable!("ScenarioArgs::GenerateDoc should not be used here") } }; match runner.with_context(|| "Mithril End to End test failed") { - Ok(()) if run_only_mode => loop { + Ok(()) if run_forever => loop { info!( "Mithril end to end is running and will remain active until manually stopped..." ); @@ -501,7 +584,7 @@ impl AppStopper { } } -fn build_logger(args: &Args) -> Logger { +fn build_logger(args: &Cli) -> Logger { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator).build().fuse(); let drain = slog::LevelFilter::new(drain, args.log_level()).fuse(); @@ -548,6 +631,20 @@ fn with_graceful_shutdown(join_set: &mut JoinSet>) { mod tests { use super::*; + #[test] + fn default_scenario_is_full_with_expected_signed_entity_types() { + assert_eq!( + ScenarioArgs::default(), + ScenarioArgs::Full { + check_client_cli_snapshot_converter: false, + signed_entity_types: DEFAULT_SIGNED_ENTITY_TYPES + .split(',') + .map(String::from) + .collect(), + } + ) + } + #[test] fn app_result_exit_code() { let expected_exit_code = ExitCode::SUCCESS; @@ -589,12 +686,12 @@ mod tests { #[test] fn args_fails_validation() { - let args = Args::parse_from(["", "--number-of-aggregators", "2"]); + let args = Cli::parse_from(["", "--number-of-aggregators", "2"]); args.validate().expect_err( "validate should fail with more than one aggregator if p2p network is not used", ); - let args = Args::parse_from(["", "--use-relays", "--number-of-aggregators", "2"]); + let args = Cli::parse_from(["", "--use-relays", "--number-of-aggregators", "2"]); args.validate() .expect("validate should succeed with more than one aggregator if p2p network is used"); } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index 92261f7aaf5..00e1e1dcc3a 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -9,10 +9,10 @@ use mithril_common::entities::{Epoch, PartyId, ProtocolParameters}; use mithril_common::{CardanoNetwork, StdResult}; use crate::mithril::relay_signer::RelaySignerConfiguration; +use crate::toolkit::ScenarioToolkit; use crate::{ AggregateSignatureType, Aggregator, AggregatorConfig, Client, DEVNET_MAGIC_ID, Devnet, DmqNodeFlavor, FullNode, PoolNode, RelayAggregator, RelayPassive, RelaySigner, Signer, - assertions, }; use super::signer::SignerConfig; @@ -32,8 +32,6 @@ pub struct MithrilInfrastructureConfig { pub mithril_era_reader_adapter: String, pub signed_entity_types: Vec, pub aggregate_signature_type: AggregateSignatureType, - pub run_only_mode: bool, - pub check_client_cli_snapshot_converter: bool, pub use_relays: bool, pub relay_signer_registration_mode: String, pub relay_signature_registration_mode: String, @@ -70,8 +68,6 @@ impl MithrilInfrastructureConfig { mithril_era_reader_adapter: "adapter1".to_string(), signed_entity_types: vec!["type1".to_string()], aggregate_signature_type: AggregateSignatureType::Concatenation, - run_only_mode: false, - check_client_cli_snapshot_converter: false, use_relays: false, relay_signer_registration_mode: "passthrough".to_string(), relay_signature_registration_mode: "passthrough".to_string(), @@ -85,6 +81,7 @@ impl MithrilInfrastructureConfig { } pub struct MithrilInfrastructure { + toolkit: ScenarioToolkit, artifacts_dir: PathBuf, bin_dir: PathBuf, devnet: Devnet, @@ -95,8 +92,6 @@ pub struct MithrilInfrastructure { relay_passives: Vec, cardano_node_version: semver::Version, cardano_chain_observer: Arc, - run_only_mode: bool, - check_client_cli_snapshot_converter: bool, current_era: RwLock, era_reader_adapter: String, use_era_specific_work_dir: bool, @@ -104,7 +99,10 @@ pub struct MithrilInfrastructure { } impl MithrilInfrastructure { - pub async fn start(config: &MithrilInfrastructureConfig) -> StdResult { + pub async fn start( + toolkit: ScenarioToolkit, + config: &MithrilInfrastructureConfig, + ) -> StdResult { let chain_observer_type = "pallas"; config.devnet.run().await?; if config.use_dmq && config.dmq_node_flavor == Some(DmqNodeFlavor::Haskell) { @@ -124,7 +122,7 @@ impl MithrilInfrastructure { Self::prepare_aggregators(config, aggregator_cardano_nodes, chain_observer_type) .await?; - Self::register_startup_era(&leader_aggregator, config).await?; + Self::register_startup_era(&toolkit, &leader_aggregator, config).await?; leader_aggregator.serve().await?; let follower_aggregator_endpoints = follower_aggregators @@ -158,6 +156,7 @@ impl MithrilInfrastructure { all_aggregators.extend(follower_aggregators); Ok(Self { + toolkit, bin_dir: config.bin_dir.to_path_buf(), artifacts_dir: config.artifacts_dir.to_path_buf(), devnet: config.devnet.clone(), @@ -168,8 +167,6 @@ impl MithrilInfrastructure { relay_passives, cardano_chain_observer, cardano_node_version: config.cardano_node_version.clone(), - run_only_mode: config.run_only_mode, - check_client_cli_snapshot_converter: config.check_client_cli_snapshot_converter, current_era: RwLock::new(config.mithril_era.clone()), era_reader_adapter: config.mithril_era_reader_adapter.clone(), use_era_specific_work_dir: config.use_era_specific_work_dir, @@ -178,18 +175,16 @@ impl MithrilInfrastructure { } async fn register_startup_era( + toolkit: &ScenarioToolkit, aggregator: &Aggregator, config: &MithrilInfrastructureConfig, ) -> StdResult<()> { let era_epoch = Epoch(0); if config.mithril_era_reader_adapter == "cardano-chain" { - assertions::register_era_marker( - aggregator, - &config.devnet, - &config.mithril_era, - era_epoch, - ) - .await?; + toolkit + .exec + .register_era_marker(aggregator, &config.devnet, &config.mithril_era, era_epoch) + .await?; } Ok(()) @@ -204,13 +199,10 @@ impl MithrilInfrastructure { + 1; if self.era_reader_adapter == "cardano-chain" { let devnet = self.devnet.clone(); - assertions::register_era_marker( - self.leader_aggregator(), - &devnet, - next_era, - next_era_epoch, - ) - .await?; + self.toolkit + .exec + .register_era_marker(self.leader_aggregator(), &devnet, next_era, next_era_epoch) + .await?; } let mut current_era = self.current_era.write().await; *current_era = next_era.to_owned(); @@ -434,7 +426,7 @@ impl MithrilInfrastructure { } pub async fn stop_nodes(&self) -> StdResult<()> { - // Note: The aggregators should be stopped *last* since signers depends on it + // Note: The aggregators should be stopped *last* since signers depend on it info!("Stopping Mithril infrastructure"); for signer in &self.signers { signer.stop().await?; @@ -528,14 +520,6 @@ impl MithrilInfrastructure { Client::new(aggregator.endpoint(), &work_dir, &self.bin_dir) } - pub fn run_only_mode(&self) -> bool { - self.run_only_mode - } - - pub fn check_client_cli_snapshot_converter(&self) -> bool { - self.check_client_cli_snapshot_converter - } - pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { for aggregator in self.aggregators() { aggregator.tail_logs(number_of_line).await?; diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/scenario/full.rs similarity index 58% rename from mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs rename to mithril-test-lab/mithril-end-to-end/src/scenario/full.rs index 5a9f948cbe2..42aaccfd921 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/scenario/full.rs @@ -9,27 +9,32 @@ use mithril_common::{ }; use crate::{ - Aggregator, MithrilInfrastructure, NodeVersion, assertions, + Aggregator, MithrilInfrastructure, NodeVersion, + toolkit::ScenarioToolkit, utils::{ randomly_take_blocks_hashes, randomly_take_transactions_hashes, retrieve_blocks_transactions_from_immutable_files, }, }; -pub struct Spec { - pub infrastructure: Arc, +pub struct FullScenario { + toolkit: ScenarioToolkit, + infrastructure: Arc, is_signing_cardano_transactions: bool, is_signing_cardano_blocks_transactions: bool, is_signing_cardano_stake_distribution: bool, is_signing_cardano_database: bool, + check_client_cli_snapshot_converter: bool, next_era: Option, regenesis_on_era_switch: bool, } -impl Spec { +impl FullScenario { pub fn new( + toolkit: ScenarioToolkit, infrastructure: Arc, signed_entity_types: Vec, + check_client_cli_snapshot_converter: bool, next_era: Option, regenesis_on_era_switch: bool, ) -> Self { @@ -51,6 +56,7 @@ impl Spec { Self { infrastructure, + toolkit, is_signing_cardano_transactions: signed_entity_types .contains(&SignedEntityTypeDiscriminants::CardanoTransactions.to_string()), is_signing_cardano_blocks_transactions, @@ -58,6 +64,7 @@ impl Spec { .contains(&SignedEntityTypeDiscriminants::CardanoStakeDistribution.to_string()), is_signing_cardano_database: signed_entity_types .contains(&SignedEntityTypeDiscriminants::CardanoDatabase.to_string()), + check_client_cli_snapshot_converter, next_era, regenesis_on_era_switch, } @@ -71,7 +78,7 @@ impl Spec { // This step needs to be executed early in the process so that the transactions are available // for signing in the penultimate immutable chunk before the end of the test. // As we get closer to the tip of the chain when signing, we'll be able to relax this constraint. - assertions::transfer_funds(spec.infrastructure.devnet()).await?; + spec.toolkit.exec.transfer_funds(spec.infrastructure.devnet()).await?; info!("Bootstrapping leader aggregator"); spec.bootstrap_leader_aggregator(&spec.infrastructure).await?; @@ -106,34 +113,44 @@ impl Spec { ) -> StdResult<()> { let leader_aggregator = infrastructure.leader_aggregator(); - assertions::wait_for_enough_immutable(leader_aggregator).await?; + self.toolkit.wait.for_enough_immutable(leader_aggregator).await?; let chain_observer = leader_aggregator.chain_observer(); let start_epoch = chain_observer.get_current_epoch().await?.unwrap_or_default(); // Wait 4 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let mut target_epoch = start_epoch + 4; - assertions::wait_for_aggregator_at_target_epoch( - leader_aggregator, - target_epoch, - "minimal epoch for the aggregator to be able to bootstrap genesis certificate" - .to_string(), - ) - .await?; - assertions::bootstrap_genesis_certificate(leader_aggregator).await?; - assertions::wait_for_epoch_settings(leader_aggregator).await?; + self.toolkit + .wait + .for_aggregator_at_target_epoch( + leader_aggregator, + target_epoch, + "minimal epoch for the aggregator to be able to bootstrap genesis certificate" + .to_string(), + ) + .await?; + self.toolkit + .exec + .bootstrap_genesis_certificate(leader_aggregator) + .await?; + self.toolkit.wait.for_epoch_settings(leader_aggregator).await?; // Wait 2 epochs before changing stake distribution, so that we use at least one original stake distribution target_epoch += 2; - assertions::wait_for_aggregator_at_target_epoch( - leader_aggregator, - target_epoch, - "epoch after which the stake distribution will change".to_string(), - ) - .await?; + self.toolkit + .wait + .for_aggregator_at_target_epoch( + leader_aggregator, + target_epoch, + "epoch after which the stake distribution will change".to_string(), + ) + .await?; // Delegate some stakes to pools let delegation_round = 1; - assertions::delegate_stakes_to_pools(infrastructure.devnet(), delegation_round).await?; + self.toolkit + .exec + .delegate_stakes_to_pools(infrastructure.devnet(), delegation_round) + .await?; Ok(()) } @@ -148,24 +165,25 @@ impl Spec { // Wait 2 epochs before changing protocol parameters let mut target_epoch = start_epoch + 2; - assertions::wait_for_aggregator_at_target_epoch( - aggregator, - target_epoch, - "epoch after which the protocol parameters will change".to_string(), - ) - .await?; - - if aggregator.is_first() || aggregator.version().is_below("0.7.94") { - assertions::update_protocol_parameters( + self.toolkit + .wait + .for_aggregator_at_target_epoch( aggregator, - infrastructure.aggregate_signature_type(), + target_epoch, + "epoch after which the protocol parameters will change".to_string(), ) .await?; + + if aggregator.is_first() || aggregator.version().is_below("0.7.94") { + self.toolkit + .exec + .update_protocol_parameters(aggregator, infrastructure.aggregate_signature_type()) + .await?; } // Wait 6 epochs after protocol parameters update, so that we make sure that we use new protocol parameters as well as new stake distribution a few times target_epoch += 6; - assertions::wait_for_aggregator_at_target_epoch( + self.toolkit.wait.for_aggregator_at_target_epoch( aggregator, target_epoch, "epoch after which the certificate chain will be long enough to catch most common troubles with stake distribution and protocol parameters".to_string(), @@ -184,23 +202,28 @@ impl Spec { infrastructure.register_switch_to_next_era(next_era).await?; } target_epoch += 5; - assertions::wait_for_aggregator_at_target_epoch( - aggregator, - target_epoch, - "epoch after which the era switch will have triggered".to_string(), - ) - .await?; - - // Proceed to a re-genesis of the certificate chain - if self.regenesis_on_era_switch { - assertions::bootstrap_genesis_certificate(aggregator).await?; - target_epoch += 5; - assertions::wait_for_aggregator_at_target_epoch( + self.toolkit + .wait + .for_aggregator_at_target_epoch( aggregator, target_epoch, - "epoch after which the re-genesis on era switch will be completed".to_string(), + "epoch after which the era switch will have triggered".to_string(), ) .await?; + + // Proceed to a re-genesis of the certificate chain + if self.regenesis_on_era_switch { + self.toolkit.exec.bootstrap_genesis_certificate(aggregator).await?; + target_epoch += 5; + self.toolkit + .wait + .for_aggregator_at_target_epoch( + aggregator, + target_epoch, + "epoch after which the re-genesis on era switch will be completed" + .to_string(), + ) + .await?; } // Verify that artifacts are produced and signed correctly @@ -209,15 +232,17 @@ impl Spec { } // Check the ledger snapshot conversion step using utxo-hd snapshot-converter - if infrastructure.check_client_cli_snapshot_converter() { + if self.check_client_cli_snapshot_converter { let mut client = infrastructure.build_client(aggregator).await?; - assertions::assert_client_can_convert_the_ledger_snapshot( - &mut client, - aggregator.full_node(), - infrastructure.devnet().artifacts_dir(), - NodeVersion::new(infrastructure.cardano_node_version().clone()), - ) - .await?; + self.toolkit + .check + .client_can_convert_the_ledger_snapshot( + &mut client, + aggregator.full_node(), + infrastructure.devnet().artifacts_dir(), + NodeVersion::new(infrastructure.cardano_node_version().clone()), + ) + .await?; } Ok(()) @@ -229,6 +254,8 @@ impl Spec { aggregator: &Aggregator, infrastructure: &MithrilInfrastructure, ) -> StdResult { + let mut client = infrastructure.build_client(aggregator).await?; + let expected_epoch_min = target_epoch - 3; let immutable_files_directory = aggregator.db_directory().join("immutable"); let blocks_transactions = @@ -239,128 +266,76 @@ impl Spec { // Verify that mithril stake distribution artifacts are produced and signed correctly { - let hash = - assertions::assert_node_producing_mithril_stake_distribution(aggregator).await?; - let certificate_hash = assertions::assert_signer_is_signing_mithril_stake_distribution( - aggregator, - &hash, - expected_epoch_min, - ) - .await?; - assertions::assert_is_creating_certificate_with_enough_signers( - aggregator, - &certificate_hash, - infrastructure.signers().len(), - ) - .await?; - let mut client = infrastructure.build_client(aggregator).await?; - assertions::assert_client_can_verify_mithril_stake_distribution(&mut client, &hash) + self.toolkit + .check + .mithril_stake_distribution + .is_certified_and_verified( + aggregator, + &mut client, + expected_epoch_min, + infrastructure.signers().len(), + ) .await?; } // Verify that Cardano database snapshot artifacts are produced and signed correctly if self.is_signing_cardano_database { - let hash = - assertions::assert_node_producing_cardano_database_snapshot(aggregator).await?; - let certificate_hash = assertions::assert_signer_is_signing_cardano_database_snapshot( - aggregator, - &hash, - expected_epoch_min, - ) - .await?; - - assertions::assert_is_creating_certificate_with_enough_signers( - aggregator, - &certificate_hash, - infrastructure.signers().len(), - ) - .await?; - - assertions::assert_node_producing_cardano_database_digests_map(aggregator).await?; - - let mut client = infrastructure.build_client(aggregator).await?; - assertions::assert_client_can_verify_cardano_database(&mut client, &hash).await?; + self.toolkit + .check + .cardano_database + .is_certified_and_verified( + aggregator, + &mut client, + expected_epoch_min, + infrastructure.signers().len(), + ) + .await?; } // Verify that Cardano transactions artifacts are produced and signed correctly if self.is_signing_cardano_transactions { - let hash = assertions::assert_node_producing_cardano_transactions(aggregator).await?; - let certificate_hash = assertions::assert_signer_is_signing_cardano_transactions( - aggregator, - &hash, - expected_epoch_min, - ) - .await?; - - assertions::assert_is_creating_certificate_with_enough_signers( - aggregator, - &certificate_hash, - infrastructure.signers().len(), - ) - .await?; - - let mut client = infrastructure.build_client(aggregator).await?; - assertions::assert_client_can_verify_transactions( - &mut client, - transaction_hashes.clone(), - ) - .await?; + self.toolkit + .check + .cardano_transactions + .is_certified_and_verified( + aggregator, + &mut client, + expected_epoch_min, + infrastructure.signers().len(), + transaction_hashes.clone(), + ) + .await?; } // Verify that Cardano blocks transactions artifacts are produced and signed correctly if self.is_signing_cardano_blocks_transactions { - let hash = - assertions::assert_node_producing_cardano_blocks_transactions(aggregator).await?; - let certificate_hash = - assertions::assert_signer_is_signing_cardano_blocks_transactions( + self.toolkit + .check + .cardano_blocks_transactions + .is_certified_and_verified( aggregator, - &hash, + &mut client, expected_epoch_min, + infrastructure.signers().len(), + block_hashes.clone(), + transaction_hashes.clone(), ) .await?; - - assertions::assert_is_creating_certificate_with_enough_signers( - aggregator, - &certificate_hash, - infrastructure.signers().len(), - ) - .await?; - - let mut client = infrastructure.build_client(aggregator).await?; - assertions::assert_client_can_verify_transactions_v2(&mut client, transaction_hashes) - .await?; - - let mut client = infrastructure.build_client(aggregator).await?; - assertions::assert_client_can_verify_blocks(&mut client, block_hashes).await?; } // Verify that Cardano stake distribution artifacts are produced and signed correctly if self.is_signing_cardano_stake_distribution { { - let (hash, epoch) = - assertions::assert_node_producing_cardano_stake_distribution(aggregator) - .await?; - let certificate_hash = - assertions::assert_signer_is_signing_cardano_stake_distribution( + self.toolkit + .check + .cardano_stake_distribution + .is_certified_and_verified( aggregator, - &hash, + &mut client, expected_epoch_min, + infrastructure.signers().len(), ) .await?; - assertions::assert_is_creating_certificate_with_enough_signers( - aggregator, - &certificate_hash, - infrastructure.signers().len(), - ) - .await?; - - let mut client = infrastructure.build_client(aggregator).await?; - assertions::assert_client_can_verify_cardano_stake_distribution( - &mut client, - &hash, - epoch, - ) - .await?; } } diff --git a/mithril-test-lab/mithril-end-to-end/src/scenario/mod.rs b/mithril-test-lab/mithril-end-to-end/src/scenario/mod.rs new file mode 100644 index 00000000000..139d5db22b0 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/scenario/mod.rs @@ -0,0 +1,5 @@ +mod full; +mod run_only; + +pub use full::FullScenario; +pub use run_only::RunOnlyScenario; diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/scenario/run_only.rs similarity index 52% rename from mithril-test-lab/mithril-end-to-end/src/run_only.rs rename to mithril-test-lab/mithril-end-to-end/src/scenario/run_only.rs index abcedfa59c2..180a2b4a68f 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/scenario/run_only.rs @@ -4,15 +4,19 @@ use slog_scope::info; use mithril_common::StdResult; -use crate::{MithrilInfrastructure, assertions}; +use crate::{MithrilInfrastructure, toolkit::ScenarioToolkit}; -pub struct RunOnly { - pub infrastructure: Arc, +pub struct RunOnlyScenario { + toolkit: ScenarioToolkit, + infrastructure: Arc, } -impl RunOnly { - pub fn new(infrastructure: Arc) -> Self { - Self { infrastructure } +impl RunOnlyScenario { + pub fn new(toolkit: ScenarioToolkit, infrastructure: Arc) -> Self { + Self { + toolkit, + infrastructure, + } } pub async fn run(self) -> StdResult<()> { @@ -34,24 +38,29 @@ impl RunOnly { ) -> StdResult<()> { let leader_aggregator = infrastructure.leader_aggregator(); - assertions::wait_for_enough_immutable(leader_aggregator).await?; + self.toolkit.wait.for_enough_immutable(leader_aggregator).await?; let chain_observer = leader_aggregator.chain_observer(); let start_epoch = chain_observer.get_current_epoch().await?.unwrap_or_default(); // Wait 3 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let target_epoch = start_epoch + 3; - assertions::wait_for_aggregator_at_target_epoch( - leader_aggregator, - target_epoch, - "minimal epoch for the aggregator to be able to bootstrap genesis certificate" - .to_string(), - ) - .await?; - assertions::bootstrap_genesis_certificate(leader_aggregator).await?; - assertions::wait_for_epoch_settings(leader_aggregator).await?; + self.toolkit + .wait + .for_aggregator_at_target_epoch( + leader_aggregator, + target_epoch, + "minimal epoch for the aggregator to be able to bootstrap genesis certificate" + .to_string(), + ) + .await?; + self.toolkit + .exec + .bootstrap_genesis_certificate(leader_aggregator) + .await?; + self.toolkit.wait.for_epoch_settings(leader_aggregator).await?; // Transfer some funds on the devnet to have some Cardano transactions to sign - assertions::transfer_funds(infrastructure.devnet()).await?; + self.toolkit.exec.transfer_funds(infrastructure.devnet()).await?; Ok(()) } diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_blocks_transactions.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_blocks_transactions.rs new file mode 100644 index 00000000000..e08d6975d2d --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_blocks_transactions.rs @@ -0,0 +1,197 @@ +use anyhow::{Context, anyhow}; +use slog_scope::{info, warn}; + +use mithril_common::{ + StdResult, + entities::{BlockHash, Epoch, TransactionHash}, + messages::CardanoBlocksTransactionsSnapshotListItemMessage, +}; + +use crate::{ + Aggregator, CardanoBlockCommand, CardanoTransactionV2Command, Client, ClientCommand, + toolkit::{CheckCertificateToolkit, ScenarioToolkitContext}, +}; + +use super::utils; + +#[derive(Debug, Clone)] +pub struct CheckCardanoBlocksTransactionsToolkit { + context: ScenarioToolkitContext, +} + +impl CheckCardanoBlocksTransactionsToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { context } + } + + pub async fn is_certified_and_verified( + &self, + aggregator: &Aggregator, + client: &mut Client, + expected_epoch_min: Epoch, + total_signers_expected: usize, + block_hashes: Vec, + tx_hashes: Vec, + ) -> StdResult<()> { + let certificate_toolkit = CheckCertificateToolkit::new(self.context.clone()); + + let artifact = self.wait_for_artifact(aggregator).await?; + self.check_artifact(&artifact, expected_epoch_min)?; + certificate_toolkit + .is_creating_certificate_with_enough_signers( + aggregator, + &artifact.certificate_hash, + total_signers_expected, + ) + .await?; + self.verify_transactions_with_client(client, tx_hashes).await?; + self.verify_blocks_with_client(client, block_hashes).await?; + + Ok(()) + } + + pub async fn wait_for_artifact( + &self, + aggregator: &Aggregator, + ) -> StdResult { + utils::wait_for_latest_artifact::( + "Cardano blocks transactions", + "/artifact/cardano-blocks-transactions", + |a| a.hash.clone(), + &self.context, + aggregator, + ) + .await + } + + pub fn check_artifact( + &self, + artifact: &CardanoBlocksTransactionsSnapshotListItemMessage, + expected_epoch_min: Epoch, + ) -> StdResult<()> { + utils::assert_minimal_epoch(artifact, |a| a.epoch, expected_epoch_min) + } + + pub async fn verify_transactions_with_client( + &self, + client: &mut Client, + tx_hashes: Vec, + ) -> StdResult<()> { + #[allow(dead_code)] + #[derive(Debug, serde::Deserialize)] + struct ClientCardanoTransactionCertifyResult { + certified_transactions: Vec, + non_certified_transactions: Vec, + } + + #[derive(Debug, serde::Deserialize)] + struct CertifiedTransactionV2 { + transaction_hash: String, + } + + if !client.version().is_above_or_equal("0.13.1") { + warn!( + "Client version is below 0.13.1, skipping `cardano-transaction certify --backend v2` check" + ); + return Ok(()); + } + + let result_file = client + .run(ClientCommand::CardanoTransactionV2( + CardanoTransactionV2Command::Certify { + tx_hashes: tx_hashes.clone(), + }, + )) + .await?; + info!("Client verified the Cardano transactions V2"; "tx_hashes" => ?tx_hashes); + + let file = std::fs::read_to_string(&result_file).with_context(|| { + format!( + "Failed to read client output from file `{}`", + result_file.display() + ) + })?; + let result: ClientCardanoTransactionCertifyResult = serde_json::from_str(&file) + .with_context(|| { + format!( + "Failed to parse client output as json from file `{}`", + result_file.display() + ) + })?; + + info!("Asserting that all Cardano transactions V2 were verified by the Client..."); + let certified_tx_hashes_result: Vec = result + .certified_transactions + .iter() + .map(|tx| tx.transaction_hash.clone()) + .collect(); + + if tx_hashes.iter().all(|tx| certified_tx_hashes_result.contains(tx)) { + Ok(()) + } else { + Err(anyhow!( + "Not all transactions V2 were certified:\n'{:#?}'", + result, + )) + } + } + + pub async fn verify_blocks_with_client( + &self, + client: &mut Client, + block_hashes: Vec, + ) -> StdResult<()> { + #[allow(dead_code)] + #[derive(Debug, serde::Deserialize)] + struct ClientCardanoBlockCertifyResult { + certified_blocks: Vec, + non_certified_blocks: Vec, + } + + #[derive(Debug, serde::Deserialize)] + struct CertifiedBlock { + block_hash: String, + } + + if !client.version().is_above_or_equal("0.13.1") { + warn!("Client version is below 0.13.1, skipping `cardano-block certify` check"); + return Ok(()); + } + + let result_file = client + .run(ClientCommand::CardanoBlock(CardanoBlockCommand::Certify { + block_hashes: block_hashes.clone(), + })) + .await?; + info!("Client verified the Cardano blocks"; "block_hashes" => ?block_hashes); + let file = std::fs::read_to_string(&result_file).with_context(|| { + format!( + "Failed to read client output from file `{}`", + result_file.display() + ) + })?; + let result: ClientCardanoBlockCertifyResult = + serde_json::from_str(&file).with_context(|| { + format!( + "Failed to parse client output as json from file `{}`", + result_file.display() + ) + })?; + + info!("Asserting that all Cardano blocks were verified by the Client..."); + let certified_blocks_hashes_result: Vec = result + .certified_blocks + .iter() + .map(|block| block.block_hash.clone()) + .collect(); + + if block_hashes + .iter() + .all(|block| certified_blocks_hashes_result.contains(block)) + { + Ok(()) + } else { + Err(anyhow!("Not all blocks were certified:\n'{:#?}'", result,)) + } + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_database.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_database.rs new file mode 100644 index 00000000000..5899aa5c707 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_database.rs @@ -0,0 +1,156 @@ +use anyhow::{Context, anyhow}; +use slog_scope::{info, warn}; + +use mithril_common::{ + StdResult, + entities::{Epoch, EpochSpecifier}, + messages::{CardanoDatabaseDigestListMessage, CardanoDatabaseSnapshotListItemMessage}, +}; + +use crate::{ + Aggregator, CardanoDbV2Command, Client, ClientCommand, attempt, + toolkit::{CheckCertificateToolkit, ScenarioToolkitContext}, + utils::AttemptResult, +}; + +use super::utils; + +#[derive(Debug, Clone)] +pub struct CheckCardanoDatabaseToolkit { + context: ScenarioToolkitContext, +} + +impl CheckCardanoDatabaseToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { context } + } + + pub async fn is_certified_and_verified( + &self, + aggregator: &Aggregator, + client: &mut Client, + expected_epoch_min: Epoch, + total_signers_expected: usize, + ) -> StdResult<()> { + let certificate_toolkit = CheckCertificateToolkit::new(self.context.clone()); + + let artifact = self.wait_for_artifact(aggregator).await?; + self.check_artifact(&artifact, expected_epoch_min)?; + certificate_toolkit + .is_creating_certificate_with_enough_signers( + aggregator, + &artifact.certificate_hash, + total_signers_expected, + ) + .await?; + self.node_producing_cardano_database_digests_map(aggregator).await?; + self.verify_with_client(client, &artifact.hash).await?; + + Ok(()) + } + + pub async fn wait_for_artifact( + &self, + aggregator: &Aggregator, + ) -> StdResult { + utils::wait_for_latest_artifact::( + "Cardano database snapshot", + "/artifact/cardano-database", + |a| a.hash.clone(), + &self.context, + aggregator, + ) + .await + } + + pub fn check_artifact( + &self, + artifact: &CardanoDatabaseSnapshotListItemMessage, + expected_epoch_min: Epoch, + ) -> StdResult<()> { + utils::assert_minimal_epoch(artifact, |a| a.beacon.epoch, expected_epoch_min) + } + + pub async fn node_producing_cardano_database_digests_map( + &self, + aggregator: &Aggregator, + ) -> StdResult> { + let url = format!( + "{}/artifact/cardano-database/digests", + aggregator.endpoint() + ); + info!("Waiting for the aggregator to produce a Cardano database digests map"; "aggregator" => &aggregator.name()); + + async fn fetch_cardano_database_digests_map( + url: String, + ) -> StdResult>> { + match utils::get_json_response::(url) + .await? + .as_deref() + { + Ok(&[]) => Ok(None), + Ok(cardano_database_digests_map) => Ok(Some( + cardano_database_digests_map + .iter() + .map(|item| (item.immutable_file_name.clone(), item.digest.clone())) + .collect(), + )), + Err(err) => Err(anyhow!("Invalid Cardano database digests map body: {err}",)), + } + } + + match attempt!(30, self.context.tenth_epoch_delay(), { + fetch_cardano_database_digests_map(url.clone()).await + }) { + AttemptResult::Ok(cardano_database_digests_map) => { + info!("Aggregator produced a Cardano database digests map"; "total_digests" => &cardano_database_digests_map.len(), "aggregator" => &aggregator.name()); + Ok(cardano_database_digests_map) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(anyhow!( + "Timeout exhausted assert_node_producing_cardano_database_digests_map, no response from `{url}`" + )), + }.with_context(|| { + format!( + "Requesting aggregator `{}`", + aggregator.name() + ) + }) + } + + pub async fn verify_with_client(&self, client: &mut Client, hash: &str) -> StdResult<()> { + client + .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::List)) + .await?; + + if client.version().is_above_or_equal("0.12.34") { + client + .run(ClientCommand::CardanoDbV2( + CardanoDbV2Command::ListPerEpoch { + epoch_specifier: EpochSpecifier::LatestMinusOffset(5), + }, + )) + .await?; + } else { + warn!( + "Client version is below 0.12.34, skipping `cardano-db snapshot list --epoch latest-5` check" + ); + } + + client + .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::Show { + hash: hash.to_string(), + })) + .await?; + info!("Client list & show the cardano database snapshot"; "hash" => &hash); + + client + .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::Download { + hash: hash.to_string(), + })) + .await?; + info!("Client downloaded & restored the cardano database snapshot"; "hash" => &hash); + + Ok(()) + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_stake_distribution.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_stake_distribution.rs new file mode 100644 index 00000000000..e9b5cf1a5bf --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_stake_distribution.rs @@ -0,0 +1,96 @@ +use slog_scope::info; + +use mithril_common::{ + StdResult, entities::Epoch, messages::CardanoStakeDistributionListItemMessage, +}; + +use crate::{ + Aggregator, CardanoStakeDistributionCommand, Client, ClientCommand, + toolkit::{CheckCertificateToolkit, ScenarioToolkitContext}, +}; + +use super::utils; + +#[derive(Debug, Clone)] +pub struct CheckCardanoStakeDistributionToolkit { + context: ScenarioToolkitContext, +} + +impl CheckCardanoStakeDistributionToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { context } + } + + pub async fn is_certified_and_verified( + &self, + aggregator: &Aggregator, + client: &mut Client, + expected_epoch_min: Epoch, + total_signers_expected: usize, + ) -> StdResult<()> { + let certificate_toolkit = CheckCertificateToolkit::new(self.context.clone()); + + let artifact = self.wait_for_artifact(aggregator).await?; + self.check_artifact(&artifact, expected_epoch_min)?; + certificate_toolkit + .is_creating_certificate_with_enough_signers( + aggregator, + &artifact.certificate_hash, + total_signers_expected, + ) + .await?; + self.verify_with_client(client, &artifact.hash, artifact.epoch) + .await?; + + Ok(()) + } + + pub async fn wait_for_artifact( + &self, + aggregator: &Aggregator, + ) -> StdResult { + utils::wait_for_latest_artifact::( + "Cardano stake distribution", + "/artifact/cardano-stake-distributions", + |a| a.hash.clone(), + &self.context, + aggregator, + ) + .await + } + + pub fn check_artifact( + &self, + artifact: &CardanoStakeDistributionListItemMessage, + expected_epoch_min: Epoch, + ) -> StdResult<()> { + utils::assert_minimal_epoch(artifact, |a| a.epoch, expected_epoch_min) + } + + pub async fn verify_with_client( + &self, + client: &mut Client, + hash: &str, + epoch: Epoch, + ) -> StdResult<()> { + client + .run(ClientCommand::CardanoStakeDistribution( + CardanoStakeDistributionCommand::Download { + unique_identifier: epoch.to_string(), + }, + )) + .await?; + info!("Client downloaded the Cardano stake distribution by epoch"; "epoch" => epoch.to_string()); + + client + .run(ClientCommand::CardanoStakeDistribution( + CardanoStakeDistributionCommand::Download { + unique_identifier: hash.to_string(), + }, + )) + .await?; + info!("Client downloaded the Cardano stake distribution by hash"; "hash" => hash.to_string()); + + Ok(()) + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_transactions.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_transactions.rs new file mode 100644 index 00000000000..9fe96d7b5bd --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_transactions.rs @@ -0,0 +1,118 @@ +use anyhow::{Context, anyhow}; +use slog_scope::info; + +use mithril_common::{ + StdResult, + entities::{Epoch, TransactionHash}, + messages::CardanoTransactionSnapshotListItemMessage, +}; + +use crate::{ + Aggregator, CardanoTransactionCommand, Client, ClientCommand, + toolkit::{CheckCertificateToolkit, ScenarioToolkitContext}, +}; + +use super::utils; + +#[derive(Debug, Clone)] +pub struct CheckCardanoTransactionsToolkit { + context: ScenarioToolkitContext, +} + +impl CheckCardanoTransactionsToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { context } + } + + pub async fn is_certified_and_verified( + &self, + aggregator: &Aggregator, + client: &mut Client, + expected_epoch_min: Epoch, + total_signers_expected: usize, + tx_hashes: Vec, + ) -> StdResult<()> { + let certificate_toolkit = CheckCertificateToolkit::new(self.context.clone()); + + let artifact = self.wait_for_artifact(aggregator).await?; + self.check_artifact(&artifact, expected_epoch_min)?; + certificate_toolkit + .is_creating_certificate_with_enough_signers( + aggregator, + &artifact.certificate_hash, + total_signers_expected, + ) + .await?; + self.verify_with_client(client, tx_hashes).await?; + + Ok(()) + } + + pub async fn wait_for_artifact( + &self, + aggregator: &Aggregator, + ) -> StdResult { + utils::wait_for_latest_artifact::( + "Cardano transactions", + "/artifact/cardano-transactions", + |a| a.hash.clone(), + &self.context, + aggregator, + ) + .await + } + + pub fn check_artifact( + &self, + artifact: &CardanoTransactionSnapshotListItemMessage, + expected_epoch_min: Epoch, + ) -> StdResult<()> { + utils::assert_minimal_epoch(artifact, |a| a.epoch, expected_epoch_min) + } + + pub async fn verify_with_client( + &self, + client: &mut Client, + tx_hashes: Vec, + ) -> StdResult<()> { + #[allow(dead_code)] + #[derive(Debug, serde::Deserialize)] + struct ClientCardanoTransactionCertifyResult { + certified_transactions: Vec, + non_certified_transactions: Vec, + } + + let result_file = client + .run(ClientCommand::CardanoTransaction( + CardanoTransactionCommand::Certify { + tx_hashes: tx_hashes.clone(), + }, + )) + .await?; + info!("Client verified the Cardano transactions"; "tx_hashes" => ?tx_hashes); + + let file = std::fs::read_to_string(&result_file).with_context(|| { + format!( + "Failed to read client output from file `{}`", + result_file.display() + ) + })?; + let result: ClientCardanoTransactionCertifyResult = serde_json::from_str(&file) + .with_context(|| { + format!( + "Failed to parse client output as json from file `{}`", + result_file.display() + ) + })?; + + info!("Asserting that all Cardano transactions were verified by the Client..."); + if tx_hashes.iter().all(|tx| result.certified_transactions.contains(tx)) { + Ok(()) + } else { + Err(anyhow!( + "Not all transactions were certified:\n'{:#?}'", + result, + )) + } + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/certificate.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/certificate.rs new file mode 100644 index 00000000000..9bccb0b9071 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/certificate.rs @@ -0,0 +1,64 @@ +use anyhow::{Context, anyhow}; +use slog_scope::info; + +use mithril_common::{StdResult, messages::CertificateMessage}; + +use crate::{Aggregator, attempt, toolkit::ScenarioToolkitContext, utils::AttemptResult}; + +use super::utils; + +#[derive(Debug, Clone)] +pub struct CheckCertificateToolkit { + context: ScenarioToolkitContext, +} + +impl CheckCertificateToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { context } + } + + pub async fn is_creating_certificate_with_enough_signers( + &self, + aggregator: &Aggregator, + certificate_hash: &str, + total_signers_expected: usize, + ) -> StdResult<()> { + let url = format!("{}/certificate/{certificate_hash}", aggregator.endpoint()); + info!("Waiting for the aggregator to create a certificate"; "aggregator" => &aggregator.name()); + + async fn fetch_certificate_message(url: String) -> StdResult> { + match utils::get_json_response::(url).await? { + Ok(certificate) => Ok(Some(certificate)), + Err(err) => Err(anyhow!(err).context("Invalid snapshot body")), + } + } + + match attempt!(10, self.context.tenth_epoch_delay(), { + fetch_certificate_message(url.clone()).await + }) { + AttemptResult::Ok(certificate) => { + info!("Aggregator produced a certificate"; "certificate" => ?certificate); + if certificate.metadata.signers.len() == total_signers_expected { + info!( + "Certificate is signed by expected number of signers: {} >= {} ", + certificate.metadata.signers.len(), + total_signers_expected ; + "aggregator" => &aggregator.name() + ); + Ok(()) + } else { + Err(anyhow!( + "Certificate is not signed by expected number of signers: {} < {} ", + certificate.metadata.signers.len(), + total_signers_expected + )) + } + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(anyhow!( + "Timeout exhausted assert_is_creating_certificate, no response from `{url}`" + )), + } + .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/mithril_stake_distribution.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/mithril_stake_distribution.rs new file mode 100644 index 00000000000..757b177a577 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/mithril_stake_distribution.rs @@ -0,0 +1,81 @@ +use slog_scope::info; + +use mithril_common::{ + StdResult, entities::Epoch, messages::MithrilStakeDistributionListItemMessage, +}; + +use crate::{ + Aggregator, Client, ClientCommand, MithrilStakeDistributionCommand, + toolkit::{CheckCertificateToolkit, ScenarioToolkitContext}, +}; + +use super::utils; + +#[derive(Debug, Clone)] +pub struct CheckMithrilStakeDistributionToolkit { + context: ScenarioToolkitContext, +} + +impl CheckMithrilStakeDistributionToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { context } + } + + pub async fn is_certified_and_verified( + &self, + aggregator: &Aggregator, + client: &mut Client, + expected_epoch_min: Epoch, + total_signers_expected: usize, + ) -> StdResult<()> { + let certificate_toolkit = CheckCertificateToolkit::new(self.context.clone()); + + let artifact = self.wait_for_artifact(aggregator).await?; + self.check_artifact(&artifact, expected_epoch_min)?; + certificate_toolkit + .is_creating_certificate_with_enough_signers( + aggregator, + &artifact.certificate_hash, + total_signers_expected, + ) + .await?; + self.verify_with_client(client, &artifact.hash).await?; + + Ok(()) + } + + pub async fn wait_for_artifact( + &self, + aggregator: &Aggregator, + ) -> StdResult { + utils::wait_for_latest_artifact::( + "Mithril stake distribution", + "/artifact/mithril-stake-distributions", + |a| a.hash.clone(), + &self.context, + aggregator, + ) + .await + } + + pub fn check_artifact( + &self, + artifact: &MithrilStakeDistributionListItemMessage, + expected_epoch_min: Epoch, + ) -> StdResult<()> { + utils::assert_minimal_epoch(artifact, |a| a.epoch, expected_epoch_min) + } + + pub async fn verify_with_client(&self, client: &mut Client, hash: &str) -> StdResult<()> { + client + .run(ClientCommand::MithrilStakeDistribution( + MithrilStakeDistributionCommand::Download { + hash: hash.to_owned(), + }, + )) + .await?; + info!("Client downloaded the Mithril stake distribution"; "hash" => &hash); + + Ok(()) + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/mod.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/mod.rs new file mode 100644 index 00000000000..c197484fb40 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/mod.rs @@ -0,0 +1,16 @@ +mod cardano_blocks_transactions; +mod cardano_database; +mod cardano_stake_distribution; +mod cardano_transactions; +mod certificate; +mod mithril_stake_distribution; +mod toolkit; +mod utils; + +pub use cardano_blocks_transactions::*; +pub use cardano_database::*; +pub use cardano_stake_distribution::*; +pub use cardano_transactions::*; +pub use certificate::*; +pub use mithril_stake_distribution::*; +pub use toolkit::*; diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/toolkit.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/toolkit.rs new file mode 100644 index 00000000000..0c273a20f6d --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/toolkit.rs @@ -0,0 +1,88 @@ +use std::path::PathBuf; + +use anyhow::Context; +use mithril_common::StdResult; +use slog_scope::{info, warn}; + +use crate::toolkit::{ + CheckCardanoBlocksTransactionsToolkit, CheckCardanoDatabaseToolkit, + CheckCardanoStakeDistributionToolkit, CheckCardanoTransactionsToolkit, CheckCertificateToolkit, + CheckMithrilStakeDistributionToolkit, ScenarioToolkitContext, +}; +use crate::utils::file_utils::copy_dir_all; +use crate::{Client, ClientCommand, FullNode, NodeVersion, ToolsCommand, UtxoHdCommand}; + +#[derive(Debug, Clone)] +pub struct CheckToolkit { + pub cardano_blocks_transactions: CheckCardanoBlocksTransactionsToolkit, + pub cardano_database: CheckCardanoDatabaseToolkit, + pub cardano_stake_distribution: CheckCardanoStakeDistributionToolkit, + pub cardano_transactions: CheckCardanoTransactionsToolkit, + pub certificate: CheckCertificateToolkit, + pub mithril_stake_distribution: CheckMithrilStakeDistributionToolkit, +} + +impl CheckToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { + cardano_blocks_transactions: CheckCardanoBlocksTransactionsToolkit::new( + context.clone(), + ), + cardano_database: CheckCardanoDatabaseToolkit::new(context.clone()), + cardano_stake_distribution: CheckCardanoStakeDistributionToolkit::new(context.clone()), + cardano_transactions: CheckCardanoTransactionsToolkit::new(context.clone()), + certificate: CheckCertificateToolkit::new(context.clone()), + mithril_stake_distribution: CheckMithrilStakeDistributionToolkit::new(context.clone()), + } + } + + pub async fn client_can_convert_the_ledger_snapshot( + &self, + client: &mut Client, + full_node: &FullNode, + artifacts_dir: PathBuf, + cardano_node_version: NodeVersion, + ) -> StdResult<()> { + if client.version().is_below("0.13.10") { + warn!("Client version is below 0.13.10, skipping snapshot conversion check"); + return Ok(()); + } + + let utxo_hd_flavor = if cardano_node_version.is_below("10.7.0") { + "LMDB" + } else { + "LSM" + }; + + let binary_path = artifacts_dir.join("bin").join("snapshot-converter"); + + // copy the db to another temporary location to avoid any risk of modifying the original one during the conversion process + let db_to_convert = artifacts_dir.join("db_to_convert"); + copy_dir_all(&full_node.db_path, &db_to_convert).with_context(|| { + format!( + "Failed to copy the ledger state database from `{}` to `{}` for the snapshot conversion process", + full_node.db_path.display(), + db_to_convert.display() + ) + })?; + + client + .run(ClientCommand::Tools(ToolsCommand::UtxoHd( + UtxoHdCommand::SnapshotConverter { + db_directory: db_to_convert.to_string_lossy().to_string(), + cardano_node_version: cardano_node_version.to_string(), + binary_path: binary_path.to_string_lossy().to_string(), + config_path: full_node + .snapshot_converter_config_path + .to_string_lossy() + .to_string(), + utxo_hd_flavor: utxo_hd_flavor.to_string(), + commit: true, + }, + ))) + .await?; + info!("Client converted the ledger state into {utxo_hd_flavor} format"); + + Ok(()) + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/utils.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/utils.rs new file mode 100644 index 00000000000..1d2afcf3bce --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/utils.rs @@ -0,0 +1,73 @@ +use anyhow::{Context, anyhow}; +use reqwest::StatusCode; +use serde::de::DeserializeOwned; +use slog_scope::info; + +use mithril_common::{StdResult, entities::Epoch}; + +use crate::{Aggregator, attempt, toolkit::ScenarioToolkitContext, utils::AttemptResult}; + +pub async fn get_json_response(url: String) -> StdResult> { + match reqwest::get(url.clone()).await { + Ok(response) => { + let r = response.status(); + match r { + StatusCode::OK => Ok(response.json::().await), + s => Err(anyhow!("Unexpected status code from Aggregator: {s}")), + } + } + Err(err) => Err(anyhow!(err).context(format!("Request to `{url}` failed"))), + } +} + +/// Wait until the aggregator produces an artifact, returning the latest one +/// +/// Note: the `artifact_list_url` must start with a `/` +pub async fn wait_for_latest_artifact( + artifact_name: &str, + artifact_list_url: &str, + hash_extractor: fn(&T) -> String, + context: &ScenarioToolkitContext, + aggregator: &Aggregator, +) -> StdResult { + let url = format!("{}{artifact_list_url}", aggregator.endpoint()); + info!("Waiting for the aggregator to produce a {artifact_name} artifact"; "aggregator" => &aggregator.name()); + + async fn fetch_last_artifact( + artifact_name: &str, + url: String, + ) -> StdResult> { + match get_json_response::>(url).await? { + // Artifact lists are sorted from newest to oldest, so the first item is the latest + Ok(list) => Ok(list.into_iter().next()), + Err(err) => Err(anyhow!("Invalid {artifact_name} artifact body: {err}",)), + } + } + + match attempt!(30, context.tenth_epoch_delay(), { + fetch_last_artifact(artifact_name, url.clone()).await + }) { + AttemptResult::Ok(last_artifact) => { + info!("Aggregator produced a {artifact_name} artifact"; "hash" => hash_extractor(&last_artifact), "aggregator" => &aggregator.name()); + Ok(last_artifact) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(anyhow!( + "Timeout exhausted waiting for {artifact_name}, no response from `{url}`" + )), + } + .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) +} + +pub fn assert_minimal_epoch( + artifact: &T, + epoch_extractor: fn(&T) -> Epoch, + expected_epoch_min: Epoch, +) -> StdResult<()> { + match epoch_extractor(artifact) { + epoch if epoch >= expected_epoch_min => Ok(()), + epoch => Err(anyhow!( + "Minimum expected artifact epoch not reached: {epoch} < {expected_epoch_min}" + )), + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/context.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/context.rs new file mode 100644 index 00000000000..0ad57309833 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/context.rs @@ -0,0 +1,80 @@ +use std::time::Duration; + +#[derive(Debug, Clone)] +pub struct ScenarioToolkitContext { + attempt_policy: AttemptPolicy, +} + +impl ScenarioToolkitContext { + pub fn new(attempt_policy: AttemptPolicy) -> Self { + Self { attempt_policy } + } + + pub fn new_from_cardano_epoch(slot_length_in_s: f64, number_of_slot_per_epoch: f64) -> Self { + Self { + attempt_policy: AttemptPolicy::from_cardano_epoch( + slot_length_in_s, + number_of_slot_per_epoch, + ), + } + } + + pub fn attempt_policy(&self) -> AttemptPolicy { + self.attempt_policy + } + + pub fn tenth_epoch_delay(&self) -> Duration { + self.attempt_policy.delay(0.10) + } + + pub fn half_epoch_delay(&self) -> Duration { + self.attempt_policy.delay(0.5) + } + + pub fn full_epoch_delay(&self) -> Duration { + self.attempt_policy.epoch_duration + } +} + +#[derive(Debug, Clone, Copy)] +pub struct AttemptPolicy { + epoch_duration: Duration, +} + +impl AttemptPolicy { + pub const fn new(base_duration: Duration) -> Self { + Self { + epoch_duration: base_duration, + } + } + + pub fn from_cardano_epoch(slot_length_in_s: f64, number_of_slot_per_epoch: f64) -> Self { + Self::new(Duration::from_secs_f64( + slot_length_in_s * number_of_slot_per_epoch, + )) + } + + pub fn delay(self, multiplier: f32) -> Duration { + self.epoch_duration.mul_f32(multiplier) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn building_attempt_policy_from_cardano_epoch() { + let slot_length_in_s = 0.5; + let number_of_slot_per_epoch = 10.0; + let policy = AttemptPolicy::from_cardano_epoch(slot_length_in_s, number_of_slot_per_epoch); + assert_eq!(policy.epoch_duration, Duration::from_secs(5)); + } + + #[test] + fn delay_calculation() { + let policy = AttemptPolicy::new(Duration::from_secs(10)); + assert_eq!(policy.delay(0.5), Duration::from_secs(5)); + assert_eq!(policy.delay(2.0), Duration::from_secs(20)); + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/exec.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/exec.rs new file mode 100644 index 00000000000..34dfb16d104 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/exec.rs @@ -0,0 +1,125 @@ +use std::path::PathBuf; + +use anyhow::Context; +use slog_scope::info; + +use mithril_common::StdResult; +use mithril_common::entities::{Epoch, ProtocolParameters}; +use mithril_common::messages::AggregatorStatusMessage; + +use crate::toolkit::ScenarioToolkitContext; +use crate::{AggregateSignatureType, Aggregator, Devnet}; + +#[derive(Debug, Clone)] +pub struct ExecToolkit { + _context: ScenarioToolkitContext, +} + +impl ExecToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { _context: context } + } + + /// Retrieve the current Mithril era from a running aggregator by querying its `/status` route. + pub async fn retrieve_current_era(&self, aggregator: &Aggregator) -> StdResult { + let url = format!("{}/status", aggregator.endpoint()); + let response = reqwest::get(&url) + .await + .with_context(|| format!("Failed to query aggregator status at `{url}`"))?; + let status_message: AggregatorStatusMessage = response + .json() + .await + .with_context(|| "Failed to parse aggregator status response")?; + + Ok(status_message.mithril_era.to_string()) + } + + pub async fn bootstrap_genesis_certificate(&self, aggregator: &Aggregator) -> StdResult<()> { + info!("Bootstrap genesis certificate"; "aggregator" => &aggregator.name()); + info!("> retrieving current era from aggregator"; "aggregator" => &aggregator.name()); + let mithril_era = self.retrieve_current_era(aggregator).await?; + info!("> stopping aggregator"; "aggregator" => &aggregator.name()); + aggregator.stop().await?; + info!("> bootstrapping genesis using signers registered two epochs ago..."; "aggregator" => &aggregator.name()); + aggregator.bootstrap_genesis(&mithril_era).await?; + info!("> done, restarting aggregator"; "aggregator" => &aggregator.name()); + aggregator.serve().await?; + + Ok(()) + } + + pub async fn register_era_marker( + &self, + aggregator: &Aggregator, + devnet: &Devnet, + mithril_era: &str, + era_epoch: Epoch, + ) -> StdResult<()> { + info!("Register '{mithril_era}' era marker"; "aggregator" => &aggregator.name()); + + info!("> generating era marker tx datum..."; "aggregator" => &aggregator.name()); + let tx_datum_file_path = devnet + .artifacts_dir() + .join(PathBuf::from("era-tx-datum.txt".to_string())); + aggregator + .era_generate_tx_datum(&tx_datum_file_path, mithril_era, era_epoch) + .await?; + + info!("> writing '{mithril_era}' era marker on the Cardano chain..."; "aggregator" => &aggregator.name()); + devnet.write_era_marker(&tx_datum_file_path).await?; + + Ok(()) + } + + pub async fn delegate_stakes_to_pools( + &self, + devnet: &Devnet, + delegation_round: u16, + ) -> StdResult<()> { + info!("Delegate stakes to the cardano pools"); + + devnet.delegate_stakes(delegation_round).await?; + + Ok(()) + } + + pub async fn transfer_funds(&self, devnet: &Devnet) -> StdResult<()> { + info!("Transfer funds on the devnet"); + + devnet.transfer_funds().await?; + + Ok(()) + } + + pub async fn update_protocol_parameters( + &self, + aggregator: &Aggregator, + aggregate_signature_type: AggregateSignatureType, + ) -> StdResult<()> { + info!("Update protocol parameters"; "aggregator" => &aggregator.name()); + + info!("> stopping aggregator"); + aggregator.stop().await?; + let protocol_parameters_new = match aggregate_signature_type { + AggregateSignatureType::Concatenation => ProtocolParameters { + k: 145, + m: 210, + phi_f: 0.80, + }, + AggregateSignatureType::Snark => ProtocolParameters { + k: 7, + m: 10, + phi_f: 0.95, + }, + }; + + info!( + "> updating protocol parameters to {protocol_parameters_new:?}..."; "aggregator" => &aggregator.name() + ); + aggregator.set_protocol_parameters(&protocol_parameters_new).await; + info!("> done, restarting aggregator"; "aggregator" => &aggregator.name()); + aggregator.serve().await?; + + Ok(()) + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/mod.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/mod.rs new file mode 100644 index 00000000000..64513f3855e --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/mod.rs @@ -0,0 +1,26 @@ +mod check; +mod context; +mod exec; +mod wait; + +pub use check::*; +pub use context::*; +pub use exec::*; +pub use wait::*; + +#[derive(Debug, Clone)] +pub struct ScenarioToolkit { + pub check: CheckToolkit, + pub exec: ExecToolkit, + pub wait: WaitToolkit, +} + +impl ScenarioToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { + check: CheckToolkit::new(context.clone()), + exec: ExecToolkit::new(context.clone()), + wait: WaitToolkit::new(context), + } + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/wait.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/wait.rs new file mode 100644 index 00000000000..d345620169a --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/wait.rs @@ -0,0 +1,124 @@ +use anyhow::{Context, anyhow}; +use reqwest::StatusCode; +use slog_scope::{info, warn}; + +use mithril_cardano_node_internal_database::entities::ImmutableFile; +use mithril_common::{StdResult, entities::Epoch, messages::EpochSettingsMessage}; + +use crate::{Aggregator, attempt, toolkit::ScenarioToolkitContext, utils::AttemptResult}; + +#[derive(Debug, Clone)] +pub struct WaitToolkit { + context: ScenarioToolkitContext, +} + +impl WaitToolkit { + pub fn new(context: ScenarioToolkitContext) -> Self { + Self { context } + } + + pub async fn for_enough_immutable(&self, aggregator: &Aggregator) -> StdResult<()> { + info!("Waiting that enough immutable have been written in the devnet"; "aggregator" => aggregator.name()); + + let db_directory = aggregator.db_directory(); + match attempt!(20, self.context.half_epoch_delay(), { + match ImmutableFile::list_completed_in_dir(db_directory) + .with_context(|| { + format!( + "Immutable file listing failed in dir `{}`", + db_directory.display(), + ) + })? + .last() + { + Some(_) => Ok(Some(())), + None => Ok(None), + } + }) { + AttemptResult::Ok(_) => Ok(()), + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(anyhow!( + "Timeout exhausted for enough immutable to be written in `{}`", + db_directory.display() + )), + } + } + + pub async fn for_epoch_settings( + &self, + aggregator: &Aggregator, + ) -> StdResult { + let aggregator_endpoint = aggregator.endpoint(); + let url = format!("{aggregator_endpoint}/epoch-settings"); + info!("Waiting for the aggregator to expose epoch settings"; "aggregator" => aggregator.name()); + + match attempt!(20, self.context.half_epoch_delay(), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => { + let epoch_settings = response + .json::() + .await + .with_context(|| "Invalid EpochSettings body")?; + info!("Aggregator ready"; "epoch_settings" => ?epoch_settings); + Ok(Some(epoch_settings)) + } + s if s.is_server_error() => { + warn!( "Server error while waiting for the Aggregator, http code: {s}"; "aggregator" => aggregator.name()); + Ok(None) + } + _ => Ok(None), + }, + Err(_) => Ok(None), + } + }) { + AttemptResult::Ok(epoch_settings) => Ok(epoch_settings), + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(anyhow!( + "Timeout exhausted for aggregator to be up, no response from `{url}`" + )), + } + } + + pub async fn for_aggregator_at_target_epoch( + &self, + aggregator: &Aggregator, + target_epoch: Epoch, + wait_reason: String, + ) -> StdResult<()> { + info!( + "Waiting for the cardano network to be at the target epoch: {}", wait_reason; + "aggregator" => aggregator.name(), + "target_epoch" => ?target_epoch + ); + + match attempt!(90, self.context.half_epoch_delay(), { + match aggregator + .chain_observer() + .get_current_epoch() + .await + .with_context(|| "Could not query current epoch")? + { + Some(epoch) => { + if epoch >= target_epoch { + Ok(Some(())) + } else { + Ok(None) + } + } + None => Ok(None), + } + }) { + AttemptResult::Ok(_) => { + info!("Target epoch reached!"; "aggregator" => aggregator.name(), "target_epoch" => ?target_epoch); + Ok(()) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => { + Err(anyhow!("Timeout exhausted for target epoch to be reached")) + } + }?; + + Ok(()) + } +}