apollo_integration_tests: add shared proof-flow infrastructure (#13760)

Author: einat-starkware

Cargo.lock

@@ -34,6 +34,7 @@ pub enum TestIdentifier {
     RevertedL1HandlerTx,
     InfraUnitTests,
     PositiveFlowIntegrationTest,
+    ProofFlowIntegrationTest,
     RestartFlowIntegrationTest,
     RestartServiceMultipleNodesFlowIntegrationTest,
     RestartServiceSingleNodeFlowIntegrationTest,

crates/apollo_infra_utils/src/test_utils.rs

@@ -20,6 +20,7 @@ apollo_batcher.workspace = true
 apollo_batcher_config.workspace = true
 apollo_class_manager = { workspace = true, features = ["testing"] }
 apollo_class_manager_config.workspace = true
+apollo_committer = { workspace = true, features = ["testing"] }
 apollo_committer_config.workspace = true
 apollo_config.workspace = true
 apollo_config_manager_config = { workspace = true, features = ["testing"] }
@@ -75,6 +76,8 @@ serde.workspace = true
 serde_json.workspace = true
 starknet-types-core.workspace = true
 starknet_api.workspace = true
+starknet_committer.workspace = true
+starknet_patricia_storage.workspace = true
 static_assertions.workspace = true
 strum.workspace = true
 tempfile.workspace = true

crates/apollo_integration_tests/Cargo.toml

@@ -57,7 +57,7 @@ use tokio::sync::Mutex;
 use tracing::{debug, instrument, Instrument};
 use url::Url;
 
-use crate::state_reader::{StorageTestHandles, StorageTestSetup};
+use crate::state_reader::{PresetTestContracts, StorageTestHandles, StorageTestSetup};
 use crate::utils::{
     create_consensus_manager_configs_from_network_configs,
     create_mempool_p2p_configs,
@@ -248,7 +248,7 @@ impl FlowSequencerSetup {
     ) -> Self {
         let path = None;
         let StorageTestSetup { storage_config, storage_handles } =
-            StorageTestSetup::new(accounts, &chain_info, path);
+            StorageTestSetup::new(accounts, &chain_info, path, PresetTestContracts::new(), None);
 
         let (recorder_url, _join_handle) =
             spawn_local_success_recorder(available_ports.get_next_port());
@@ -285,6 +285,7 @@ impl FlowSequencerSetup {
             block_max_capacity_gas,
             validator_id,
             allow_bootstrap_txs,
+            true,
         );
         let num_l1_txs = u64::try_from(NUM_L1_TRANSACTIONS).unwrap();
         node_config.l1_gas_price_scraper_config.as_mut().unwrap().number_of_blocks_for_mean =

crates/apollo_integration_tests/src/flow_test_setup.rs

@@ -35,7 +35,7 @@ use mempool_test_utils::starknet_api_test_utils::{
 use papyrus_base_layer::ethereum_base_layer_contract::EthereumBaseLayerConfig;
 use papyrus_base_layer::test_utils::anvil_mine_blocks;
 use starknet_api::block::{BlockHash, BlockNumber};
-use starknet_api::core::{ChainId, Nonce};
+use starknet_api::core::{ChainId, Nonce, StateDiffCommitment};
 use starknet_api::execution_resources::GasAmount;
 use starknet_api::rpc_transaction::RpcTransaction;
 use starknet_api::state::SierraContractClass;
@@ -62,7 +62,7 @@ use crate::node_component_configs::{
     create_hybrid_component_configs,
 };
 use crate::sequencer_simulator_utils::SequencerSimulator;
-use crate::state_reader::StorageTestHandles;
+use crate::state_reader::{proof_flow_chain_info, PresetTestContracts, StorageTestHandles};
 use crate::storage::{get_integration_test_storage, CustomPaths};
 use crate::utils::{
     create_consensus_manager_configs_from_network_configs,
@@ -735,7 +735,7 @@ impl IntegrationTestManager {
     ///
     /// The function verifies the initial state, runs the test with the given number of
     /// transactions, waits for execution to complete, and then verifies the final state.
-    async fn test_and_verify(
+    pub async fn test_and_verify(
         &mut self,
         test_scenario: impl TestScenario,
         sender_account: AccountId,
@@ -1242,7 +1242,9 @@ async fn get_sequencer_setup_configs(
     }
 
     info!("Creating node configurations.");
-    let chain_info = ChainInfo::create_for_testing();
+    let is_proof_flow = matches!(test_unique_id, TestIdentifier::ProofFlowIntegrationTest);
+    let chain_info =
+        if is_proof_flow { proof_flow_chain_info() } else { ChainInfo::create_for_testing() };
     let accounts = tx_generator.accounts();
     let component_configs_len = node_component_configs.len();
 
@@ -1319,11 +1321,15 @@ async fn get_sequencer_setup_configs(
         let validator_id = set_validator_id(&mut consensus_manager_config, node_index);
         let chain_info = chain_info.clone();
 
+        let preset_test_contracts = PresetTestContracts::new();
+        let initial_state_diff_commitment = is_proof_flow.then(StateDiffCommitment::default);
         let storage_setup = get_integration_test_storage(
             node_index,
             custom_paths.clone(),
             accounts.to_vec(),
             &chain_info,
+            preset_test_contracts,
+            initial_state_diff_commitment,
         );
 
         // Per node, create the executables constituting it.
@@ -1348,6 +1354,7 @@ async fn get_sequencer_setup_configs(
                 block_max_capacity_gas(),
                 validator_id,
                 ALLOW_BOOTSTRAP_TXS,
+                !is_proof_flow,
             );
 
             let base_app_config = DeploymentBaseAppConfig::new(config, config_pointers_map);

crates/apollo_integration_tests/src/integration_test_manager.rs

@@ -6,9 +6,12 @@ use apollo_class_manager::{ClassStorage, FsClassStorage};
 use apollo_class_manager_config::config::FsClassStorageConfig;
 use apollo_proof_manager::test_utils::FsProofStorageBuilderForTesting;
 use apollo_proof_manager_config::config::ProofManagerConfig;
+use apollo_storage::block_hash::BlockHashStorageWriter;
 use apollo_storage::body::BodyStorageWriter;
 use apollo_storage::class::ClassStorageWriter;
 use apollo_storage::compiled_class::CasmStorageWriter;
+use apollo_storage::global_root::GlobalRootStorageWriter;
+use apollo_storage::global_root_marker::GlobalRootMarkerStorageWriter;
 use apollo_storage::header::HeaderStorageWriter;
 use apollo_storage::partial_block_hash::PartialBlockHashComponentsStorageWriter;
 use apollo_storage::state::StateStorageWriter;
@@ -24,6 +27,7 @@ use mempool_test_utils::starknet_api_test_utils::{AccountTransactionGenerator, C
 use starknet_api::abi::abi_utils::get_fee_token_var_address;
 use starknet_api::block::{
     BlockBody,
+    BlockHash,
     BlockHeader,
     BlockHeaderWithoutHash,
     BlockNumber,
@@ -32,6 +36,7 @@ use starknet_api::block::{
     GasPricePerToken,
 };
 use starknet_api::block_hash::block_hash_calculator::{
+    calculate_block_hash,
     BlockHeaderCommitments,
     PartialBlockHashComponents,
 };
@@ -42,6 +47,7 @@ use starknet_api::core::{
     ClassHash,
     ContractAddress,
     EventCommitment,
+    GlobalRoot,
     Nonce,
     ReceiptCommitment,
     SequencerContractAddress,
@@ -67,6 +73,7 @@ use strum::IntoEnumIterator;
 use tempfile::TempDir;
 
 use crate::storage::StorageExecutablePaths;
+use crate::utils::seed_committer_offset;
 
 pub type TempDirHandlePair = (TempDir, TempDir);
 type ContractClassesMap = (
@@ -154,8 +161,9 @@ impl StorageTestSetup {
         test_defined_accounts: Vec<AccountTransactionGenerator>,
         chain_info: &ChainInfo,
         storage_exec_paths: Option<StorageExecutablePaths>,
+        preset_test_contracts: PresetTestContracts,
+        initial_state_diff_commitment: Option<StateDiffCommitment>,
     ) -> Self {
-        let preset_test_contracts = PresetTestContracts::new();
         // TODO(yair): Avoid cloning.
         let classes = TestClasses::new(&test_defined_accounts, preset_test_contracts.clone());
 
@@ -172,6 +180,7 @@ impl StorageTestSetup {
             &test_defined_accounts,
             preset_test_contracts.clone(),
             &classes,
+            initial_state_diff_commitment,
         );
 
         let state_sync_db_path =
@@ -190,6 +199,7 @@ impl StorageTestSetup {
             &test_defined_accounts,
             preset_test_contracts,
             &classes,
+            initial_state_diff_commitment,
         );
 
         let fs_class_storage_db_path =
@@ -232,6 +242,17 @@ impl StorageTestSetup {
             storage_exec_paths.as_ref().map(|p| p.get_committer_path_with_db_suffix());
         let (committer_db_path, committer_storage_handle) =
             create_dir_for_testing(committer_db_path);
+        if initial_state_diff_commitment.is_some() {
+            let seed_db_path = committer_db_path.clone();
+            std::thread::spawn(move || {
+                tokio::runtime::Builder::new_current_thread()
+                    .build()
+                    .unwrap()
+                    .block_on(seed_committer_offset(seed_db_path, BlockNumber(1)));
+            })
+            .join()
+            .unwrap();
+        }
         Self {
             storage_config: StorageTestConfig::new(
                 batcher_storage_config,
@@ -254,11 +275,17 @@ impl StorageTestSetup {
 }
 
 #[derive(Clone)]
-struct PresetTestContracts {
+pub struct PresetTestContracts {
     pub default_test_contracts: Vec<Contract>,
     pub erc20_contract: Contract,
 }
 
+impl Default for PresetTestContracts {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl PresetTestContracts {
     pub fn new() -> Self {
         let into_contract = |contract: FeatureContract| Contract {
@@ -336,10 +363,21 @@ fn initialize_papyrus_test_state(
     test_defined_accounts: &[AccountTransactionGenerator],
     preset_test_contracts: PresetTestContracts,
     classes: &TestClasses,
+    initial_state_diff_commitment: Option<StateDiffCommitment>,
 ) {
     let state_diff = prepare_state_diff(chain_info, test_defined_accounts, &preset_test_contracts);
 
-    write_state_to_apollo_storage(storage_writer, state_diff, classes)
+    write_state_to_apollo_storage(
+        storage_writer,
+        state_diff,
+        classes,
+        initial_state_diff_commitment,
+    );
+}
+
+/// Global root that matches the proof flow fixtures.
+fn integration_test_genesis_global_root() -> GlobalRoot {
+    GlobalRoot(felt!("0x68fed6a062d385db8d1dd9096060e822196feb311fc3bb4f0018635461ca85e"))
 }
 
 fn prepare_state_diff(
@@ -410,7 +448,12 @@ fn write_state_to_apollo_storage(
     storage_writer: &mut StorageWriter,
     state_diff: ThinStateDiff,
     classes: &TestClasses,
+    initial_state_diff_commitment: Option<StateDiffCommitment>,
 ) {
+    let is_proof_flow = initial_state_diff_commitment.is_some();
+    let state_diff_commitment =
+        initial_state_diff_commitment.unwrap_or_else(|| calculate_state_diff_hash(&state_diff));
+
     let block_number = BlockNumber(0);
     let block_header = test_block_header(block_number, state_diff.len());
     let TestClasses { cairo0_contract_classes, cairo1_contract_classes } = classes;
@@ -422,10 +465,7 @@ fn write_state_to_apollo_storage(
         l1_data_gas_price: block_header.block_header_without_hash.l1_data_gas_price,
         l2_gas_price: block_header.block_header_without_hash.l2_gas_price,
         sequencer: block_header.block_header_without_hash.sequencer,
-        header_commitments: BlockHeaderCommitments {
-            state_diff_commitment: calculate_state_diff_hash(&state_diff),
-            ..Default::default()
-        },
+        header_commitments: BlockHeaderCommitments { state_diff_commitment, ..Default::default() },
         ..Default::default()
     };
 
@@ -437,7 +477,7 @@ fn write_state_to_apollo_storage(
         sierras.push((*class_hash, sierra));
     }
 
-    write_txn
+    write_txn = write_txn
         .append_header(block_number, &block_header)
         .unwrap()
         .append_body(block_number, BlockBody::default())
@@ -447,9 +487,26 @@ fn write_state_to_apollo_storage(
         .append_classes(block_number, &sierras, &cairo0_contract_classes)
         .unwrap()
         .set_partial_block_hash_components(&block_number, &partial_block_hash)
-        .unwrap()
-        .commit()
         .unwrap();
+
+    if is_proof_flow {
+        let global_root = integration_test_genesis_global_root();
+        let genesis_block_hash =
+            calculate_block_hash(&partial_block_hash, global_root, BlockHash::GENESIS_PARENT_HASH)
+                .expect(
+                    "Integration test genesis block hash must be computable from seeded \
+                     components.",
+                );
+        write_txn = write_txn
+            .set_global_root(&block_number, global_root)
+            .unwrap()
+            .checked_increment_global_root_marker(block_number)
+            .unwrap()
+            .set_block_hash(&block_number, genesis_block_hash)
+            .unwrap();
+    }
+
+    write_txn.commit().unwrap();
 }
 
 fn test_block_header(block_number: BlockNumber, state_diff_length: usize) -> BlockHeader {
@@ -606,3 +663,10 @@ impl<'a> ThinStateDiffBuilder<'a> {
         }
     }
 }
+
+pub fn proof_flow_chain_info() -> ChainInfo {
+    let mut chain_info = ChainInfo::create_for_testing();
+    chain_info.fee_token_addresses.strk_fee_token_address =
+        contract_address!("0x4ff17bf76a1c6cebb82601a43bcab4f9650aea543c44f28e8863f8b624e4b58");
+    chain_info
+}

crates/apollo_integration_tests/src/state_reader.rs

@@ -2,9 +2,11 @@ use std::path::{Path, PathBuf};
 
 use blockifier::context::ChainInfo;
 use mempool_test_utils::starknet_api_test_utils::AccountTransactionGenerator;
+use starknet_api::core::StateDiffCommitment;
 
 use crate::executable_setup::NodeExecutableId;
 use crate::state_reader::{
+    PresetTestContracts,
     StorageTestConfig,
     StorageTestSetup,
     BATCHER_DB_PATH_SUFFIX,
@@ -117,13 +119,20 @@ pub fn get_integration_test_storage(
     custom_paths: Option<CustomPaths>,
     accounts: Vec<AccountTransactionGenerator>,
     chain_info: &ChainInfo,
+    preset_test_contracts: PresetTestContracts,
+    initial_state_diff_commitment: Option<StateDiffCommitment>,
 ) -> StorageTestSetup {
     let storage_exec_paths = custom_paths.as_ref().and_then(|paths| {
         paths.get_db_base().map(|db_base| StorageExecutablePaths::new(db_base, node_index))
     });
 
-    let StorageTestSetup { mut storage_config, storage_handles } =
-        StorageTestSetup::new(accounts, chain_info, storage_exec_paths);
+    let StorageTestSetup { mut storage_config, storage_handles } = StorageTestSetup::new(
+        accounts,
+        chain_info,
+        storage_exec_paths,
+        preset_test_contracts,
+        initial_state_diff_commitment,
+    );
 
     // Allow overriding the path with a custom prefix for Docker mode in system tests.
     if let Some(paths) = custom_paths {