From cdddfaf89944da6f499ff361978e66eed8b02861 Mon Sep 17 00:00:00 2001 From: Nando Vieira Date: Tue, 16 Sep 2025 10:02:33 -0700 Subject: [PATCH 1/5] Fetch values dynamically rather than hardcoding them. --- .../src/commands/snapshot/create.rs | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 1434fe2ac..0693091a9 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -165,6 +165,47 @@ impl Cmd { print.infoln(format!("Network Passphrase: {network_passphrase}")); print.infoln(format!("Network id: {}", hex::encode(network_id))); + // Get ledger close time and base reserve from RPC + let mut ledger_close_time = 0u64; + let mut base_reserve = 1u32; // Default value + if let Ok(network) = self.network.get(&global_args.locator) { + if let Ok(client) = network.rpc_client() { + match client + .get_ledgers( + crate::rpc::LedgerStart::Ledger(ledger), + Some(1), + Some("json".to_string()), + ) + .await + { + Ok(result) if !result.ledgers.is_empty() => { + let ledger_data = &result.ledgers[0]; + + if let Ok(parsed_time) = ledger_data.ledger_close_time.parse::() { + ledger_close_time = parsed_time; + print.infoln(format!("Ledger Close Time: {ledger_close_time}")); + } else { + print.warnln(format!( + "Failed to parse ledger close time: {}", + ledger_data.ledger_close_time + )); + } + + if let Some(header_json) = &ledger_data.header_json { + base_reserve = header_json.header.base_reserve; + print.infoln(format!("Base Reserve: {base_reserve}")); + } + } + Ok(_) => print.warnln("No ledger data returned from RPC"), + Err(e) => print.warnln(format!("Failed to get ledger data from RPC: {e}")), + } + } else { + print.warnln("Failed to create RPC client for ledger data"); + } + } else { + print.warnln("Network configuration not available for RPC access"); + } + // Prepare a flat list of buckets to read. They'll be ordered by their // level so that they can iterated higher level to lower level. let buckets = history @@ -182,12 +223,11 @@ impl Cmd { // The snapshot is what will be written to file at the end. Fields will // be updated while parsing the history archive. let mut snapshot = LedgerSnapshot { - // TODO: Update more of the fields. protocol_version: 0, sequence_number: ledger, - timestamp: 0, + timestamp: ledger_close_time, network_id: network_id.into(), - base_reserve: 1, + base_reserve, min_persistent_entry_ttl: 0, min_temp_entry_ttl: 0, max_entry_ttl: 0, @@ -287,6 +327,18 @@ impl Cmd { if seen.contains(&key) { continue; } + + // Extract TTL settings from StateArchival config entries before filtering + let Some(val) = val else { continue }; + if let LedgerEntryData::ConfigSetting(ConfigSettingEntry::StateArchival( + state_archival, + )) = &val.data + { + snapshot.min_persistent_entry_ttl = state_archival.min_persistent_ttl; + snapshot.min_temp_entry_ttl = state_archival.min_temporary_ttl; + snapshot.max_entry_ttl = state_archival.max_entry_ttl; + } + let keep = match &key { LedgerKey::Account(k) => current.account_ids.contains(&k.account_id), LedgerKey::Trustline(k) => current.account_ids.contains(&k.account_id), @@ -298,7 +350,6 @@ impl Cmd { continue; } seen.insert(key.clone()); - let Some(val) = val else { continue }; match &val.data { LedgerEntryData::ContractData(e) => { // If a contract instance references contract From 929d20726f5633593261e0f7c76b5810b6c598a5 Mon Sep 17 00:00:00 2001 From: Nando Vieira Date: Tue, 23 Sep 2025 10:46:01 -0700 Subject: [PATCH 2/5] Fetch values from archive instead. --- FULL_HELP_DOCS.md | 2 +- cmd/soroban-cli/src/commands/mod.rs | 1 + .../src/commands/snapshot/create.rs | 177 ++++++++++++------ 3 files changed, 123 insertions(+), 57 deletions(-) diff --git a/FULL_HELP_DOCS.md b/FULL_HELP_DOCS.md index 06bbc2c11..c4f2589be 100644 --- a/FULL_HELP_DOCS.md +++ b/FULL_HELP_DOCS.md @@ -1587,13 +1587,13 @@ Any invalid contract id passed as `--address` will be ignored. * `--out ` — Out path that the snapshot is written to Default value: `snapshot.json` +* `--archive-url ` — Archive URL * `--global` — ⚠️ Deprecated: global config is always on * `--config-dir ` — Location of config directory. By default, it uses `$XDG_CONFIG_HOME/stellar` if set, falling back to `~/.config/stellar` otherwise. Contains configuration files, aliases, and other persistent settings * `--rpc-url ` — RPC server endpoint * `--rpc-header ` — RPC Header(s) to include in requests to the RPC provider * `--network-passphrase ` — Network passphrase to sign the transaction sent to the rpc server * `-n`, `--network ` — Name of network to use from config -* `--archive-url ` — Archive URL diff --git a/cmd/soroban-cli/src/commands/mod.rs b/cmd/soroban-cli/src/commands/mod.rs index 163ba96ad..b867b9ae0 100644 --- a/cmd/soroban-cli/src/commands/mod.rs +++ b/cmd/soroban-cli/src/commands/mod.rs @@ -26,6 +26,7 @@ pub mod version; pub mod txn_result; pub const HEADING_RPC: &str = "Options (RPC)"; +pub const HEADING_ARCHIVE: &str = "Options (Archive)"; pub const HEADING_GLOBAL: &str = "Options (Global)"; const ABOUT: &str = "Work seamlessly with Stellar accounts, contracts, and assets from the command line. diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 0693091a9..24e90890f 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -16,11 +16,11 @@ use std::{ }; use stellar_xdr::curr::{ self as xdr, AccountId, Asset, BucketEntry, ConfigSettingEntry, ConfigSettingId, - ContractExecutable, Frame, Hash, LedgerEntry, LedgerEntryData, LedgerKey, LedgerKeyAccount, - LedgerKeyClaimableBalance, LedgerKeyConfigSetting, LedgerKeyContractCode, - LedgerKeyContractData, LedgerKeyData, LedgerKeyLiquidityPool, LedgerKeyOffer, - LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, ReadXdr, ScAddress, ScContractInstance, - ScVal, + ContractExecutable, Frame, Hash, LedgerEntry, LedgerEntryData, LedgerHeaderHistoryEntry, + LedgerKey, LedgerKeyAccount, LedgerKeyClaimableBalance, LedgerKeyConfigSetting, + LedgerKeyContractCode, LedgerKeyContractData, LedgerKeyData, LedgerKeyLiquidityPool, + LedgerKeyOffer, LedgerKeyTrustLine, LedgerKeyTtl, Limited, Limits, ReadXdr, ScAddress, + ScContractInstance, ScVal, }; use tokio::fs::OpenOptions; use tokio::io::BufReader; @@ -28,7 +28,7 @@ use tokio_util::io::StreamReader; use url::Url; use crate::{ - commands::{config::data, global, HEADING_RPC}, + commands::{config::data, global, HEADING_ARCHIVE}, config::{self, locator, network::passphrase}, print, tx::builder, @@ -73,25 +73,32 @@ pub struct Cmd { /// The ledger sequence number to snapshot. Defaults to latest history archived ledger. #[arg(long)] ledger: Option, + /// Account or contract address/alias to include in the snapshot. #[arg(long = "address", help_heading = "Filter Options")] address: Vec, + /// WASM hashes to include in the snapshot. #[arg(long = "wasm-hash", help_heading = "Filter Options")] wasm_hashes: Vec, + /// Format of the out file. #[arg(long)] output: Output, + /// Out path that the snapshot is written to. #[arg(long, default_value=default_out_path().into_os_string())] out: PathBuf, + + /// Archive URL + #[arg(long, help_heading = HEADING_ARCHIVE, env = "STELLAR_ARCHIVE_URL")] + archive_url: Option, + #[command(flatten)] locator: locator::Args, + #[command(flatten)] network: config::network::Args, - /// Archive URL - #[arg(long, help_heading = HEADING_RPC, env = "STELLAR_ARCHIVE_URL")] - archive_url: Option, } #[derive(thiserror::Error, Debug)] @@ -140,6 +147,10 @@ pub enum Error { ParseAssetName(String), #[error(transparent)] Asset(#[from] builder::asset::Error), + #[error("ledger not found in archive")] + LedgerNotFound, + #[error("xdr parsing error: {0}")] + Xdr(#[from] xdr::Error), } /// Checkpoint frequency is usually 64 ledgers, but in local test nets it'll @@ -165,46 +176,20 @@ impl Cmd { print.infoln(format!("Network Passphrase: {network_passphrase}")); print.infoln(format!("Network id: {}", hex::encode(network_id))); - // Get ledger close time and base reserve from RPC - let mut ledger_close_time = 0u64; - let mut base_reserve = 1u32; // Default value - if let Ok(network) = self.network.get(&global_args.locator) { - if let Ok(client) = network.rpc_client() { - match client - .get_ledgers( - crate::rpc::LedgerStart::Ledger(ledger), - Some(1), - Some("json".to_string()), - ) - .await - { - Ok(result) if !result.ledgers.is_empty() => { - let ledger_data = &result.ledgers[0]; - - if let Ok(parsed_time) = ledger_data.ledger_close_time.parse::() { - ledger_close_time = parsed_time; - print.infoln(format!("Ledger Close Time: {ledger_close_time}")); - } else { - print.warnln(format!( - "Failed to parse ledger close time: {}", - ledger_data.ledger_close_time - )); - } - - if let Some(header_json) = &ledger_data.header_json { - base_reserve = header_json.header.base_reserve; - print.infoln(format!("Base Reserve: {base_reserve}")); - } - } - Ok(_) => print.warnln("No ledger data returned from RPC"), - Err(e) => print.warnln(format!("Failed to get ledger data from RPC: {e}")), + // Get ledger close time and base reserve from archive + let (ledger_close_time, base_reserve) = + match get_ledger_metadata_from_archive(&print, &archive_url, ledger).await { + Ok((close_time, reserve)) => { + print.infoln(format!("Ledger Close Time: {close_time}")); + print.infoln(format!("Base Reserve: {reserve}")); + (close_time, reserve) } - } else { - print.warnln("Failed to create RPC client for ledger data"); - } - } else { - print.warnln("Network configuration not available for RPC access"); - } + Err(e) => { + print.warnln(format!("Failed to get ledger metadata from archive: {e}")); + print.infoln("Using default values: close_time=0, base_reserve=1"); + (0u64, 1u32) // Default values + } + }; // Prepare a flat list of buckets to read. They'll be ordered by their // level so that they can iterated higher level to lower level. @@ -480,7 +465,6 @@ impl Cmd { // Resolve an account address to an account id. The address can be a // G-address or a key name (as in `stellar keys address NAME`). - async fn resolve_account(&self, address: &str) -> Option { let address: UnresolvedMuxedAccount = address.parse().ok()?; Some(AccountId(xdr::PublicKey::PublicKeyTypeEd25519( @@ -521,6 +505,14 @@ impl Cmd { } } +fn ledger_to_path_components(ledger: u32) -> (String, String, String, String) { + let ledger_hex = format!("{ledger:08x}"); + let ledger_hex_0 = ledger_hex[0..=1].to_string(); + let ledger_hex_1 = ledger_hex[2..=3].to_string(); + let ledger_hex_2 = ledger_hex[4..=5].to_string(); + (ledger_hex, ledger_hex_0, ledger_hex_1, ledger_hex_2) +} + async fn get_history( print: &print::Print, archive_url: &Url, @@ -529,17 +521,15 @@ async fn get_history( let archive_url = archive_url.to_string(); let archive_url = archive_url.strip_suffix('/').unwrap_or(&archive_url); let history_url = if let Some(ledger) = ledger { - let ledger_hex = format!("{ledger:08x}"); - let ledger_hex_0 = &ledger_hex[0..=1]; - let ledger_hex_1 = &ledger_hex[2..=3]; - let ledger_hex_2 = &ledger_hex[4..=5]; + let (ledger_hex, ledger_hex_0, ledger_hex_1, ledger_hex_2) = + ledger_to_path_components(ledger); format!("{archive_url}/history/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/history-{ledger_hex}.json") } else { format!("{archive_url}/.well-known/stellar-history.json") }; let history_url = Url::from_str(&history_url).unwrap(); - print.globe(format!("Downloading history {history_url}")); + print.globeln(format!("Downloading history {history_url}")); let response = http::client() .get(history_url.as_str()) @@ -553,7 +543,6 @@ async fn get_history( let ledger_offset = (ledger + 1) % CHECKPOINT_FREQUENCY; if ledger_offset != 0 { - print.println(""); print.errorln(format!( "Ledger {ledger} may not be a checkpoint ledger, try {} or {}", ledger - ledger_offset, @@ -569,12 +558,88 @@ async fn get_history( .await .map_err(Error::ReadHistoryHttpStream)?; - print.clear_line(); print.globeln(format!("Downloaded history {}", &history_url)); serde_json::from_slice::(&body).map_err(Error::JsonDecodingHistory) } +async fn get_ledger_metadata_from_archive( + print: &print::Print, + archive_url: &Url, + ledger: u32, +) -> Result<(u64, u32), Error> { + let archive_url = archive_url.to_string(); + let archive_url = archive_url.strip_suffix('/').unwrap_or(&archive_url); + + // Calculate the path to the ledger header file + let (ledger_hex, ledger_hex_0, ledger_hex_1, ledger_hex_2) = ledger_to_path_components(ledger); + let ledger_url = format!( + "{archive_url}/ledger/{ledger_hex_0}/{ledger_hex_1}/{ledger_hex_2}/ledger-{ledger_hex}.xdr.gz" + ); + + print.globeln(format!("Downloading ledger headers {ledger_url}")); + + let ledger_url = Url::from_str(&ledger_url).map_err(Error::ParsingBucketUrl)?; + let response = http::client() + .get(ledger_url.as_str()) + .send() + .await + .map_err(Error::DownloadingHistory)?; + + if !response.status().is_success() { + return Err(Error::DownloadingHistoryGotStatusCode(response.status())); + } + + // Cache the ledger file to disk like bucket files + let ledger_dir = data::bucket_dir().map_err(Error::GetBucketDir)?; + let cache_path = ledger_dir.join(format!("ledger-{ledger_hex}.xdr")); + let dl_path = cache_path.with_extension("dl"); + + let stream = response + .bytes_stream() + .map(|result| result.map_err(std::io::Error::other)); + let stream_reader = StreamReader::new(stream); + let buf_reader = BufReader::new(stream_reader); + let mut decoder = GzipDecoder::new(buf_reader); + + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&dl_path) + .await + .map_err(Error::WriteOpeningCachedBucket)?; + + tokio::io::copy(&mut decoder, &mut file) + .await + .map_err(Error::StreamingBucket)?; + + fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; + + print.globeln(format!("Downloaded ledger headers for ledger {ledger}")); + + // Now read the cached file + let file = std::fs::File::open(&cache_path).map_err(Error::ReadOpeningCachedBucket)?; + let limited = &mut Limited::new(file, Limits::none()); + + // Find the specific ledger header entry we need + let entries = Frame::::read_xdr_iter(limited); + for entry in entries { + let Frame(header_entry) = entry.map_err(Error::Xdr)?; + + if header_entry.header.ledger_seq == ledger { + let close_time = header_entry.header.scp_value.close_time.0; + let base_reserve = header_entry.header.base_reserve; + + print.infoln(format!("Found ledger header for ledger {ledger}")); + + return Ok((close_time, base_reserve)); + } + } + + Err(Error::LedgerNotFound) +} + async fn cache_bucket( print: &print::Print, archive_url: &Url, From faa544c252feddfea19206c8fb58390eddd48717 Mon Sep 17 00:00:00 2001 From: Nando Vieira Date: Tue, 23 Sep 2025 13:01:11 -0700 Subject: [PATCH 3/5] wip --- cmd/soroban-cli/src/commands/snapshot/create.rs | 12 +++++------- cmd/soroban-cli/src/print.rs | 13 +++++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 24e90890f..935d7d7f7 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -558,6 +558,7 @@ async fn get_history( .await .map_err(Error::ReadHistoryHttpStream)?; + print.clear_previous_line(); print.globeln(format!("Downloaded history {}", &history_url)); serde_json::from_slice::(&body).map_err(Error::JsonDecodingHistory) @@ -616,6 +617,7 @@ async fn get_ledger_metadata_from_archive( fs::rename(&dl_path, &cache_path).map_err(Error::RenameDownloadFile)?; + print.clear_previous_line(); print.globeln(format!("Downloaded ledger headers for ledger {ledger}")); // Now read the cached file @@ -631,8 +633,6 @@ async fn get_ledger_metadata_from_archive( let close_time = header_entry.header.scp_value.close_time.0; let base_reserve = header_entry.header.base_reserve; - print.infoln(format!("Found ledger header for ledger {ledger}")); - return Ok((close_time, base_reserve)); } } @@ -655,7 +655,7 @@ async fn cache_bucket( let bucket_url = format!("{archive_url}/bucket/{bucket_0}/{bucket_1}/{bucket_2}/bucket-{bucket}.xdr.gz"); - print.globe(format!("Downloading bucket {bucket_index} {bucket}…")); + print.globeln(format!("Downloading bucket {bucket_index} {bucket}…")); let bucket_url = Url::from_str(&bucket_url).map_err(Error::ParsingBucketUrl)?; @@ -671,15 +671,13 @@ async fn cache_bucket( } if let Some(len) = response.content_length() { - print.clear_line(); - print.globe(format!( + print.clear_previous_line(); + print.globeln(format!( "Downloaded bucket {bucket_index} {bucket} ({})", ByteSize(len) )); } - print.println(""); - let stream = response .bytes_stream() .map(|result| result.map_err(std::io::Error::other)); diff --git a/cmd/soroban-cli/src/print.rs b/cmd/soroban-cli/src/print.rs index 51e81aea3..c312f9e80 100644 --- a/cmd/soroban-cli/src/print.rs +++ b/cmd/soroban-cli/src/print.rs @@ -1,3 +1,4 @@ +use std::io::{self, Write}; use std::{env, fmt::Display}; use crate::xdr::{Error as XdrError, Transaction}; @@ -38,6 +39,18 @@ impl Print { } } + pub fn clear_previous_line(&self) { + if !self.quiet { + if cfg!(windows) { + eprint!("\x1b[1A\x1b[2K\r"); + } else { + // Move up one line, clear entire line, move to beginning + eprint!("\x1b[1A\x1b[2K\r"); + } + io::stderr().flush().unwrap(); + } + } + // Some terminals like vscode's and macOS' default terminal will not render // the subsequent space if the emoji codepoints size is 2; in this case, // we need an additional space. From 6f3e622911d0b5694757fb1d17cdb44491f6a583 Mon Sep 17 00:00:00 2001 From: Nando Vieira Date: Wed, 24 Sep 2025 10:13:36 -0700 Subject: [PATCH 4/5] Refactor metadata extraction. --- .../src/commands/snapshot/create.rs | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/cmd/soroban-cli/src/commands/snapshot/create.rs b/cmd/soroban-cli/src/commands/snapshot/create.rs index 935d7d7f7..87384d51f 100644 --- a/cmd/soroban-cli/src/commands/snapshot/create.rs +++ b/cmd/soroban-cli/src/commands/snapshot/create.rs @@ -309,40 +309,47 @@ impl Cmd { continue; } }; + if seen.contains(&key) { continue; } - // Extract TTL settings from StateArchival config entries before filtering - let Some(val) = val else { continue }; - if let LedgerEntryData::ConfigSetting(ConfigSettingEntry::StateArchival( - state_archival, - )) = &val.data - { - snapshot.min_persistent_entry_ttl = state_archival.min_persistent_ttl; - snapshot.min_temp_entry_ttl = state_archival.min_temporary_ttl; - snapshot.max_entry_ttl = state_archival.max_entry_ttl; - } - let keep = match &key { LedgerKey::Account(k) => current.account_ids.contains(&k.account_id), LedgerKey::Trustline(k) => current.account_ids.contains(&k.account_id), LedgerKey::ContractData(k) => current.contract_ids.contains(&k.contract), LedgerKey::ContractCode(e) => current.wasm_hashes.contains(&e.hash), + LedgerKey::ConfigSetting(_) => true, _ => false, }; + if !keep { continue; } + seen.insert(key.clone()); + + let Some(val) = val else { + continue; + }; + match &val.data { + LedgerEntryData::ConfigSetting(ConfigSettingEntry::StateArchival( + state_archival, + )) => { + snapshot.min_persistent_entry_ttl = state_archival.min_persistent_ttl; + snapshot.min_temp_entry_ttl = state_archival.min_temporary_ttl; + snapshot.max_entry_ttl = state_archival.max_entry_ttl; + false + } + LedgerEntryData::ContractData(e) => { // If a contract instance references contract // executable stored in another ledger entry, add // that ledger entry to the filter so that Wasm for // any filtered contract is collected too in the // second pass. - if keep && e.key == ScVal::LedgerKeyContractInstance { + if e.key == ScVal::LedgerKeyContractInstance { match &e.val { ScVal::ContractInstance(ScContractInstance { executable: ContractExecutable::Wasm(hash), From d95b85d345cbc18cce9f0d36c0ce15eb1396885b Mon Sep 17 00:00:00 2001 From: Nando Vieira Date: Wed, 24 Sep 2025 14:32:02 -0700 Subject: [PATCH 5/5] Fix output on windows (CRLF). --- cmd/soroban-cli/src/print.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-cli/src/print.rs b/cmd/soroban-cli/src/print.rs index c312f9e80..8924722fa 100644 --- a/cmd/soroban-cli/src/print.rs +++ b/cmd/soroban-cli/src/print.rs @@ -42,11 +42,11 @@ impl Print { pub fn clear_previous_line(&self) { if !self.quiet { if cfg!(windows) { - eprint!("\x1b[1A\x1b[2K\r"); + eprint!("\x1b[2A\r\x1b[2K"); } else { - // Move up one line, clear entire line, move to beginning eprint!("\x1b[1A\x1b[2K\r"); } + io::stderr().flush().unwrap(); } }