feat: introduce and configure node with tiered KVStore

Introduces TierStore, a KVStore implementation that manages data
across three storage layers:

- Primary: Main/remote data store
- Ephemeral: Secondary store for non-critical, easily-rebuildable
  data (e.g., network graph) with fast local access
- Backup: Tertiary store for disaster recovery with async/lazy
  operations to avoid blocking primary store

Adds four configuration methods to NodeBuilder:

- set_tier_store_backup: Configure backup data store
- set_tier_store_ephemeral: Configure ephemeral data store
- set_tier_store_retry_config: Configure retry parameters with
  exponential backoff
- build_with_tier_store: Build node with primary data store

These methods are exposed to the foreign interface via additions
in ffi/types.rs:

- ForeignDynStoreTrait: An FFI-safe version of DynStoreTrait
- FfiDynStore: A concrete wrapper over foreign language stores
  that implement ForeignDynStoreTrait.

Author: enigbe

Cargo.toml

@@ -63,6 +63,7 @@ bitcoin = "0.32.7"
 bip39 = { version = "2.0.0", features = ["rand"] }
 bip21 = { version = "0.5", features = ["std"], default-features = false }
 
+async-trait = {version = "0.1.89"}
 base64 = { version = "0.22.1", default-features = false, features = ["std"] }
 rand = { version = "0.9.2", default-features = false, features = ["std", "thread_rng", "os_rng"] }
 chrono = { version = "0.4", default-features = false, features = ["clock"] }

bindings/ldk_node.udl

@@ -79,6 +79,12 @@ enum WordCount {
 	"Words24",
 };
 
+dictionary RetryConfig {
+	u16 initial_retry_delay_ms;
+	u16 maximum_delay_secs;
+	f32 backoff_multiplier;
+};
+
 enum LogLevel {
 	"Gossip",
 	"Trace",
@@ -103,6 +109,53 @@ interface LogWriter {
 	void log(LogRecord record);
 };
 
+interface FfiDynStore {
+    [Name=from_store]
+    constructor(ForeignDynStoreTrait store);
+};
+
+[Trait, WithForeign]
+interface ForeignDynStoreTrait {
+    [Throws=IOError]
+    sequence<u8> read(string primary_namespace, string secondary_namespace, string key);
+    [Throws=IOError]
+    void write(string primary_namespace, string secondary_namespace, string key, sequence<u8> buf);
+    [Throws=IOError]
+    void remove(string primary_namespace, string secondary_namespace, string key, boolean lazy);
+    [Throws=IOError]
+    sequence<string> list(string primary_namespace, string secondary_namespace);
+    [Throws=IOError, Async]
+    sequence<u8> read_async(string primary_namespace, string secondary_namespace, string key);
+    [Throws=IOError, Async]
+    void write_async(string primary_namespace, string secondary_namespace, string key, sequence<u8> buf);
+    [Throws=IOError, Async]
+    void remove_async(string primary_namespace, string secondary_namespace, string key,  boolean lazy);
+    [Throws=IOError, Async]
+    sequence<string> list_async(string primary_namespace, string secondary_namespace);
+};
+
+[Error]
+enum IOError {
+  	"NotFound",
+    "PermissionDenied",
+    "ConnectionRefused",
+    "ConnectionReset",
+    "ConnectionAborted",
+    "NotConnected",
+    "AddrInUse",
+    "AddrNotAvailable",
+    "BrokenPipe",
+    "AlreadyExists",
+    "WouldBlock",
+    "InvalidInput",
+    "InvalidData",
+    "TimedOut",
+    "WriteZero",
+    "Interrupted",
+    "UnexpectedEof",
+    "Other",
+};
+
 interface Builder {
 	constructor();
 	[Name=from_config]
@@ -127,6 +180,9 @@ interface Builder {
 	void set_announcement_addresses(sequence<SocketAddress> announcement_addresses);
 	[Throws=BuildError]
 	void set_node_alias(string node_alias);
+	void set_tier_store_retry_config(RetryConfig retry_config);
+	void set_tier_store_backup(FfiDynStore backup_store);
+	void set_tier_store_ephemeral(FfiDynStore ephemeral_store);
 	[Throws=BuildError]
 	void set_async_payments_role(AsyncPaymentsRole? role);
 	[Throws=BuildError]
@@ -139,6 +195,8 @@ interface Builder {
 	Node build_with_vss_store_and_fixed_headers(NodeEntropy node_entropy, string vss_url, string store_id, record<string, string> fixed_headers);
 	[Throws=BuildError]
 	Node build_with_vss_store_and_header_provider(NodeEntropy node_entropy, string vss_url, string store_id, VssHeaderProvider header_provider);
+	[Throws=BuildError]
+	Node build_with_tier_store(NodeEntropy node_entropy, FfiDynStore primary_store);
 };
 
 interface Node {

src/builder.rs

@@ -52,8 +52,11 @@ use crate::connection::ConnectionManager;
 use crate::entropy::NodeEntropy;
 use crate::event::EventQueue;
 use crate::fee_estimator::OnchainFeeEstimator;
+#[cfg(feature = "uniffi")]
+use crate::ffi::FfiDynStore;
 use crate::gossip::GossipSource;
 use crate::io::sqlite_store::SqliteStore;
+use crate::io::tier_store::{RetryConfig, TierStore};
 use crate::io::utils::{
 	read_event_queue, read_external_pathfinding_scores_from_cache, read_network_graph,
 	read_node_metrics, read_output_sweeper, read_payments, read_peer_info, read_pending_payments,
@@ -150,6 +153,23 @@ impl std::fmt::Debug for LogWriterConfig {
 	}
 }
 
+#[derive(Default)]
+struct TierStoreConfig {
+	ephemeral: Option<Arc<DynStore>>,
+	backup: Option<Arc<DynStore>>,
+	retry: Option<RetryConfig>,
+}
+
+impl std::fmt::Debug for TierStoreConfig {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("TierStoreConfig")
+			.field("ephemeral", &self.ephemeral.as_ref().map(|_| "Arc<DynStore>"))
+			.field("backup", &self.backup.as_ref().map(|_| "Arc<DynStore>"))
+			.field("retry", &self.retry)
+			.finish()
+	}
+}
+
 /// An error encountered during building a [`Node`].
 ///
 /// [`Node`]: crate::Node
@@ -242,6 +262,7 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	log_writer_config: Option<LogWriterConfig>,
 	async_payments_role: Option<AsyncPaymentsRole>,
+	tier_store_config: Option<TierStoreConfig>,
 	runtime_handle: Option<tokio::runtime::Handle>,
 	pathfinding_scores_sync_config: Option<PathfindingScoresSyncConfig>,
 }
@@ -259,6 +280,7 @@ impl NodeBuilder {
 		let gossip_source_config = None;
 		let liquidity_source_config = None;
 		let log_writer_config = None;
+		let tier_store_config = None;
 		let runtime_handle = None;
 		let pathfinding_scores_sync_config = None;
 		Self {
@@ -267,6 +289,7 @@ impl NodeBuilder {
 			gossip_source_config,
 			liquidity_source_config,
 			log_writer_config,
+			tier_store_config,
 			runtime_handle,
 			async_payments_role: None,
 			pathfinding_scores_sync_config,
@@ -544,6 +567,51 @@ impl NodeBuilder {
 		Ok(self)
 	}
 
+	/// Configures retry behavior for transient errors when accessing the primary store.
+	///
+	/// When building with [`build_with_tier_store`], controls the exponential backoff parameters
+	/// used when retrying failed operations on the primary store due to transient errors
+	/// (network issues, timeouts, etc.).
+	///
+	/// If not set, default retry parameters are used. See [`RetryConfig`] for details.
+	///
+	/// [`build_with_tier_store`]: Self::build_with_tier_store
+	pub fn set_tier_store_retry_config(&mut self, config: RetryConfig) -> &mut Self {
+		let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+		tier_store_config.retry = Some(config);
+		self
+	}
+
+	/// Configures the backup store for local disaster recovery.
+	///
+	/// When building with [`build_with_tier_store`], this store receives asynchronous copies
+	/// of all critical data written to the primary store. If the primary store becomes
+	/// unavailable, reads will fall back to this backup store.
+	///
+	/// Backup writes are non-blocking and do not affect primary store operation performance.
+	///
+	/// [`build_with_tier_store`]: Self::build_with_tier_store
+	pub fn set_tier_store_backup(&mut self, backup_store: Arc<DynStore>) -> &mut Self {
+		let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+		tier_store_config.backup = Some(backup_store);
+		self
+	}
+
+	/// Configures the ephemeral store for non-critical, frequently-accessed data.
+	///
+	/// When building with [`build_with_tier_store`], this store is used for data like
+	/// the network graph and scorer data to reduce latency for reads. Data stored here
+	/// can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	///
+	/// [`build_with_tier_store`]: Self::build_with_tier_store
+	pub fn set_tier_store_ephemeral(&mut self, ephemeral_store: Arc<DynStore>) -> &mut Self {
+		let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default());
+		tier_store_config.ephemeral = Some(ephemeral_store);
+		self
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	pub fn build(&self, node_entropy: NodeEntropy) -> Result<Node, BuildError> {
@@ -556,6 +624,7 @@ impl NodeBuilder {
 			Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
 		)
 		.map_err(|_| BuildError::KVStoreSetupFailed)?;
+
 		self.build_with_store(node_entropy, kv_store)
 	}
 
@@ -568,6 +637,7 @@ impl NodeBuilder {
 		fs::create_dir_all(storage_dir_path.clone())
 			.map_err(|_| BuildError::StoragePathAccessFailed)?;
 		let kv_store = FilesystemStore::new(storage_dir_path);
+
 		self.build_with_store(node_entropy, kv_store)
 	}
 
@@ -654,6 +724,91 @@ impl NodeBuilder {
 		self.build_with_store(node_entropy, vss_store)
 	}
 
+	/// Builds a [`Node`] instance with tiered storage for managing data across multiple storage layers.
+	///
+	/// This build method enables a three-tier storage architecture optimized for different data types
+	/// and access patterns:
+	///
+	/// ### Storage Tiers
+	///
+	/// - **Primary Store** (required): The authoritative store for critical channel state and payment data.
+	///   Typically a remote/cloud storage service for durability and accessibility across devices.
+	///
+	/// - **Ephemeral Store** (optional): Local storage for non-critical, frequently-accessed data like
+	///   the network graph and scorer. Improves performance by reducing latency for data that can be
+	///   rebuilt if lost. Configure with [`set_tier_store_ephemeral`].
+	///
+	/// - **Backup Store** (optional): Local backup of critical data for disaster recovery scenarios.
+	///   Provides a safety net if the primary store becomes temporarily unavailable. Writes are
+	///   asynchronous to avoid blocking primary operations. Configure with [`set_tier_store_backup`].
+	///
+	/// ## Configuration
+	///
+	/// Use the setter methods to configure optional stores and retry behavior:
+	/// - [`set_tier_store_ephemeral`] - Set local store for network graph and scorer
+	/// - [`set_tier_store_backup`] - Set local backup store for disaster recovery
+	/// - [`set_tier_store_retry_config`] - Configure retry delays and backoff for transient errors
+	///
+	/// ## Example
+	///
+	/// ```ignore
+	/// # use ldk_node::{Builder, Config};
+	/// # use ldk_node::io::tier_store::RetryConfig;
+	/// # use std::sync::Arc;
+	/// let config = Config::default();
+	/// let mut builder = NodeBuilder::from_config(config);
+	///
+	/// let primary = Arc::new(VssStore::new(...));
+	/// let ephemeral = Arc::new(FilesystemStore::new(...));
+	/// let backup = Arc::new(SqliteStore::new(...));
+	///	let retry_config = RetryConfig::default();
+	///
+	/// builder
+	///     .set_tier_store_ephemeral(ephemeral)
+	///     .set_tier_store_backup(backup)
+	/// 	.set_tier_store_retry_config(retry_config);
+	///
+	/// let node = builder.build_with_tier_store(primary)?;
+	/// # Ok::<(), ldk_node::BuildError>(())
+	/// ```
+	///
+	/// [`set_tier_store_ephemeral`]: Self::set_tier_store_ephemeral
+	/// [`set_tier_store_backup`]: Self::set_tier_store_backup
+	/// [`set_tier_store_retry_config`]: Self::set_tier_store_retry_config
+	#[cfg(not(feature = "uniffi"))]
+	pub fn build_with_tier_store(
+		&self, node_entropy: NodeEntropy, primary_store: Arc<DynStore>,
+	) -> Result<Node, BuildError> {
+		self.build_with_tier_store_internal(node_entropy, primary_store)
+	}
+
+	fn build_with_tier_store_internal(
+		&self, node_entropy: NodeEntropy, primary_store: Arc<DynStore>,
+	) -> Result<Node, BuildError> {
+		let logger = setup_logger(&self.log_writer_config, &self.config)?;
+		let runtime = if let Some(handle) = self.runtime_handle.as_ref() {
+			Arc::new(Runtime::with_handle(handle.clone(), Arc::clone(&logger)))
+		} else {
+			Arc::new(Runtime::new(Arc::clone(&logger)).map_err(|e| {
+				log_error!(logger, "Failed to setup tokio runtime: {}", e);
+				BuildError::RuntimeSetupFailed
+			})?)
+		};
+
+		let ts_config = self.tier_store_config.as_ref();
+		let retry_config = ts_config.and_then(|c| c.retry).unwrap_or_default();
+
+		let mut tier_store =
+			TierStore::new(primary_store, Arc::clone(&runtime), Arc::clone(&logger), retry_config);
+
+		if let Some(config) = ts_config {
+			config.ephemeral.as_ref().map(|s| tier_store.set_ephemeral_store(Arc::clone(s)));
+			config.backup.as_ref().map(|s| tier_store.set_backup_store(Arc::clone(s)));
+		}
+
+		self.build_with_store(node_entropy, tier_store)
+	}
+
 	/// Builds a [`Node`] instance according to the options previously configured.
 	pub fn build_with_store<S: SyncAndAsyncKVStore + Send + Sync + 'static>(
 		&self, node_entropy: NodeEntropy, kv_store: S,
@@ -919,6 +1074,49 @@ impl ArcedNodeBuilder {
 		self.inner.write().unwrap().set_async_payments_role(role).map(|_| ())
 	}
 
+	/// Configures retry behavior for transient errors when accessing the primary store.
+	///
+	/// When building with [`build_with_tier_store`], controls the exponential backoff parameters
+	/// used when retrying failed operations on the primary store due to transient errors
+	/// (network issues, timeouts, etc.).
+	///
+	/// If not set, default retry parameters are used. See [`RetryConfig`] for details.
+	///
+	/// [`build_with_tier_store`]: Self::build_with_tier_store
+	pub fn set_tier_store_retry_config(&self, config: RetryConfig) {
+		self.inner.write().unwrap().set_tier_store_retry_config(config);
+	}
+
+	/// Configures the backup store for local disaster recovery.
+	///
+	/// When building with [`build_with_tier_store`], this store receives asynchronous copies
+	/// of all critical data written to the primary store. If the primary store becomes
+	/// unavailable, reads will fall back to this backup store.
+	///
+	/// Backup writes are non-blocking and do not affect primary store operation performance.
+	///
+	/// [`build_with_tier_store`]: Self::build_with_tier_store
+	pub fn set_tier_store_backup(&self, backup_store: Arc<FfiDynStore>) {
+		let wrapper = DynStoreWrapper((*backup_store).clone());
+		let store: Arc<DynStore> = Arc::new(wrapper);
+		self.inner.write().unwrap().set_tier_store_backup(store);
+	}
+
+	/// Configures the ephemeral store for non-critical, frequently-accessed data.
+	///
+	/// When building with [`build_with_tier_store`], this store is used for data like
+	/// the network graph and scorer data to reduce latency for reads. Data stored here
+	/// can be rebuilt if lost.
+	///
+	/// If not set, non-critical data will be stored in the primary store.
+	///
+	/// [`build_with_tier_store`]: Self::build_with_tier_store
+	pub fn set_tier_store_ephemeral(&self, ephemeral_store: Arc<FfiDynStore>) {
+		let wrapper = DynStoreWrapper((*ephemeral_store).clone());
+		let store: Arc<DynStore> = Arc::new(wrapper);
+		self.inner.write().unwrap().set_tier_store_ephemeral(store);
+	}
+
 	/// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options
 	/// previously configured.
 	pub fn build(&self, node_entropy: Arc<NodeEntropy>) -> Result<Arc<Node>, BuildError> {
@@ -1017,6 +1215,24 @@ impl ArcedNodeBuilder {
 			.map(Arc::new)
 	}
 
+	// pub fn build_with_tier_store(
+	// 	&self, node_entropy: Arc<NodeEntropy>, primary_store: Arc<DynStore>,
+	// ) -> Result<Arc<Node>, BuildError> {
+	// 	self.inner.read().unwrap().build_with_tier_store(*node_entropy, primary_store).map(Arc::new)
+	// }
+
+	pub fn build_with_tier_store(
+		&self, node_entropy: Arc<NodeEntropy>, primary_store: Arc<FfiDynStore>,
+	) -> Result<Arc<Node>, BuildError> {
+		let wrapper = DynStoreWrapper((*primary_store).clone());
+		let store: Arc<DynStore> = Arc::new(wrapper);
+		self.inner
+			.read()
+			.unwrap()
+			.build_with_tier_store_internal(*node_entropy, store)
+			.map(Arc::new)
+	}
+
 	/// Builds a [`Node`] instance according to the options previously configured.
 	// Note that the generics here don't actually work for Uniffi, but we don't currently expose
 	// this so its not needed.