diff --git a/crates/bitcoind_rpc/examples/filter_iter.rs b/crates/bitcoind_rpc/examples/filter_iter.rs index 21300e700..b4ca0bffd 100644 --- a/crates/bitcoind_rpc/examples/filter_iter.rs +++ b/crates/bitcoind_rpc/examples/filter_iter.rs @@ -31,6 +31,7 @@ fn main() -> anyhow::Result<()> { let (descriptor, _) = Descriptor::parse_descriptor(&secp, EXTERNAL)?; let (change_descriptor, _) = Descriptor::parse_descriptor(&secp, INTERNAL)?; let (mut chain, _) = LocalChain::from_genesis_hash(genesis_block(NETWORK).block_hash()); + let mut graph = IndexedTxGraph::>::new({ let mut index = KeychainTxOutIndex::default(); index.insert_descriptor("external", descriptor.clone())?; diff --git a/crates/chain/benches/canonicalization.rs b/crates/chain/benches/canonicalization.rs index d425ecc6c..df9c08b01 100644 --- a/crates/chain/benches/canonicalization.rs +++ b/crates/chain/benches/canonicalization.rs @@ -82,7 +82,7 @@ fn setup(f: F) -> (KeychainTxGraph, Lo let (desc, _) = >::parse_descriptor(&Secp256k1::new(), DESC).unwrap(); - let mut index = KeychainTxOutIndex::new(10); + let mut index = KeychainTxOutIndex::new(10, true); index.insert_descriptor((), desc).unwrap(); let mut tx_graph = KeychainTxGraph::new(index); diff --git a/crates/chain/src/indexed_tx_graph.rs b/crates/chain/src/indexed_tx_graph.rs index 16339624c..9ba3395eb 100644 --- a/crates/chain/src/indexed_tx_graph.rs +++ b/crates/chain/src/indexed_tx_graph.rs @@ -15,12 +15,14 @@ use crate::{ Anchor, BlockId, ChainOracle, Indexer, Merge, TxPosInBlock, }; -/// The [`IndexedTxGraph`] combines a [`TxGraph`] and an [`Indexer`] implementation. +/// A [`TxGraph`] paired with an indexer `I`, enforcing that every insertion into the graph is +/// simultaneously fed through the indexer. /// -/// It ensures that [`TxGraph`] and [`Indexer`] are updated atomically. +/// This guarantees that `tx_graph` and `index` remain in sync: any transaction or floating txout +/// you add to `tx_graph` has already been processed by `index`. #[derive(Debug, Clone)] pub struct IndexedTxGraph { - /// Transaction index. + /// The indexer used for filtering transactions and floating txouts that we are interested in. pub index: I, graph: TxGraph, } @@ -35,14 +37,6 @@ impl Default for IndexedTxGraph { } impl IndexedTxGraph { - /// Construct a new [`IndexedTxGraph`] with a given `index`. - pub fn new(index: I) -> Self { - Self { - index, - graph: TxGraph::default(), - } - } - /// Get a reference of the internal transaction graph. pub fn graph(&self) -> &TxGraph { &self.graph @@ -79,6 +73,87 @@ impl IndexedTxGraph where I::ChangeSet: Default + Merge, { + /// Create a new, empty [`IndexedTxGraph`]. + /// + /// The underlying `TxGraph` is initialized with `TxGraph::default()`, and the provided + /// `index`er is used as‐is (since there are no existing transactions to process). + pub fn new(index: I) -> Self { + Self { + index, + graph: TxGraph::default(), + } + } + + /// Reconstruct an [`IndexedTxGraph`] from persisted graph + indexer state. + /// + /// 1. Rebuilds the `TxGraph` from `changeset.tx_graph`. + /// 2. Calls your `indexer_from_changeset` closure on `changeset.indexer` to restore any state + /// your indexer needs beyond its raw changeset. + /// 3. Runs a full `.reindex()`, returning its `ChangeSet` to describe any additional updates + /// applied. + /// + /// # Errors + /// + /// Returns `Err(E)` if `indexer_from_changeset` fails. + /// + /// # Examples + /// + /// ```rust,no_run + /// use bdk_chain::IndexedTxGraph; + /// # use bdk_chain::indexed_tx_graph::ChangeSet; + /// # use bdk_chain::indexer::keychain_txout::{KeychainTxOutIndex, DEFAULT_LOOKAHEAD}; + /// # use bdk_core::BlockId; + /// # use bdk_testenv::anyhow; + /// # use miniscript::{Descriptor, DescriptorPublicKey}; + /// # use std::str::FromStr; + /// # let persisted_changeset = ChangeSet::::default(); + /// # let persisted_desc = Some(Descriptor::::from_str("")?); + /// # let persisted_change_desc = Some(Descriptor::::from_str("")?); + /// + /// let (graph, reindex_cs) = + /// IndexedTxGraph::from_changeset(persisted_changeset, move |idx_cs| -> anyhow::Result<_> { + /// // e.g. KeychainTxOutIndex needs descriptors that weren’t in its CS + /// let mut idx = KeychainTxOutIndex::from_changeset(DEFAULT_LOOKAHEAD, true, idx_cs); + /// if let Some(desc) = persisted_desc { + /// idx.insert_descriptor("external", desc)?; + /// } + /// if let Some(desc) = persisted_change_desc { + /// idx.insert_descriptor("internal", desc)?; + /// } + /// Ok(idx) + /// })?; + /// # Ok::<(), anyhow::Error>(()) + /// ``` + pub fn from_changeset( + changeset: ChangeSet, + indexer_from_changeset: F, + ) -> Result<(Self, ChangeSet), E> + where + F: FnOnce(I::ChangeSet) -> Result, + { + let graph = TxGraph::::from_changeset(changeset.tx_graph); + let index = indexer_from_changeset(changeset.indexer)?; + let mut out = Self { graph, index }; + let out_changeset = out.reindex(); + Ok((out, out_changeset)) + } + + /// Synchronizes the indexer to reflect every entry in the transaction graph. + /// + /// Iterates over **all** full transactions and floating outputs in `self.graph`, passing each + /// into `self.index`. Any indexer-side changes produced (via `index_tx` or `index_txout`) are + /// merged into a fresh `ChangeSet`, which is then returned. + pub fn reindex(&mut self) -> ChangeSet { + let mut changeset = ChangeSet::::default(); + for tx in self.graph.full_txs() { + changeset.indexer.merge(self.index.index_tx(&tx)); + } + for (op, txout) in self.graph.floating_txouts() { + changeset.indexer.merge(self.index.index_txout(op, txout)); + } + changeset + } + fn index_tx_graph_changeset( &mut self, tx_graph_changeset: &tx_graph::ChangeSet, @@ -443,6 +518,12 @@ impl From> for ChangeSet { } } +impl From<(tx_graph::ChangeSet, IA)> for ChangeSet { + fn from((tx_graph, indexer): (tx_graph::ChangeSet, IA)) -> Self { + Self { tx_graph, indexer } + } +} + #[cfg(feature = "miniscript")] impl From for ChangeSet { fn from(indexer: crate::keychain_txout::ChangeSet) -> Self { diff --git a/crates/chain/src/indexer/keychain_txout.rs b/crates/chain/src/indexer/keychain_txout.rs index 11d234e68..8cac1f214 100644 --- a/crates/chain/src/indexer/keychain_txout.rs +++ b/crates/chain/src/indexer/keychain_txout.rs @@ -2,6 +2,7 @@ //! indexes [`TxOut`]s with them. use crate::{ + alloc::boxed::Box, collections::*, miniscript::{Descriptor, DescriptorPublicKey}, spk_client::{FullScanRequestBuilder, SyncRequestBuilder}, @@ -10,7 +11,9 @@ use crate::{ DescriptorExt, DescriptorId, Indexed, Indexer, KeychainIndexed, SpkIterator, }; use alloc::{borrow::ToOwned, vec::Vec}; -use bitcoin::{Amount, OutPoint, ScriptBuf, SignedAmount, Transaction, TxOut, Txid}; +use bitcoin::{ + key::Secp256k1, Amount, OutPoint, ScriptBuf, SignedAmount, Transaction, TxOut, Txid, +}; use core::{ fmt::Debug, ops::{Bound, RangeBounds}, @@ -90,7 +93,8 @@ pub const DEFAULT_LOOKAHEAD: u32 = 25; /// } /// } /// -/// let mut txout_index = KeychainTxOutIndex::::default(); +/// // Construct index with lookahead of 21 and enable spk caching. +/// let mut txout_index = KeychainTxOutIndex::::new(21, true); /// /// # let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only(); /// # let (external_descriptor,_) = Descriptor::::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap(); @@ -128,11 +132,19 @@ pub struct KeychainTxOutIndex { descriptors: HashMap>, last_revealed: HashMap, lookahead: u32, + + /// If `true`, the script pubkeys are persisted across restarts to avoid re-derivation. + /// If `false`, `spk_cache` and `spk_cache_stage` will remain empty. + persist_spks: bool, + /// Cache of derived spks. + spk_cache: BTreeMap>, + /// Staged script pubkeys waiting to be written out in the next ChangeSet. + spk_cache_stage: BTreeMap>, } impl Default for KeychainTxOutIndex { fn default() -> Self { - Self::new(DEFAULT_LOOKAHEAD) + Self::new(DEFAULT_LOOKAHEAD, false) } } @@ -147,32 +159,34 @@ impl Indexer for KeychainTxOutIndex { fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet { let mut changeset = ChangeSet::default(); - if let Some((keychain, index)) = self.inner.scan_txout(outpoint, txout).cloned() { - let did = self - .keychain_to_descriptor_id - .get(&keychain) - .expect("invariant"); - if self.last_revealed.get(did) < Some(&index) { - self.last_revealed.insert(*did, index); - changeset.last_revealed.insert(*did, index); - self.replenish_inner_index(*did, &keychain, self.lookahead); - } - } + self._index_txout(&mut changeset, outpoint, txout); + self._empty_stage_into_changeset(&mut changeset); changeset } fn index_tx(&mut self, tx: &bitcoin::Transaction) -> Self::ChangeSet { let mut changeset = ChangeSet::default(); let txid = tx.compute_txid(); - for (op, txout) in tx.output.iter().enumerate() { - changeset.merge(self.index_txout(OutPoint::new(txid, op as u32), txout)); + for (vout, txout) in tx.output.iter().enumerate() { + self._index_txout(&mut changeset, OutPoint::new(txid, vout as u32), txout); } + self._empty_stage_into_changeset(&mut changeset); changeset } fn initial_changeset(&self) -> Self::ChangeSet { ChangeSet { last_revealed: self.last_revealed.clone().into_iter().collect(), + spk_cache: self + .spk_cache + .iter() + .map(|(desc, spks)| { + ( + *desc, + spks.iter().map(|(i, spk)| (*i, spk.clone())).collect(), + ) + }) + .collect(), } } @@ -186,17 +200,34 @@ impl Indexer for KeychainTxOutIndex { } impl KeychainTxOutIndex { - /// Construct a [`KeychainTxOutIndex`] with the given `lookahead`. + /// Construct a [`KeychainTxOutIndex`] with the given `lookahead` and `use_spk_cache` boolean. + /// + /// # Lookahead /// - /// The `lookahead` is the number of script pubkeys to derive and cache from the internal - /// descriptors over and above the last revealed script index. Without a lookahead the index - /// will miss outputs you own when processing transactions whose output script pubkeys lie - /// beyond the last revealed index. In certain situations, such as when performing an initial - /// scan of the blockchain during wallet import, it may be uncertain or unknown what the index - /// of the last revealed script pubkey actually is. + /// The `lookahead` parameter controls how many script pubkeys to derive *beyond* the highest + /// revealed index for each keychain (external/internal). Without any lookahead, the index will + /// miss outputs sent to addresses you haven’t explicitly revealed yet. A nonzero `lookahead` + /// lets you catch outputs on those “future” addresses automatically. /// /// Refer to [struct-level docs](KeychainTxOutIndex) for more about `lookahead`. - pub fn new(lookahead: u32) -> Self { + /// + /// # Script pubkey persistence + /// + /// Derived script pubkeys remain in memory. If `persist_spks` is `true`, they're saved and + /// reloaded via the `ChangeSet` on startup, avoiding re-derivation. Otherwise, they must be + /// re-derived on init, affecting startup only for very large or complex wallets. + /// + /// # Examples + /// + /// ```rust + /// # use bdk_chain::keychain_txout::KeychainTxOutIndex; + /// // Derive 20 future addresses per chain and persist + reload script pubkeys via ChangeSets: + /// let idx = KeychainTxOutIndex::<&'static str>::new(20, true); + /// + /// // Derive 10 future addresses per chain without persistence: + /// let idx = KeychainTxOutIndex::<&'static str>::new(10, false); + /// ``` + pub fn new(lookahead: u32, persist_spks: bool) -> Self { Self { inner: SpkTxOutIndex::default(), keychain_to_descriptor_id: Default::default(), @@ -204,6 +235,9 @@ impl KeychainTxOutIndex { descriptor_id_to_keychain: Default::default(), last_revealed: Default::default(), lookahead, + persist_spks, + spk_cache: Default::default(), + spk_cache_stage: Default::default(), } } @@ -215,6 +249,64 @@ impl KeychainTxOutIndex { /// Methods that are *re-exposed* from the internal [`SpkTxOutIndex`]. impl KeychainTxOutIndex { + /// Construct `KeychainTxOutIndex` from the given `changeset`. + /// + /// Shorthand for called [`new`] and then [`apply_changeset`]. + /// + /// [`new`]: Self::new + /// [`apply_changeset`]: Self::apply_changeset + pub fn from_changeset(lookahead: u32, use_spk_cache: bool, changeset: ChangeSet) -> Self { + let mut out = Self::new(lookahead, use_spk_cache); + out.apply_changeset(changeset); + out + } + + fn _index_txout(&mut self, changeset: &mut ChangeSet, outpoint: OutPoint, txout: &TxOut) { + if let Some((keychain, index)) = self.inner.scan_txout(outpoint, txout).cloned() { + let did = self + .keychain_to_descriptor_id + .get(&keychain) + .expect("invariant"); + let index_updated = match self.last_revealed.entry(*did) { + hash_map::Entry::Occupied(mut e) if e.get() < &index => { + e.insert(index); + true + } + hash_map::Entry::Vacant(e) => { + e.insert(index); + true + } + _ => false, + }; + if index_updated { + changeset.last_revealed.insert(*did, index); + self.replenish_inner_index(*did, &keychain, self.lookahead); + } + } + } + + fn _empty_stage_into_changeset(&mut self, changeset: &mut ChangeSet) { + if !self.persist_spks { + return; + } + for (did, spks) in core::mem::take(&mut self.spk_cache_stage) { + debug_assert!( + { + let desc = self.descriptors.get(&did).expect("invariant"); + spks.iter().all(|(i, spk)| { + let exp_spk = desc + .at_derivation_index(*i) + .expect("must derive") + .script_pubkey(); + &exp_spk == spk + }) + }, + "all staged spks must be correct" + ); + changeset.spk_cache.entry(did).or_default().extend(spks); + } + } + /// Get the set of indexed outpoints, corresponding to tracked keychains. pub fn outpoints(&self) -> &BTreeSet> { self.inner.outpoints() @@ -381,7 +473,7 @@ impl KeychainTxOutIndex { let descriptor = self.descriptors.get(existing_desc_id).expect("invariant"); if *existing_desc_id != did { return Err(InsertDescriptorError::KeychainAlreadyAssigned { - existing_assignment: descriptor.clone(), + existing_assignment: Box::new(descriptor.clone()), keychain, }); } @@ -393,7 +485,7 @@ impl KeychainTxOutIndex { if *existing_keychain != keychain { return Err(InsertDescriptorError::DescriptorAlreadyAssigned { existing_assignment: existing_keychain.clone(), - descriptor, + descriptor: Box::new(descriptor), }); } } @@ -420,7 +512,8 @@ impl KeychainTxOutIndex { /// Store lookahead scripts until `target_index` (inclusive). /// /// This does not change the global `lookahead` setting. - pub fn lookahead_to_target(&mut self, keychain: K, target_index: u32) { + pub fn lookahead_to_target(&mut self, keychain: K, target_index: u32) -> ChangeSet { + let mut changeset = ChangeSet::default(); if let Some((next_index, _)) = self.next_index(keychain.clone()) { let temp_lookahead = (target_index + 1) .checked_sub(next_index) @@ -430,6 +523,8 @@ impl KeychainTxOutIndex { self.replenish_inner_index_keychain(keychain, temp_lookahead); } } + self._empty_stage_into_changeset(&mut changeset); + changeset } fn replenish_inner_index_did(&mut self, did: DescriptorId, lookahead: u32) { @@ -447,20 +542,68 @@ impl KeychainTxOutIndex { /// Syncs the state of the inner spk index after changes to a keychain fn replenish_inner_index(&mut self, did: DescriptorId, keychain: &K, lookahead: u32) { let descriptor = self.descriptors.get(&did).expect("invariant"); - let next_store_index = self + + let mut next_index = self .inner .all_spks() .range(&(keychain.clone(), u32::MIN)..=&(keychain.clone(), u32::MAX)) .last() .map_or(0, |((_, index), _)| *index + 1); - let next_reveal_index = self.last_revealed.get(&did).map_or(0, |v| *v + 1); - for (new_index, new_spk) in - SpkIterator::new_with_range(descriptor, next_store_index..next_reveal_index + lookahead) - { - let _inserted = self - .inner - .insert_spk((keychain.clone(), new_index), new_spk); - debug_assert!(_inserted, "replenish lookahead: must not have existing spk: keychain={:?}, lookahead={}, next_store_index={}, next_reveal_index={}", keychain, lookahead, next_store_index, next_reveal_index); + + // Exclusive: index to stop at. + let stop_index = if descriptor.has_wildcard() { + let next_reveal_index = self.last_revealed.get(&did).map_or(0, |v| *v + 1); + (next_reveal_index + lookahead).min(BIP32_MAX_INDEX) + } else { + 1 + }; + + if self.persist_spks { + let derive_spk = { + let secp = Secp256k1::verification_only(); + let _desc = &descriptor; + move |spk_i: u32| -> ScriptBuf { + _desc + .derived_descriptor(&secp, spk_i) + .expect("The descriptor cannot have hardened derivation") + .script_pubkey() + } + }; + let cached_spk_iter = core::iter::from_fn({ + let spk_cache = self.spk_cache.entry(did).or_default(); + let spk_stage = self.spk_cache_stage.entry(did).or_default(); + let _i = &mut next_index; + move || -> Option> { + if *_i >= stop_index { + return None; + } + let spk_i = *_i; + *_i = spk_i.saturating_add(1); + + if let Some(spk) = spk_cache.get(&spk_i) { + debug_assert_eq!(spk, &derive_spk(spk_i), "cached spk must equal derived"); + return Some((spk_i, spk.clone())); + } + let spk = derive_spk(spk_i); + spk_stage.push((spk_i, spk.clone())); + spk_cache.insert(spk_i, spk.clone()); + Some((spk_i, spk)) + } + }); + for (new_index, new_spk) in cached_spk_iter { + let _inserted = self + .inner + .insert_spk((keychain.clone(), new_index), new_spk); + debug_assert!(_inserted, "replenish lookahead: must not have existing spk: keychain={:?}, lookahead={}, next_index={}", keychain, lookahead, next_index); + } + } else { + let spk_iter = SpkIterator::new_with_range(descriptor, next_index..stop_index); + for (new_index, new_spk) in spk_iter { + let _inserted = self + .inner + .insert_spk((keychain.clone(), new_index), new_spk); + debug_assert!(_inserted, "replenish lookahead: must not have existing spk: keychain={:?}, lookahead={}, next_index={}", keychain, lookahead, next_index); + } } } @@ -629,11 +772,10 @@ impl KeychainTxOutIndex { let mut changeset = ChangeSet::default(); for (keychain, &index) in keychains { - if let Some((_, new_changeset)) = self.reveal_to_target(keychain.clone(), index) { - changeset.merge(new_changeset); - } + self._reveal_to_target(&mut changeset, keychain.clone(), index); } + self._empty_stage_into_changeset(&mut changeset); changeset } @@ -656,21 +798,28 @@ impl KeychainTxOutIndex { target_index: u32, ) -> Option<(Vec>, ChangeSet)> { let mut changeset = ChangeSet::default(); + let revealed_spks = self._reveal_to_target(&mut changeset, keychain, target_index)?; + self._empty_stage_into_changeset(&mut changeset); + Some((revealed_spks, changeset)) + } + fn _reveal_to_target( + &mut self, + changeset: &mut ChangeSet, + keychain: K, + target_index: u32, + ) -> Option>> { let mut spks: Vec> = vec![]; - while let Some((i, new)) = self.next_index(keychain.clone()) { + loop { + let (i, new) = self.next_index(keychain.clone())?; if !new || i > target_index { break; } - match self.reveal_next_spk(keychain.clone()) { - Some(((i, spk), change)) => { - spks.push((i, spk)); - changeset.merge(change); - } + match self._reveal_next_spk(changeset, keychain.clone()) { + Some(indexed_spk) => spks.push(indexed_spk), None => break, } } - - Some((spks, changeset)) + Some(spks) } /// Attempts to reveal the next script pubkey for `keychain`. @@ -686,9 +835,17 @@ impl KeychainTxOutIndex { /// 2. The descriptor has already revealed scripts up to the numeric bound. /// 3. There is no descriptor associated with the given keychain. pub fn reveal_next_spk(&mut self, keychain: K) -> Option<(Indexed, ChangeSet)> { - let (next_index, new) = self.next_index(keychain.clone())?; let mut changeset = ChangeSet::default(); - + let indexed_spk = self._reveal_next_spk(&mut changeset, keychain)?; + self._empty_stage_into_changeset(&mut changeset); + Some((indexed_spk, changeset)) + } + fn _reveal_next_spk( + &mut self, + changeset: &mut ChangeSet, + keychain: K, + ) -> Option> { + let (next_index, new) = self.next_index(keychain.clone())?; if new { let did = self.keychain_to_descriptor_id.get(&keychain)?; self.last_revealed.insert(*did, next_index); @@ -699,7 +856,7 @@ impl KeychainTxOutIndex { .inner .spk_at_index(&(keychain.clone(), next_index)) .expect("we just inserted it"); - Some(((next_index, script), changeset)) + Some((next_index, script)) } /// Gets the next unused script pubkey in the keychain. I.e., the script pubkey with the lowest @@ -716,12 +873,14 @@ impl KeychainTxOutIndex { /// /// [`reveal_next_spk`]: Self::reveal_next_spk pub fn next_unused_spk(&mut self, keychain: K) -> Option<(Indexed, ChangeSet)> { + let mut changeset = ChangeSet::default(); let next_unused = self .unused_keychain_spks(keychain.clone()) .next() - .map(|(i, spk)| ((i, spk.to_owned()), ChangeSet::default())); - - next_unused.or_else(|| self.reveal_next_spk(keychain)) + .map(|(i, spk)| (i, spk.to_owned())); + let spk = next_unused.or_else(|| self._reveal_next_spk(&mut changeset, keychain))?; + self._empty_stage_into_changeset(&mut changeset); + Some((spk, changeset)) } /// Iterate over all [`OutPoint`]s that have `TxOut`s with script pubkeys derived from @@ -779,10 +938,15 @@ impl KeychainTxOutIndex { /// Applies the `ChangeSet` to the [`KeychainTxOutIndex`] pub fn apply_changeset(&mut self, changeset: ChangeSet) { - for (&desc_id, &index) in &changeset.last_revealed { - let v = self.last_revealed.entry(desc_id).or_default(); + for (did, index) in changeset.last_revealed { + let v = self.last_revealed.entry(did).or_default(); *v = index.max(*v); - self.replenish_inner_index_did(desc_id, self.lookahead); + self.replenish_inner_index_did(did, self.lookahead); + } + if self.persist_spks { + for (did, spks) in changeset.spk_cache { + self.spk_cache.entry(did).or_default().extend(spks); + } } } } @@ -793,7 +957,7 @@ pub enum InsertDescriptorError { /// The descriptor has already been assigned to a keychain so you can't assign it to another DescriptorAlreadyAssigned { /// The descriptor you have attempted to reassign - descriptor: Descriptor, + descriptor: Box>, /// The keychain that the descriptor is already assigned to existing_assignment: K, }, @@ -802,7 +966,7 @@ pub enum InsertDescriptorError { /// The keychain that you have attempted to reassign keychain: K, /// The descriptor that the keychain is already assigned to - existing_assignment: Descriptor, + existing_assignment: Box>, }, } @@ -834,14 +998,22 @@ impl core::fmt::Display for InsertDescriptorError { #[cfg(feature = "std")] impl std::error::Error for InsertDescriptorError {} -/// Represents updates to the derivation index of a [`KeychainTxOutIndex`]. -/// It maps each keychain `K` to a descriptor and its last revealed index. +/// `ChangeSet` represents persistent updates to a [`KeychainTxOutIndex`]. +/// +/// It tracks: +/// 1. `last_revealed`: the highest derivation index revealed per descriptor. +/// 2. `spks`: the cache of derived script pubkeys to persist across runs. /// -/// It can be applied to [`KeychainTxOutIndex`] with [`apply_changeset`]. +/// You can apply a `ChangeSet` to a `KeychainTxOutIndex` via +/// [`KeychainTxOutIndex::apply_changeset`], or merge two change sets with [`ChangeSet::merge`]. /// -/// The `last_revealed` field is monotone in that [`merge`] will never decrease it. -/// `keychains_added` is *not* monotone, once it is set any attempt to change it is subject to the -/// same *one-to-one* keychain <-> descriptor mapping invariant as [`KeychainTxOutIndex`] itself. +/// # Monotonicity +/// +/// - `last_revealed` is monotonic: merging retains the maximum index for each descriptor and never +/// decreases. +/// - `spks` accumulates entries: once a script pubkey is persisted, it remains available for +/// reload. If the same descriptor and index appear again with a new script pubkey, the latter +/// value overrides the former. /// /// [`KeychainTxOutIndex`]: crate::keychain_txout::KeychainTxOutIndex /// [`apply_changeset`]: crate::keychain_txout::KeychainTxOutIndex::apply_changeset @@ -850,8 +1022,13 @@ impl std::error::Error for InsertDescriptorError {} #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] #[must_use] pub struct ChangeSet { - /// Contains for each descriptor_id the last revealed index of derivation + /// Maps each `DescriptorId` to its last revealed derivation index. pub last_revealed: BTreeMap, + + /// Cache of derived script pubkeys to persist, keyed by descriptor ID and derivation index + /// (`u32`). + #[cfg_attr(feature = "serde", serde(default))] + pub spk_cache: BTreeMap>, } impl Merge for ChangeSet { @@ -872,11 +1049,22 @@ impl Merge for ChangeSet { } } } + + for (did, spks) in other.spk_cache { + let orig_spks = self.spk_cache.entry(did).or_default(); + debug_assert!( + orig_spks + .iter() + .all(|(i, orig_spk)| spks.get(i).map_or(true, |spk| spk == orig_spk)), + "spk of the same descriptor-id and derivation index must not be different" + ); + orig_spks.extend(spks); + } } /// Returns whether the changeset are empty. fn is_empty(&self) -> bool { - self.last_revealed.is_empty() + self.last_revealed.is_empty() && self.spk_cache.is_empty() } } diff --git a/crates/chain/src/rusqlite_impl.rs b/crates/chain/src/rusqlite_impl.rs index 3bc105d0b..6f3eaf71d 100644 --- a/crates/chain/src/rusqlite_impl.rs +++ b/crates/chain/src/rusqlite_impl.rs @@ -521,6 +521,8 @@ impl keychain_txout::ChangeSet { pub const SCHEMA_NAME: &'static str = "bdk_keychaintxout"; /// Name for table that stores last revealed indices per descriptor id. pub const LAST_REVEALED_TABLE_NAME: &'static str = "bdk_descriptor_last_revealed"; + /// Name for table that stores derived spks. + pub const DERIVED_SPKS_TABLE_NAME: &'static str = "bdk_descriptor_derived_spks"; /// Get v0 of sqlite [keychain_txout::ChangeSet] schema pub fn schema_v0() -> String { @@ -533,10 +535,27 @@ impl keychain_txout::ChangeSet { ) } + /// Get v1 of sqlite [keychain_txout::ChangeSet] schema + pub fn schema_v1() -> String { + format!( + "CREATE TABLE {} ( \ + descriptor_id TEXT NOT NULL, \ + spk_index INTEGER NOT NULL, \ + spk BLOB NOT NULL, \ + PRIMARY KEY (descriptor_id, spk_index) \ + ) STRICT", + Self::DERIVED_SPKS_TABLE_NAME, + ) + } + /// Initialize sqlite tables for persisting /// [`KeychainTxOutIndex`](keychain_txout::KeychainTxOutIndex). pub fn init_sqlite_tables(db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> { - migrate_schema(db_tx, Self::SCHEMA_NAME, &[&Self::schema_v0()]) + migrate_schema( + db_tx, + Self::SCHEMA_NAME, + &[&Self::schema_v0(), &Self::schema_v1()], + ) } /// Construct [`KeychainTxOutIndex`](keychain_txout::KeychainTxOutIndex) from sqlite database @@ -561,6 +580,26 @@ impl keychain_txout::ChangeSet { changeset.last_revealed.insert(descriptor_id, last_revealed); } + let mut statement = db_tx.prepare(&format!( + "SELECT descriptor_id, spk_index, spk FROM {}", + Self::DERIVED_SPKS_TABLE_NAME + ))?; + let row_iter = statement.query_map([], |row| { + Ok(( + row.get::<_, Impl>("descriptor_id")?, + row.get::<_, u32>("spk_index")?, + row.get::<_, Impl>("spk")?, + )) + })?; + for row in row_iter { + let (Impl(descriptor_id), spk_index, Impl(spk)) = row?; + changeset + .spk_cache + .entry(descriptor_id) + .or_default() + .insert(spk_index, spk); + } + Ok(changeset) } @@ -579,6 +618,20 @@ impl keychain_txout::ChangeSet { })?; } + let mut statement = db_tx.prepare_cached(&format!( + "REPLACE INTO {}(descriptor_id, spk_index, spk) VALUES(:descriptor_id, :spk_index, :spk)", + Self::DERIVED_SPKS_TABLE_NAME, + ))?; + for (&descriptor_id, spks) in &self.spk_cache { + for (&spk_index, spk) in spks { + statement.execute(named_params! { + ":descriptor_id": Impl(descriptor_id), + ":spk_index": spk_index, + ":spk": Impl(spk.clone()), + })?; + } + } + Ok(()) } } diff --git a/crates/chain/src/spk_iter.rs b/crates/chain/src/spk_iter.rs index 33d78c331..64f38c6dc 100644 --- a/crates/chain/src/spk_iter.rs +++ b/crates/chain/src/spk_iter.rs @@ -153,16 +153,16 @@ mod test { Descriptor, Descriptor, ) { - let mut txout_index = KeychainTxOutIndex::::new(0); + let mut txout_index = KeychainTxOutIndex::::new(0, true); let secp = Secp256k1::signing_only(); let (external_descriptor,_) = Descriptor::::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap(); let (internal_descriptor,_) = Descriptor::::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap(); - let _ = txout_index + txout_index .insert_descriptor(TestKeychain::External, external_descriptor.clone()) .unwrap(); - let _ = txout_index + txout_index .insert_descriptor(TestKeychain::Internal, internal_descriptor.clone()) .unwrap(); diff --git a/crates/chain/tests/test_indexed_tx_graph.rs b/crates/chain/tests/test_indexed_tx_graph.rs index 4846841c2..ee95dde79 100644 --- a/crates/chain/tests/test_indexed_tx_graph.rs +++ b/crates/chain/tests/test_indexed_tx_graph.rs @@ -10,6 +10,7 @@ use bdk_chain::{ indexer::keychain_txout::KeychainTxOutIndex, local_chain::LocalChain, tx_graph, Balance, CanonicalizationParams, ChainPosition, ConfirmationBlockTime, DescriptorExt, + SpkIterator, }; use bdk_testenv::{ block_id, hash, @@ -32,14 +33,14 @@ fn insert_relevant_txs() { .expect("must be valid"); let spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey(); let spk_1 = descriptor.at_derivation_index(9).unwrap().script_pubkey(); + let lookahead = 10; - let mut graph = IndexedTxGraph::>::new( - KeychainTxOutIndex::new(10), - ); - let _ = graph - .index - .insert_descriptor((), descriptor.clone()) - .unwrap(); + let mut graph = IndexedTxGraph::>::new({ + let mut indexer = KeychainTxOutIndex::new(lookahead, true); + let is_inserted = indexer.insert_descriptor((), descriptor.clone()).unwrap(); + assert!(is_inserted); + indexer + }); let tx_a = Transaction { output: vec![ @@ -80,6 +81,17 @@ fn insert_relevant_txs() { }, indexer: keychain_txout::ChangeSet { last_revealed: [(descriptor.descriptor_id(), 9_u32)].into(), + spk_cache: [(descriptor.descriptor_id(), { + let index_after_spk_1 = 9 /* index of spk_1 */ + 1; + SpkIterator::new_with_range( + &descriptor, + // This will also persist the staged spk cache inclusions from prev call to + // `.insert_descriptor`. + 0..index_after_spk_1 + lookahead, + ) + .collect() + })] + .into(), }, }; @@ -93,6 +105,12 @@ fn insert_relevant_txs() { tx_graph: changeset.tx_graph, indexer: keychain_txout::ChangeSet { last_revealed: changeset.indexer.last_revealed, + spk_cache: [( + descriptor.descriptor_id(), + SpkIterator::new_with_range(&descriptor, 0..=9 /* index of spk_1*/ + lookahead) + .collect(), + )] + .into(), }, }; @@ -140,18 +158,16 @@ fn test_list_owned_txouts() { let (desc_2, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), DESCRIPTORS[3]).unwrap(); - let mut graph = IndexedTxGraph::>::new( - KeychainTxOutIndex::new(10), - ); - - assert!(graph - .index - .insert_descriptor("keychain_1".into(), desc_1) - .unwrap()); - assert!(graph - .index - .insert_descriptor("keychain_2".into(), desc_2) - .unwrap()); + let mut graph = IndexedTxGraph::>::new({ + let mut indexer = KeychainTxOutIndex::new(10, true); + assert!(indexer + .insert_descriptor("keychain_1".into(), desc_1) + .unwrap()); + assert!(indexer + .insert_descriptor("keychain_2".into(), desc_2) + .unwrap()); + indexer + }); // Get trusted and untrusted addresses diff --git a/crates/chain/tests/test_keychain_txout_index.rs b/crates/chain/tests/test_keychain_txout_index.rs index 8b299b896..0cce8476d 100644 --- a/crates/chain/tests/test_keychain_txout_index.rs +++ b/crates/chain/tests/test_keychain_txout_index.rs @@ -3,7 +3,7 @@ use bdk_chain::{ collections::BTreeMap, indexer::keychain_txout::{ChangeSet, KeychainTxOutIndex}, - DescriptorExt, DescriptorId, Indexer, Merge, + DescriptorExt, DescriptorId, Indexer, Merge, SpkIterator, }; use bdk_testenv::{ hash, @@ -29,8 +29,9 @@ fn init_txout_index( external_descriptor: Descriptor, internal_descriptor: Descriptor, lookahead: u32, + use_spk_cache: bool, ) -> KeychainTxOutIndex { - let mut txout_index = KeychainTxOutIndex::::new(lookahead); + let mut txout_index = KeychainTxOutIndex::::new(lookahead, use_spk_cache); let _ = txout_index .insert_descriptor(TestKeychain::External, external_descriptor) @@ -81,9 +82,11 @@ fn merge_changesets_check_last_revealed() { let mut lhs = ChangeSet { last_revealed: lhs_di, + ..Default::default() }; let rhs = ChangeSet { last_revealed: rhs_di, + ..Default::default() }; lhs.merge(rhs); @@ -99,10 +102,15 @@ fn merge_changesets_check_last_revealed() { #[test] fn test_set_all_derivation_indices() { + let lookahead = 0; let external_descriptor = parse_descriptor(DESCRIPTORS[0]); let internal_descriptor = parse_descriptor(DESCRIPTORS[1]); - let mut txout_index = - init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0); + let mut txout_index = init_txout_index( + external_descriptor.clone(), + internal_descriptor.clone(), + lookahead, + true, + ); let derive_to: BTreeMap<_, _> = [(TestKeychain::External, 12), (TestKeychain::Internal, 24)].into(); let last_revealed: BTreeMap<_, _> = [ @@ -110,10 +118,22 @@ fn test_set_all_derivation_indices() { (internal_descriptor.descriptor_id(), 24), ] .into(); + let spk_cache: BTreeMap> = [ + ( + external_descriptor.descriptor_id(), + SpkIterator::new_with_range(&external_descriptor, 0..=12).collect(), + ), + ( + internal_descriptor.descriptor_id(), + SpkIterator::new_with_range(&internal_descriptor, 0..=24).collect(), + ), + ] + .into(); assert_eq!( txout_index.reveal_to_target_multi(&derive_to), ChangeSet { - last_revealed: last_revealed.clone() + last_revealed: last_revealed.clone(), + spk_cache: spk_cache.clone(), } ); assert_eq!(txout_index.last_revealed_indices(), derive_to); @@ -134,6 +154,7 @@ fn test_lookahead() { external_descriptor.clone(), internal_descriptor.clone(), lookahead, + true, ); // given: @@ -315,8 +336,12 @@ fn test_lookahead() { fn test_scan_with_lookahead() { let external_descriptor = parse_descriptor(DESCRIPTORS[0]); let internal_descriptor = parse_descriptor(DESCRIPTORS[1]); - let mut txout_index = - init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 10); + let mut txout_index = init_txout_index( + external_descriptor.clone(), + internal_descriptor.clone(), + 10, + true, + ); let spks: BTreeMap = [0, 10, 20, 30] .into_iter() @@ -372,7 +397,7 @@ fn test_scan_with_lookahead() { fn test_wildcard_derivations() { let external_descriptor = parse_descriptor(DESCRIPTORS[0]); let internal_descriptor = parse_descriptor(DESCRIPTORS[1]); - let mut txout_index = init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0); + let mut txout_index = init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0, true); let external_spk_0 = external_descriptor.at_derivation_index(0).unwrap().script_pubkey(); let external_spk_16 = external_descriptor.at_derivation_index(16).unwrap().script_pubkey(); let external_spk_26 = external_descriptor.at_derivation_index(26).unwrap().script_pubkey(); @@ -430,7 +455,7 @@ fn test_wildcard_derivations() { #[test] fn test_non_wildcard_derivations() { - let mut txout_index = KeychainTxOutIndex::::new(0); + let mut txout_index = KeychainTxOutIndex::::new(0, true); let secp = bitcoin::secp256k1::Secp256k1::signing_only(); let (no_wildcard_descriptor, _) = @@ -556,6 +581,7 @@ fn lookahead_to_target() { external_descriptor.clone(), internal_descriptor.clone(), t.lookahead, + true, ); if let Some(last_revealed) = t.external_last_revealed { @@ -589,7 +615,7 @@ fn lookahead_to_target() { } None => target, }; - index.lookahead_to_target(keychain.clone(), target); + let _ = index.lookahead_to_target(keychain.clone(), target); let keys: Vec<_> = (0..) .take_while(|&i| index.spk_at_index(keychain.clone(), i).is_some()) .collect(); @@ -606,22 +632,24 @@ fn applying_changesets_one_by_one_vs_aggregate_must_have_same_result() { let changesets: &[ChangeSet] = &[ ChangeSet { last_revealed: [(desc.descriptor_id(), 10)].into(), + ..Default::default() }, ChangeSet { last_revealed: [(desc.descriptor_id(), 12)].into(), + ..Default::default() }, ]; - let mut indexer_a = KeychainTxOutIndex::::new(0); - indexer_a + let mut indexer_a = KeychainTxOutIndex::::new(0, true); + let _ = indexer_a .insert_descriptor(TestKeychain::External, desc.clone()) .expect("must insert keychain"); for changeset in changesets { indexer_a.apply_changeset(changeset.clone()); } - let mut indexer_b = KeychainTxOutIndex::::new(0); - indexer_b + let mut indexer_b = KeychainTxOutIndex::::new(0, true); + let _ = indexer_b .insert_descriptor(TestKeychain::External, desc.clone()) .expect("must insert keychain"); let aggregate_changesets = changesets @@ -655,7 +683,7 @@ fn applying_changesets_one_by_one_vs_aggregate_must_have_same_result() { #[test] fn assigning_same_descriptor_to_multiple_keychains_should_error() { let desc = parse_descriptor(DESCRIPTORS[0]); - let mut indexer = KeychainTxOutIndex::::new(0); + let mut indexer = KeychainTxOutIndex::::new(0, true); let _ = indexer .insert_descriptor(TestKeychain::Internal, desc.clone()) .unwrap(); @@ -668,7 +696,7 @@ fn assigning_same_descriptor_to_multiple_keychains_should_error() { fn reassigning_keychain_to_a_new_descriptor_should_error() { let desc1 = parse_descriptor(DESCRIPTORS[0]); let desc2 = parse_descriptor(DESCRIPTORS[1]); - let mut indexer = KeychainTxOutIndex::::new(0); + let mut indexer = KeychainTxOutIndex::::new(0, true); let _ = indexer.insert_descriptor(TestKeychain::Internal, desc1); assert!(indexer .insert_descriptor(TestKeychain::Internal, desc2) @@ -677,7 +705,7 @@ fn reassigning_keychain_to_a_new_descriptor_should_error() { #[test] fn when_querying_over_a_range_of_keychains_the_utxos_should_show_up() { - let mut indexer = KeychainTxOutIndex::::new(0); + let mut indexer = KeychainTxOutIndex::::new(0, true); let mut tx = new_tx(0); for (i, descriptor) in DESCRIPTORS.iter().enumerate() { diff --git a/examples/example_cli/src/lib.rs b/examples/example_cli/src/lib.rs index d432d12b6..d70ee1250 100644 --- a/examples/example_cli/src/lib.rs +++ b/examples/example_cli/src/lib.rs @@ -1,3 +1,4 @@ +use bdk_chain::keychain_txout::DEFAULT_LOOKAHEAD; use serde_json::json; use std::cmp; use std::collections::HashMap; @@ -22,7 +23,6 @@ use bdk_chain::miniscript::{ use bdk_chain::CanonicalizationParams; use bdk_chain::ConfirmationBlockTime; use bdk_chain::{ - indexed_tx_graph, indexer::keychain_txout::{self, KeychainTxOutIndex}, local_chain::{self, LocalChain}, tx_graph, ChainOracle, DescriptorExt, FullTxOut, IndexedTxGraph, Merge, @@ -818,11 +818,10 @@ pub fn init_or_load( Commands::Generate { network } => generate_bip86_helper(network).map(|_| None), // try load _ => { - let (db, changeset) = + let (mut db, changeset) = Store::::load(db_magic, db_path).context("could not open file store")?; let changeset = changeset.expect("should not be empty"); - let network = changeset.network.expect("changeset network"); let chain = Mutex::new({ @@ -832,23 +831,27 @@ pub fn init_or_load( chain }); - let graph = Mutex::new({ - // insert descriptors and apply loaded changeset - let mut index = KeychainTxOutIndex::default(); - if let Some(desc) = changeset.descriptor { - index.insert_descriptor(Keychain::External, desc)?; - } - if let Some(change_desc) = changeset.change_descriptor { - index.insert_descriptor(Keychain::Internal, change_desc)?; - } - let mut graph = KeychainTxGraph::new(index); - graph.apply_changeset(indexed_tx_graph::ChangeSet { - tx_graph: changeset.tx_graph, - indexer: changeset.indexer, - }); - graph - }); + let (graph, changeset) = IndexedTxGraph::from_changeset( + (changeset.tx_graph, changeset.indexer).into(), + |c| -> anyhow::Result<_> { + let mut indexer = + KeychainTxOutIndex::from_changeset(DEFAULT_LOOKAHEAD, true, c); + if let Some(desc) = changeset.descriptor { + indexer.insert_descriptor(Keychain::External, desc)?; + } + if let Some(change_desc) = changeset.change_descriptor { + indexer.insert_descriptor(Keychain::Internal, change_desc)?; + } + Ok(indexer) + }, + )?; + db.append(&ChangeSet { + indexer: changeset.indexer, + tx_graph: changeset.tx_graph, + ..Default::default() + })?; + let graph = Mutex::new(graph); let db = Mutex::new(db); Ok(Some(Init {