Index UTXOs by address with creation height

Store UTXO creation height in LMDB so lite wallets can request delta
updates. Add  to serve those wallet queries directly and
speed up existing  (now using LMDB range queries).

Route UTXO put/delete/spend/unspend paths through helpers that keep the
primary outpoint index, address index, and STXO table in sync.

Author: akumaigorodski

app/app.rs

@@ -61,7 +61,7 @@ fn update_wallet(node: &Node, wallet: &Wallet) -> Result<(), Error> {
     let addresses = wallet.get_addresses()?;
     let unconfirmed_utxos =
         node.get_unconfirmed_utxos_by_addresses(&addresses)?;
-    let utxos = node.get_utxos_by_addresses(&addresses)?;
+    let utxos = node.get_utxos_by_addresses(&addresses, 0)?;
     let confirmed_outpoints: Vec<_> = wallet.get_utxos()?.into_keys().collect();
     let confirmed_spent = node
         .get_spent_utxos(&confirmed_outpoints)?

lib/node/mod.rs

@@ -519,7 +519,7 @@ where
                 .try_get(&rotxn, &outpoint_key)
                 .map_err(state::Error::from)?
             {
-                spent.push((*outpoint, output));
+                spent.push((*outpoint, output.output));
             }
         }
         Ok(spent)
@@ -564,9 +564,10 @@ where
     pub fn get_utxos_by_addresses(
         &self,
         addresses: &HashSet<Address>,
+        height_threshold: u32,
     ) -> Result<HashMap<OutPoint, FilledOutput>, Error> {
         let rotxn = self.env.read_txn()?;
-        let utxos = self.state.get_utxos_by_addresses(&rotxn, addresses)?;
+        let utxos = self.state.get_utxos_by_addresses(&rotxn, addresses, height_threshold)?;
         Ok(utxos)
     }
 

lib/state/block.rs

@@ -6,10 +6,9 @@ use sneed::{RoTxn, RwTxn};
 use crate::{
     state::{Error, PrevalidatedBlock, State, amm, dutch_auction, error},
     types::{
-        AmountOverflowError, Authorization, BitAssetId, BlockHash, Body,
-        FilledOutput, FilledOutputContent, GetAddress as _,
-        GetBitcoinValue as _, Hash, Header, InPoint, OutPoint, OutPointKey,
-        OutputContent, SpentOutput, TxData, Verify as _,
+        AddressOutPointKey, AmountOverflowError, Authorization, BitAssetId, BlockHash, Body,
+        FilledOutput, FilledOutputContent, GetAddress as _, GetBitcoinValue as _, Hash, Header,
+        InPoint, OutPoint, OutPointKey, OutputContent, SpentOutput, TxData, Verify as _
     },
 };
 
@@ -207,8 +206,6 @@ pub fn connect_prevalidated(
         .sum::<usize>()
         + body.coinbase.len();
 
-    // Use Vec + sort_unstable instead of individual DB operations for better performance
-    let mut utxo_deletes: Vec<OutPointKey> = Vec::with_capacity(total_inputs);
     let mut stxo_puts: Vec<(OutPointKey, SpentOutput)> =
         Vec::with_capacity(total_inputs);
     let mut utxo_puts: Vec<(OutPointKey, FilledOutput)> =
@@ -263,7 +260,6 @@ pub fn connect_prevalidated(
                     vin: vin as u32,
                 },
             };
-            utxo_deletes.push(key);
             stxo_puts.push((key, spent_output));
         }
 
@@ -358,21 +354,20 @@ pub fn connect_prevalidated(
     }
 
     // Sort all vectors in parallel for optimal cursor access
-    utxo_deletes.par_sort_unstable();
     stxo_puts.par_sort_unstable_by_key(|(key, _)| *key);
     utxo_puts.par_sort_unstable_by_key(|(key, _)| *key);
 
-    // Apply all database operations using pre-sorted keys for optimal B-tree access
-    for key in &utxo_deletes {
-        state.utxos.delete(rwtxn, key)?;
-    }
-
-    for (key, spent_output) in &stxo_puts {
-        state.stxos.put(rwtxn, key, spent_output)?;
+    for (key, output) in stxo_puts {
+        if !state.spend_utxo(rwtxn, key, output)? {
+            return Err(error::NoUtxo {
+                outpoint: key.to_outpoint(),
+            }
+            .into());
+        }
     }
 
-    for (key, filled_output) in &utxo_puts {
-        state.utxos.put(rwtxn, key, filled_output)?;
+    for (key, filled_output) in utxo_puts {
+        state.put_utxo(rwtxn, key, filled_output, prevalidated.next_height)?;
     }
 
     // Update tip and height using precomputed values
@@ -433,7 +428,7 @@ pub fn connect(
             content: filled_content,
             memo: output.memo.clone(),
         };
-        state.utxos.put(rwtxn, &outpoint_key, &filled_output)?;
+        state.put_utxo(rwtxn, outpoint_key, filled_output, height)?;
     }
     for transaction in &body.transactions {
         let filled_tx = state.fill_transaction(rwtxn, transaction)?;
@@ -451,20 +446,24 @@ pub fn connect(
                     vin: vin as u32,
                 },
             };
-            state.utxos.delete(rwtxn, &input_key)?;
-            state.stxos.put(rwtxn, &input_key, &spent_output)?;
+            if !state.spend_utxo(rwtxn, input_key, spent_output)? {
+                return Err(error::NoUtxo {
+                    outpoint: *input,
+                }
+                .into());
+            }
         }
         let Some(filled_outputs) = filled_tx.filled_outputs() else {
             let err = error::FillTxOutputContents(Box::new(filled_tx));
             return Err(err.into());
         };
-        for (vout, filled_output) in filled_outputs.iter().enumerate() {
+        for (vout, filled_output) in filled_outputs.into_iter().enumerate() {
             let outpoint = OutPoint::Regular {
                 txid,
                 vout: vout as u32,
             };
             let outpoint_key = OutPointKey::from_outpoint(&outpoint);
-            state.utxos.put(rwtxn, &outpoint_key, filled_output)?;
+            state.put_utxo(rwtxn, outpoint_key, filled_output, height)?;
         }
         match &transaction.data {
             None => (),
@@ -646,13 +645,14 @@ pub fn disconnect_tip(
         }
         // delete UTXOs, last-to-first
         tx.outputs.iter().enumerate().rev().try_for_each(
-            |(vout, _output)| {
+            |(vout, output)| {
                 let outpoint = OutPoint::Regular {
                     txid,
                     vout: vout as u32,
                 };
                 let outpoint_key = OutPointKey::from_outpoint(&outpoint);
-                if state.utxos.delete(rwtxn, &outpoint_key)? {
+                let address_key = AddressOutPointKey::new(output.address, outpoint_key);
+                if state.delete_utxo(rwtxn, &address_key)? {
                     Ok::<_, Error>(())
                 } else {
                     Err(error::NoUtxo { outpoint }.into())
@@ -662,13 +662,7 @@ pub fn disconnect_tip(
         // unspend STXOs, last-to-first
         tx.inputs.iter().rev().try_for_each(|outpoint| {
             let outpoint_key = OutPointKey::from_outpoint(outpoint);
-            if let Some(spent_output) =
-                state.stxos.try_get(rwtxn, &outpoint_key)?
-            {
-                state.stxos.delete(rwtxn, &outpoint_key)?;
-                state
-                    .utxos
-                    .put(rwtxn, &outpoint_key, &spent_output.output)?;
+            if state.unspend_utxo(rwtxn, &outpoint_key)? {
                 Ok(())
             } else {
                 Err(Error::NoStxo {
@@ -679,13 +673,14 @@ pub fn disconnect_tip(
     })?;
     // delete coinbase UTXOs, last-to-first
     body.coinbase.iter().enumerate().rev().try_for_each(
-        |(vout, _output)| {
+        |(vout, output)| {
             let outpoint = OutPoint::Coinbase {
                 merkle_root: header.merkle_root,
                 vout: vout as u32,
             };
             let outpoint_key = OutPointKey::from_outpoint(&outpoint);
-            if state.utxos.delete(rwtxn, &outpoint_key)? {
+            let address_key = AddressOutPointKey::new(output.address, outpoint_key);
+            if state.delete_utxo(rwtxn, &address_key)? {
                 Ok::<_, Error>(())
             } else {
                 Err(error::NoUtxo { outpoint }.into())

lib/state/mod.rs

@@ -13,8 +13,8 @@ use crate::{
         Address, AmountOverflowError, Authorized, AuthorizedTransaction,
         BitAssetId, BlockHash, Body, FilledOutput, FilledTransaction,
         GetAddress as _, GetBitcoinValue as _, Header, InPoint, M6id, OutPoint,
-        OutPointKey, SpentOutput, Transaction, TxData, VERSION, Verify as _,
-        Version, WithdrawalBundle, WithdrawalBundleStatus,
+        OutPointKey, AddressOutPointKey, SpentOutput, Transaction, TxData, VERSION,
+        Verify as _, Version, WithdrawalBundle, WithdrawalBundleStatus,
         proto::mainchain::TwoWayPegData,
     },
     util::Watchable,
@@ -68,6 +68,18 @@ type WithdrawalBundlesDb = DatabaseUnique<
     )>,
 >;
 
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct UtxoEntry {
+    pub created_height: u32,
+    pub output: FilledOutput,
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct SpentUtxoEntry {
+    pub created_height: u32,
+    pub output: SpentOutput,
+}
+
 #[derive(Clone)]
 pub struct State {
     /// Current tip
@@ -80,7 +92,9 @@ pub struct State {
     /// Associates Dutch auction sequence numbers with auction state
     dutch_auctions: dutch_auction::Db,
     utxos: DatabaseUnique<OutPointKey, SerdeBincode<FilledOutput>>,
-    stxos: DatabaseUnique<OutPointKey, SerdeBincode<SpentOutput>>,
+    utxos_by_address:
+        DatabaseUnique<AddressOutPointKey, SerdeBincode<UtxoEntry>>,
+    stxos: DatabaseUnique<OutPointKey, SerdeBincode<SpentUtxoEntry>>,
     /// Pending withdrawal bundle. MUST exist in withdrawal_bundles
     pending_withdrawal_bundle: DatabaseUnique<UnitKey, SerdeBincode<M6id>>,
     /// Latest failed (known) withdrawal bundle
@@ -104,7 +118,7 @@ pub struct State {
 }
 
 impl State {
-    pub const NUM_DBS: u32 = bitassets::Dbs::NUM_DBS + 12;
+    pub const NUM_DBS: u32 = bitassets::Dbs::NUM_DBS + 12 + 1;
 
     pub fn new(env: &sneed::Env) -> Result<Self, Error> {
         let mut rwtxn = env.write_txn()?;
@@ -115,6 +129,8 @@ impl State {
         let dutch_auctions =
             DatabaseUnique::create(env, &mut rwtxn, "dutch_auctions")?;
         let utxos = DatabaseUnique::create(env, &mut rwtxn, "utxos")?;
+        let utxos_by_address =
+            DatabaseUnique::create(env, &mut rwtxn, "utxos_by_address")?;
         let stxos = DatabaseUnique::create(env, &mut rwtxn, "stxos")?;
         let pending_withdrawal_bundle = DatabaseUnique::create(
             env,
@@ -147,6 +163,7 @@ impl State {
             bitassets,
             dutch_auctions,
             utxos,
+            utxos_by_address,
             stxos,
             pending_withdrawal_bundle,
             latest_failed_withdrawal_bundle,
@@ -180,10 +197,74 @@ impl State {
 
     pub fn stxos(
         &self,
-    ) -> &RoDatabaseUnique<OutPointKey, SerdeBincode<SpentOutput>> {
+    ) -> &RoDatabaseUnique<OutPointKey, SerdeBincode<SpentUtxoEntry>> {
         &self.stxos
     }
 
+    fn put_utxo(
+        &self,
+        rwtxn: &mut RwTxn,
+        outpoint_key: OutPointKey,
+        output: FilledOutput,
+        created_height: u32,
+    ) -> Result<AddressOutPointKey, sneed::db::Error> {
+        let entry = UtxoEntry { created_height, output };
+        let address_key = AddressOutPointKey::new(entry.output.address, outpoint_key);
+        self.utxos_by_address.put(rwtxn, &address_key, &entry)?;
+        self.utxos.put(rwtxn, &outpoint_key, &entry.output)?;
+        Ok(address_key)
+    }
+
+    fn delete_utxo(
+        &self,
+        rwtxn: &mut RwTxn,
+        address_key: &AddressOutPointKey
+    ) -> Result<bool, sneed::db::Error> {
+        self.utxos_by_address.delete(rwtxn, &address_key)?;
+        Ok(self.utxos.delete(rwtxn, &address_key.outpoint_key())?)
+    }
+
+    fn spend_utxo(
+        &self,
+        rwtxn: &mut RwTxn,
+        outpoint_key: OutPointKey,
+        spent_output: SpentOutput,
+    ) -> Result<bool, sneed::db::Error> {
+        let address_key =
+            AddressOutPointKey::new(spent_output.output.address, outpoint_key);
+
+        let Some(entry) = self.utxos_by_address.try_get(rwtxn, &address_key)? else {
+            return Ok(false);
+        };
+
+        let entry = SpentUtxoEntry {
+            created_height: entry.created_height,
+            output: spent_output,
+        };
+
+        self.stxos.put(rwtxn, &outpoint_key, &entry)?;
+        self.delete_utxo(rwtxn, &address_key)
+    }
+
+    fn unspend_utxo(
+        &self,
+        rwtxn: &mut RwTxn,
+        outpoint_key: &OutPointKey,
+    ) -> Result<bool, sneed::db::Error> {
+        let Some(entry) = self.stxos.try_get(rwtxn, outpoint_key)? else {
+            return Ok(false);
+        };
+
+        self.put_utxo(
+            rwtxn,
+            *outpoint_key,
+            entry.output.output,
+            entry.created_height,
+        )?;
+
+        Ok(self.stxos.delete(rwtxn, outpoint_key)?)
+    }
+
     pub fn withdrawal_bundle_event_blocks(
         &self,
     ) -> &RoDatabaseUnique<
@@ -222,13 +303,23 @@ impl State {
         &self,
         rotxn: &RoTxn,
         addresses: &HashSet<Address>,
+        height_threshold: u32,
     ) -> Result<HashMap<OutPoint, FilledOutput>, Error> {
-        let utxos: HashMap<OutPoint, FilledOutput> = self
-            .utxos
-            .iter(rotxn)?
-            .filter(|(_, output)| Ok(addresses.contains(&output.address)))
-            .map(|(key, output)| Ok((key.to_outpoint(), output)))
-            .collect()?;
+        let mut utxos = HashMap::new();
+        for address in addresses {
+            let start = AddressOutPointKey::start(*address);
+            let end = AddressOutPointKey::end(*address);
+            let mut iter = self
+                .utxos_by_address
+                .range(rotxn, &(start..=end))
+                .map_err(sneed::db::Error::from)?;
+            while let Some((key, entry)) = iter.next()? {
+                if entry.created_height >= height_threshold {
+                    let outpoint = key.outpoint_key().to_outpoint();
+                    utxos.insert(outpoint, entry.output);
+                }
+            }
+        }
         Ok(utxos)
     }
 
@@ -300,13 +391,13 @@ impl State {
                 .try_get(rotxn, &key)?
                 .ok_or(Error::NoStxo { outpoint: *input })?;
             assert_eq!(
-                stxo.inpoint,
+                stxo.output.inpoint,
                 InPoint::Regular {
                     txid,
                     vin: vin as u32
                 }
             );
-            spent_utxos.push(stxo.output);
+            spent_utxos.push(stxo.output.output);
         }
         spent_utxos.reverse();
         Ok(FilledTransaction {
@@ -656,7 +747,8 @@ impl State {
         let mut total_deposit_stxo_value = bitcoin::Amount::ZERO;
         let mut total_withdrawal_stxo_value = bitcoin::Amount::ZERO;
         self.stxos.iter(rotxn)?.map_err(Error::from).for_each(
-            |(outpoint_key, spent_output)| {
+            |(outpoint_key, spent_entry)| {
+                let spent_output = spent_entry.output;
                 let outpoint = outpoint_key.to_outpoint();
                 if let OutPoint::Deposit(_) = outpoint {
                     total_deposit_stxo_value = total_deposit_stxo_value