Use bitcoin core's txospenderindex to find spending transactions

docs/release-notes/eclair-vnext.md

@@ -253,6 +253,11 @@ eclair.relay.peer-reputation.enabled = false
 eclair.relay.reserved-for-accountable = 0.0
 ```
 
+### Faster scanning for spending transactions with Bitcoin Core's txospenderindex
+
+If Bitcoin Core's `txospenderindex` (available in Bitcoin Core 31.0 and newer) is available and synced, Eclair will use it to find channel spending transactions, which is much faster and less expensive than scanning blocks.
+To enable this index, start Bitcoin Core with `-txospenderindex` or add `txospenderindex=1` to your `bitcoin.conf`.
+
 ### Configuration changes
 
 <insert changes>

eclair-core/pom.xml

@@ -87,8 +87,8 @@
                 <activeByDefault>true</activeByDefault>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-x86_64-linux-gnu.tar.gz</bitcoind.url>
-                <bitcoind.sha256>6aa7bb4feb699c4c6262dd23e4004191f6df7f373b5d5978b5bcdd4bb72f75d8</bitcoind.sha256>
+                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-x86_64-linux-gnu.tar.gz</bitcoind.url>
+                <bitcoind.sha256>d3e4c58a35b1d0a97a457462c94f55501ad167c660c245cb1ffa565641c65074</bitcoind.sha256>
             </properties>
         </profile>
         <profile>
@@ -99,8 +99,8 @@
                 </os>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-x86_64-apple-darwin.tar.gz</bitcoind.url>
-                <bitcoind.sha256>99d5cee9b9c37be506396c30837a4b98e320bfea71c474d6120a7e8eb6075c7b</bitcoind.sha256>
+                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-x86_64-apple-darwin.tar.gz</bitcoind.url>
+                <bitcoind.sha256>56824dd705bc2a3b22d42e8aa02ed53498d491ff7c2c8aa96831333871887ead</bitcoind.sha256>
             </properties>
         </profile>
         <profile>
@@ -111,8 +111,8 @@
                 </os>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-win64.zip</bitcoind.url>
-                <bitcoind.sha256>0d7e1f16f8823aa26d29b44855ff6dbac11c03d75631a6c1d2ea5fab3a84fdf8</bitcoind.sha256>
+                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-win64.zip</bitcoind.url>
+                <bitcoind.sha256>82fd2c504a0f20a31d4d13bd407783d6fc7bf17622d0ce85228a9b92694e03f0</bitcoind.sha256>
             </properties>
         </profile>
     </profiles>

eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala

@@ -292,6 +292,9 @@ class Setup(val datadir: File,
         }
       }
       _ = if (bitcoinClient.useEclairSigner) logger.info("using eclair to sign bitcoin core transactions")
+      indexInfos <- bitcoinClient.getIndexInfo()
+      _ = if (!indexInfos.get("txindex").exists(_.synced)) throw new RuntimeException("txindex is disabled or not synchronized")
+      _ = if (!indexInfos.get("txospenderindex").exists(_.synced)) throw new RuntimeException("txospenderindex is disabled or not synchronized")
       // We use the default address type configured on the Bitcoin Core node.
       initialPubkeyScript <- bitcoinClient.getReceivePublicKeyScript(addressType_opt = None)
       _ = finalPubkeyScript.set(initialPubkeyScript)

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/OnChainWallet.scala

@@ -81,7 +81,7 @@ trait OnChainChannelFunder {
    * Note that if this function returns false, that doesn't mean the output cannot be spent. The output could be unknown
    * (not in the blockchain nor in the mempool) but could reappear later and be spendable at that point.
    */
-  def isTransactionOutputSpendable(txid: TxId, outputIndex: Int, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean]
+  def isTransactionOutputSpendable(outPoint: OutPoint, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean]
 
   /** Rollback a transaction that we failed to commit: this probably translates to "release locks on utxos". */
   def rollback(tx: Transaction)(implicit ec: ExecutionContext): Future[Boolean]

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala

@@ -103,6 +103,8 @@ object ZmqWatcher {
     def txId: TxId
     /** Index of the outpoint to watch. */
     def outputIndex: Int
+    /** outpoint to watch */
+    def outPoint: OutPoint = OutPoint(txId, outputIndex)
     /**
      * TxIds of potential spending transactions; most of the time we know the txs, and it allows for optimizations.
      * This argument can safely be ignored by watcher implementations.
@@ -427,6 +429,7 @@ private class ZmqWatcher(nodeParams: NodeParams, blockHeight: AtomicLong, client
     }
   }
 
+
   private def checkSpent(w: WatchSpent[_ <: WatchSpentTriggered]): Future[Unit] = {
     // First let's see if the parent tx was published or not before checking whether it has been spent.
     client.getTxConfirmations(w.txId).collect {
@@ -435,49 +438,31 @@ private class ZmqWatcher(nodeParams: NodeParams, blockHeight: AtomicLong, client
           // This is an external channels: funds are not at risk, so we don't need to scan the blockchain to find the
           // spending transaction, it is costly and unnecessary. We simply check whether the output has already been
           // spent by a confirmed transaction.
-          client.isTransactionOutputSpent(w.txId, w.outputIndex).collect {
+          client.isTransactionOutputSpent(w.outPoint).collect {
             case true =>
               // The output has been spent, so we trigger the watch without including the spending transaction.
               context.self ! TriggerEvent(w.replyTo, w, WatchExternalChannelSpentTriggered(w.shortChannelId, None))
           }
         case _ =>
           // The parent tx was published, we need to make sure this particular output has not been spent.
-          client.isTransactionOutputSpendable(w.txId, w.outputIndex, includeMempool = true).collect {
+          client.isTransactionOutputSpendable(w.outPoint, includeMempool = true).collect {
             case false =>
-              // The output has been spent, let's find the spending tx.
-              // If we know some potential spending txs, we try to fetch them directly.
-              Future.sequence(w.hints.map(txid => client.getTransaction(txid).map(Some(_)).recover { case _ => None }))
-                .map(_.flatten) // filter out errors and hint transactions that can't be found
-                .map(hintTxs => {
-                  hintTxs.find(tx => tx.txIn.exists(i => i.outPoint.txid == w.txId && i.outPoint.index == w.outputIndex)) match {
-                    case Some(spendingTx) =>
-                      log.info("{}:{} has already been spent by a tx provided in hints: txid={}", w.txId, w.outputIndex, spendingTx.txid)
-                      context.self ! ProcessNewTransaction(spendingTx)
-                    case None =>
-                      // The hints didn't help us, let's search for the spending transaction in the mempool.
-                      log.info("{}:{} has already been spent, looking for the spending tx in the mempool", w.txId, w.outputIndex)
-                      client.lookForMempoolSpendingTx(w.txId, w.outputIndex).map(Some(_)).recover { case _ => None }.map {
-                        case Some(spendingTx) =>
-                          log.info("found tx spending {}:{} in the mempool: txid={}", w.txId, w.outputIndex, spendingTx.txid)
-                          context.self ! ProcessNewTransaction(spendingTx)
-                        case None =>
-                          // The spending transaction isn't in the mempool, so it must be a transaction that confirmed
-                          // before we set the watch. We have to scan the blockchain to find it, which is expensive
-                          // since bitcoind doesn't provide indexes for this scenario.
-                          log.warn("{}:{} has already been spent, spending tx not in the mempool, looking in the blockchain...", w.txId, w.outputIndex)
-                          client.lookForSpendingTx(None, w.txId, w.outputIndex, nodeParams.channelConf.maxChannelSpentRescanBlocks).map { spendingTx =>
-                            log.warn("found the spending tx of {}:{} in the blockchain: txid={}", w.txId, w.outputIndex, spendingTx.txid)
-                            context.self ! ProcessNewTransaction(spendingTx)
-                          }.recover {
-                            case _ => log.warn("could not find the spending tx of {}:{} in the blockchain, funds are at risk", w.txId, w.outputIndex)
-                          }
-                      }
-                  }
-                })
+                // The output has been spent, let's find the spending tx in the txospenderindex.
+                log.info("{} has already been spent, looking for the spending tx", w.outPoint)
+                client.findSpendingTx(w.outPoint) map {
+                  case Some((spendingTx, _)) =>
+                    log.info("found tx spending {} txid={}", w.outPoint, spendingTx.txid)
+                    context.self ! ProcessNewTransaction(spendingTx)
+                  case None =>
+                    log.warn("could not find the spending tx of {}, funds are at risk", w.outPoint)
+                } recover {
+                  case error =>
+                    log.warn("error finding the spending tx of {}, funds are at risk", w.outPoint, error)
+                }
+            }
           }
       }
     }
-  }
 
   private def checkPublished(w: WatchPublished): Future[Unit] = {
     log.debug("checking publication of txid={}", w.txId)

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala

@@ -116,17 +116,17 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool
    * (not in the blockchain nor in the mempool) but could reappear later and be spendable at that point. If you want to
    * ensure that an output is not spendable anymore, you should use [[isTransactionOutputSpent]].
    */
-  def isTransactionOutputSpendable(txid: TxId, outputIndex: Int, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] =
+  def isTransactionOutputSpendable(outPoint: OutPoint, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] =
     for {
-      json <- rpcClient.invoke("gettxout", txid, outputIndex, includeMempool)
+      json <- rpcClient.invoke("gettxout", outPoint.txid, outPoint.index, includeMempool)
     } yield json != JNull
 
   /**
    * Return true if this output has already been spent by a confirmed transaction.
    * Note that a reorg may invalidate the result of this function and make a spent output spendable again.
    */
-  def isTransactionOutputSpent(txid: TxId, outputIndex: Int)(implicit ec: ExecutionContext): Future[Boolean] = {
-    getTxConfirmations(txid).flatMap {
+  def isTransactionOutputSpent(outPoint: OutPoint)(implicit ec: ExecutionContext): Future[Boolean] = {
+    getTxConfirmations(outPoint.txid).flatMap {
       case Some(confirmations) if confirmations > 0 =>
         // There is an important limitation when using isTransactionOutputSpendable: if it returns false, it can mean a
         // few different things:
@@ -137,7 +137,7 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool
         // The only way to make sure that our output has been spent is to verify that it is coming from a confirmed
         // transaction and that it has been spent by another confirmed transaction. We want to ignore the mempool to
         // only consider spending transactions that have been confirmed.
-        isTransactionOutputSpendable(txid, outputIndex, includeMempool = false).map(r => !r)
+        isTransactionOutputSpendable(outPoint, includeMempool = false).map(r => !r)
       case _ =>
         // If the output itself isn't in the blockchain, it cannot be spent by a confirmed transaction.
         Future.successful(false)
@@ -166,47 +166,37 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool
         // themselves been double-spent, we will never be able to consider our transaction double-spent. With the
         // information we have, these unknown inputs could eventually reappear and the transaction could be broadcast
         // again.
-        Future.sequence(tx.txIn.map(txIn => isTransactionOutputSpent(txIn.outPoint.txid, txIn.outPoint.index.toInt))).map(_.exists(_ == true))
+        Future.sequence(tx.txIn.map(txIn => isTransactionOutputSpent(txIn.outPoint))).map(_.exists(_ == true))
       }
     } yield doubleSpent
 
   /** Search for mempool transaction spending a given output. */
-  def lookForMempoolSpendingTx(txid: TxId, outputIndex: Int)(implicit ec: ExecutionContext): Future[Transaction] = {
-    rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(txid, outputIndex))).collect {
+  def lookForMempoolSpendingTx(outPoint: OutPoint)(implicit ec: ExecutionContext): Future[Transaction] = {
+    val options = JObject(List("return_spending_tx" -> JBool(true), "mempool_only" -> JBool(true)))
+    rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(outPoint.txid, outPoint.index)), options).collect {
       case JArray(results) => results.flatMap(result => (result \ "spendingtxid").extractOpt[String].map(TxId.fromValidHex))
     }.flatMap { spendingTxIds =>
       spendingTxIds.headOption match {
         case Some(spendingTxId) => getTransaction(spendingTxId)
-        case None => Future.failed(new RuntimeException(s"mempool doesn't contain any transaction spending $txid:$outputIndex"))
+        case None => Future.failed(new RuntimeException(s"mempool doesn't contain any transaction spending $outPoint"))
       }
     }
   }
 
   /**
-   * Iterate over blocks to find the transaction that has spent a given output.
-   * It isn't useful to look at the whole blockchain history: if the transaction was confirmed long ago, an attacker
-   * will have already claimed all possible outputs and there's nothing we can do about it.
+   * Find the transaction spending a given output. Requires `txospenderindex` on the bitcoin code node we're connecting to.
    *
-   * @param blockHash_opt hash of a block *after* the output has been spent. If not provided, we will use the blockchain tip.
-   * @param txid          id of the transaction output that has been spent.
-   * @param outputIndex   index of the transaction output that has been spent.
-   * @param limit         maximum number of previous blocks to scan.
-   * @return the transaction spending the given output.
+   * @param outPoint transaction output
+   * @return the transaction that spent this output along with the id of the block it was published in if any, or None if no spending transaction was found
    */
-  def lookForSpendingTx(blockHash_opt: Option[BlockHash], txid: TxId, outputIndex: Int, limit: Int)(implicit ec: ExecutionContext): Future[Transaction] = {
-    for {
-      blockId <- blockHash_opt match {
-        case Some(blockHash) => Future.successful(BlockId(blockHash))
-        // NB: bitcoind confusingly returns the blockId instead of the blockHash.
-        case None => rpcClient.invoke("getbestblockhash").collect { case JString(blockId) => BlockId(ByteVector32.fromValidHex(blockId)) }
-      }
-      block <- getBlock(blockId)
-      res <- block.tx.asScala.find(tx => tx.txIn.asScala.exists(i => i.outPoint.txid == KotlinUtils.scala2kmp(txid) && i.outPoint.index == outputIndex)) match {
-        case Some(tx) => Future.successful(KotlinUtils.kmp2scala(tx))
-        case None if limit > 0 => lookForSpendingTx(Some(KotlinUtils.kmp2scala(block.header.hashPreviousBlock)), txid, outputIndex, limit - 1)
-        case None => Future.failed(new RuntimeException(s"couldn't find tx spending $txid:$outputIndex in the blockchain"))
-      }
-    } yield res
+  def findSpendingTx(outPoint: OutPoint)(implicit ec: ExecutionContext): Future[Option[(Transaction, Option[BlockId])]] = {
+    val options = JObject(List("return_spending_tx" -> JBool(true)))
+    rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(outPoint.txid, outPoint.index)), options).collect {
+      case JArray(results) => results.flatMap(result => {
+        val tx_opt = (result \ "spendingtx").extractOpt[String].map(Transaction.read)
+        tx_opt.map(tx => tx -> (result \ "blockhash").extractOpt[String].map(s => BlockId(ByteVector32.fromValidHex(s))))
+      }).headOption
+    }
   }
 
   def listTransactions(count: Int, skip: Int)(implicit ec: ExecutionContext): Future[List[WalletTx]] = rpcClient.invoke("listtransactions", "*", count, skip).map {
@@ -755,12 +745,13 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool
         TxId.fromValidHex(txs(txIndex).extract[String])
       }.getOrElse(TxId(ByteVector32.Zeroes)))
       tx <- getRawTransaction(txid)
-      unspent <- isTransactionOutputSpendable(txid, outputIndex, includeMempool = true)
+      outPoint = OutPoint(txid, outputIndex)
+      unspent <- isTransactionOutputSpendable(outPoint, includeMempool = true)
       fundingTxStatus <- if (unspent) {
         Future.successful(UtxoStatus.Unspent)
       } else {
         // if this returns true, it means that the spending tx is *not* in the blockchain
-        isTransactionOutputSpendable(txid, outputIndex, includeMempool = false).map(res => UtxoStatus.Spent(spendingTxConfirmed = !res))
+        isTransactionOutputSpendable(outPoint, includeMempool = false).map(res => UtxoStatus.Spent(spendingTxConfirmed = !res))
       }
     } yield ValidateResult(c, Right((Transaction.read(tx), fundingTxStatus)))
   } recover {
@@ -794,6 +785,15 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool
     })
   }
 
+  //------------------------- MISC  -------------------------//
+
+  /**
+   *
+   * @return information about enabled bitcoin core indexes, in a map where the key is the index name
+   */
+  def getIndexInfo()(implicit ec: ExecutionContext): Future[Map[String, IndexInfo]] = rpcClient.invoke("getindexinfo").collect {
+    case JObject(results) => results.map { case (name, o) => name -> BitcoinCoreClient.IndexInfo((o \ "synced").extract[Boolean], (o \ "best_block_height").extract[Int]) }.toMap
+  }
 }
 
 object BitcoinCoreClient {
@@ -871,4 +871,11 @@ object BitcoinCoreClient {
     // @formatter:on
   }
 
+  /**
+   * Information about a bitcoin core inedx
+   *
+   * @param synced          true if the index is synced
+   * @param bestBlockHeight height of the last indexed block
+   */
+  case class IndexInfo(synced: Boolean, bestBlockHeight: Int)
 }

eclair-core/src/main/scala/fr/acinq/eclair/channel/fund/InteractiveTxBuilder.scala

@@ -758,7 +758,7 @@ private class InteractiveTxBuilder(replyTo: ActorRef[InteractiveTxBuilder.Respon
             // unconfirmed inputs, because if they are valid but not in our mempool we would incorrectly consider
             // them unspendable (unknown). We want to reject unspendable inputs to immediately fail the funding
             // attempt, instead of waiting to detect the double-spend later.
-            wallet.isTransactionOutputSpendable(outpoint.txid, outpoint.index.toInt, includeMempool = true)
+            wallet.isTransactionOutputSpendable(outpoint, includeMempool = true)
           case _ => Future.successful(false)
         }
         case Success(false) => Future.successful(false)

eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/MempoolTxMonitor.scala

@@ -200,8 +200,8 @@ private class MempoolTxMonitor(nodeParams: NodeParams,
   private def checkInputStatus(input: OutPoint): Unit = {
     val checkInputTask = for {
       parentConfirmations <- bitcoinClient.getTxConfirmations(input.txid)
-      spendableMempoolExcluded <- bitcoinClient.isTransactionOutputSpendable(input.txid, input.index.toInt, includeMempool = false)
-      spendableMempoolIncluded <- bitcoinClient.isTransactionOutputSpendable(input.txid, input.index.toInt, includeMempool = true)
+      spendableMempoolExcluded <- bitcoinClient.isTransactionOutputSpendable(input, includeMempool = false)
+      spendableMempoolIncluded <- bitcoinClient.isTransactionOutputSpendable(input, includeMempool = true)
     } yield computeInputStatus(parentConfirmations, spendableMempoolExcluded, spendableMempoolIncluded)
     context.pipeToSelf(checkInputTask) {
       case Success(status) => status