Use bitcoin core's txospenderindex if available

If bitcoin core's txospenderindex is enabled and sync, we'll automatically use it to find transactions that spent our channels.

Author: sstone

docs/release-notes/eclair-vnext.md

@@ -231,6 +231,11 @@ eclair.relay.peer-reputation.enabled = false
 eclair.relay.reserved-for-accountable = 0.0
 ```
 
+### Faster scanning for spending transactions with Bitcoin Core's txospenderindex
+
+If Bitcoin Core's `txospenderindex` (available in Bitcoin Core 31.0 and newer) is available and synced, Eclair will use it to find channel spending transactions, which is much faster and less expensive than scanning blocks.
+To enable this index, start Bitcoin Core with `-txospenderindex` or add `txospenderindex=1` to your `bitcoin.conf`.
+
 ### Configuration changes
 
 <insert changes>

eclair-core/pom.xml

@@ -87,8 +87,8 @@
                 <activeByDefault>true</activeByDefault>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-x86_64-linux-gnu.tar.gz</bitcoind.url>
-                <bitcoind.sha256>6aa7bb4feb699c4c6262dd23e4004191f6df7f373b5d5978b5bcdd4bb72f75d8</bitcoind.sha256>
+                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-x86_64-linux-gnu.tar.gz</bitcoind.url>
+                <bitcoind.sha256>d3e4c58a35b1d0a97a457462c94f55501ad167c660c245cb1ffa565641c65074</bitcoind.sha256>
             </properties>
         </profile>
         <profile>
@@ -99,8 +99,8 @@
                 </os>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-x86_64-apple-darwin.tar.gz</bitcoind.url>
-                <bitcoind.sha256>99d5cee9b9c37be506396c30837a4b98e320bfea71c474d6120a7e8eb6075c7b</bitcoind.sha256>
+                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-x86_64-apple-darwin.tar.gz</bitcoind.url>
+                <bitcoind.sha256>56824dd705bc2a3b22d42e8aa02ed53498d491ff7c2c8aa96831333871887ead</bitcoind.sha256>
             </properties>
         </profile>
         <profile>
@@ -111,8 +111,8 @@
                 </os>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-win64.zip</bitcoind.url>
-                <bitcoind.sha256>0d7e1f16f8823aa26d29b44855ff6dbac11c03d75631a6c1d2ea5fab3a84fdf8</bitcoind.sha256>
+                <bitcoind.url>https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-win64.zip</bitcoind.url>
+                <bitcoind.sha256>82fd2c504a0f20a31d4d13bd407783d6fc7bf17622d0ce85228a9b92694e03f0</bitcoind.sha256>
             </properties>
         </profile>
     </profiles>

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala

@@ -429,6 +429,8 @@ private class ZmqWatcher(nodeParams: NodeParams, blockHeight: AtomicLong, client
     }
   }
 
+  private def canUseTxoSpenderIndex: Future[Boolean] = client.getIndexInfo().map(_.get("txospenderindex").exists(_.synced))
+
   private def checkSpent(w: WatchSpent[_ <: WatchSpentTriggered]): Future[Unit] = {
     // First let's see if the parent tx was published or not before checking whether it has been spent.
     client.getTxConfirmations(w.txId).collect {
@@ -445,37 +447,52 @@ private class ZmqWatcher(nodeParams: NodeParams, blockHeight: AtomicLong, client
         case _ =>
           // The parent tx was published, we need to make sure this particular output has not been spent.
           client.isTransactionOutputSpendable(w.outPoint, includeMempool = true).collect {
-            case false =>
-              // The output has been spent, let's find the spending tx.
-              // If we know some potential spending txs, we try to fetch them directly.
-              Future.sequence(w.hints.map(txid => client.getTransaction(txid).map(Some(_)).recover { case _ => None }))
-                .map(_.flatten) // filter out errors and hint transactions that can't be found
-                .map(hintTxs => {
-                  hintTxs.find(tx => tx.txIn.exists(i => i.outPoint.txid == w.txId && i.outPoint.index == w.outputIndex)) match {
-                    case Some(spendingTx) =>
-                      log.info("{}:{} has already been spent by a tx provided in hints: txid={}", w.txId, w.outputIndex, spendingTx.txid)
-                      context.self ! ProcessNewTransaction(spendingTx)
-                    case None =>
-                      // The hints didn't help us, let's search for the spending transaction in the mempool.
-                      log.info("{}:{} has already been spent, looking for the spending tx in the mempool", w.txId, w.outputIndex)
-                      client.lookForMempoolSpendingTx(w.outPoint).map(Some(_)).recover { case _ => None }.map {
-                        case Some(spendingTx) =>
-                          log.info("found tx spending {}:{} in the mempool: txid={}", w.txId, w.outputIndex, spendingTx.txid)
-                          context.self ! ProcessNewTransaction(spendingTx)
-                        case None =>
-                          // The spending transaction isn't in the mempool, so it must be a transaction that confirmed
-                          // before we set the watch. We have to scan the blockchain to find it, which is expensive
-                          // since bitcoind doesn't provide indexes for this scenario.
-                          log.warn("{}:{} has already been spent, spending tx not in the mempool, looking in the blockchain...", w.txId, w.outputIndex)
-                          client.lookForSpendingTx(None, w.outPoint, nodeParams.channelConf.maxChannelSpentRescanBlocks).map { spendingTx =>
-                            log.warn("found the spending tx of {}:{} in the blockchain: txid={}", w.txId, w.outputIndex, spendingTx.txid)
+            case false => canUseTxoSpenderIndex.foreach {
+              case true =>
+                // The output has been spent, let's find the spending tx in the txospenderindex.
+                log.info("{} has already been spent, looking for the spending tx", w.outPoint)
+                client.findSpendingTx(w.outPoint) map {
+                  case Some((spendingTx, _)) =>
+                    log.info("found tx spending {} txid={}", w.outPoint, spendingTx.txid)
+                    context.self ! ProcessNewTransaction(spendingTx)
+                  case None =>
+                    log.warn("could not find the spending tx of {}, funds are at risk", w.outPoint)
+                } recover {
+                  case error =>
+                    log.warn("error finding the spending tx of {}, funds are at risk", w.outPoint, error)
+                }
+              case false =>
+                // txospenderindex is not available, scan the mempool and the blockchain
+                // If we know some potential spending txs, we try to fetch them directly.
+                Future.sequence(w.hints.map(txid => client.getTransaction(txid).map(Some(_)).recover { case _ => None }))
+                  .map(_.flatten) // filter out errors and hint transactions that can't be found
+                  .map(hintTxs => {
+                    hintTxs.find(tx => tx.txIn.exists(i => i.outPoint.txid == w.txId && i.outPoint.index == w.outputIndex)) match {
+                      case Some(spendingTx) =>
+                        log.info("{}:{} has already been spent by a tx provided in hints: txid={}", w.txId, w.outputIndex, spendingTx.txid)
+                        context.self ! ProcessNewTransaction(spendingTx)
+                      case None =>
+                        // The hints didn't help us, let's search for the spending transaction in the mempool.
+                        log.info("{}:{} has already been spent, looking for the spending tx in the mempool", w.txId, w.outputIndex)
+                        client.lookForMempoolSpendingTx(w.outPoint).map(Some(_)).recover { case _ => None }.map {
+                          case Some(spendingTx) =>
+                            log.info("found tx spending {}:{} in the mempool: txid={}", w.txId, w.outputIndex, spendingTx.txid)
                             context.self ! ProcessNewTransaction(spendingTx)
-                          }.recover {
-                            case _ => log.warn("could not find the spending tx of {}:{} in the blockchain, funds are at risk", w.txId, w.outputIndex)
-                          }
-                      }
-                  }
-                })
+                          case None =>
+                            // The spending transaction isn't in the mempool, so it must be a transaction that confirmed
+                            // before we set the watch. We have to scan the blockchain to find it, which is expensive
+                            // since bitcoind doesn't provide indexes for this scenario.
+                            log.warn("{}:{} has already been spent, spending tx not in the mempool, looking in the blockchain...", w.txId, w.outputIndex)
+                            client.lookForSpendingTx(None, w.outPoint, nodeParams.channelConf.maxChannelSpentRescanBlocks).map { spendingTx =>
+                              log.warn("found the spending tx of {}:{} in the blockchain: txid={}", w.txId, w.outputIndex, spendingTx.txid)
+                              context.self ! ProcessNewTransaction(spendingTx)
+                            }.recover {
+                              case _ => log.warn("could not find the spending tx of {}:{} in the blockchain, funds are at risk", w.txId, w.outputIndex)
+                            }
+                        }
+                    }
+                  })
+            }
           }
       }
     }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala

@@ -172,7 +172,8 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool
 
   /** Search for mempool transaction spending a given output. */
   def lookForMempoolSpendingTx(outPoint: OutPoint)(implicit ec: ExecutionContext): Future[Transaction] = {
-    rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(outPoint.txid, outPoint.index))).collect {
+    val options = JObject(List("return_spending_tx" -> JBool(true), "mempool_only" -> JBool(true)))
+    rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(outPoint.txid, outPoint.index)), options).collect {
       case JArray(results) => results.flatMap(result => (result \ "spendingtxid").extractOpt[String].map(TxId.fromValidHex))
     }.flatMap { spendingTxIds =>
       spendingTxIds.headOption match {
@@ -182,6 +183,22 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool
     }
   }
 
+  /**
+   * Find the transaction spending a given output. Requires `txospenderindex` on the bitcoin code node we're connecting to.
+   *
+   * @param outPoint transaction output
+   * @return the transaction that spent this output along with the id of the block it was published in if any, or None if no spending transaction was found
+   */
+  def findSpendingTx(outPoint: OutPoint)(implicit ec: ExecutionContext): Future[Option[(Transaction, Option[BlockId])]] = {
+    val options = JObject(List("return_spending_tx" -> JBool(true)))
+    rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(outPoint.txid, outPoint.index)), options).collect {
+      case JArray(results) => results.flatMap(result => {
+        val tx_opt = (result \ "spendingtx").extractOpt[String].map(Transaction.read)
+        tx_opt.map(tx => tx -> (result \ "blockhash").extractOpt[String].map(s => BlockId(ByteVector32.fromValidHex(s))))
+      }).headOption
+    }
+  }
+
   /**
    * Iterate over blocks to find the transaction that has spent a given output.
    * It isn't useful to look at the whole blockchain history: if the transaction was confirmed long ago, an attacker
@@ -794,6 +811,15 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool
     })
   }
 
+  //------------------------- MISC  -------------------------//
+
+  /**
+   *
+   * @return information about enabled bitcoin core indexes, in a map where the key is the index name
+   */
+  def getIndexInfo()(implicit ec: ExecutionContext): Future[Map[String, IndexInfo]] = rpcClient.invoke("getindexinfo").collect {
+    case JObject(results) => results.map { case (name, o) => name -> BitcoinCoreClient.IndexInfo((o \ "synced").extract[Boolean], (o \ "best_block_height").extract[Int]) }.toMap
+  }
 }
 
 object BitcoinCoreClient {
@@ -871,4 +897,11 @@ object BitcoinCoreClient {
     // @formatter:on
   }
 
+  /**
+   * Information about a bitcoin core inedx
+   *
+   * @param synced          true if the index is synced
+   * @param bestBlockHeight height of the last indexed block
+   */
+  case class IndexInfo(synced: Boolean, bestBlockHeight: Int)
 }

eclair-core/src/test/resources/integration/bitcoin.conf

@@ -4,6 +4,7 @@ server=1
 rpcuser=foo
 rpcpassword=bar
 txindex=1
+txospenderindex=1
 zmqpubhashblock=tcp://127.0.0.1:28334
 zmqpubrawtx=tcp://127.0.0.1:28335
 rpcworkqueue=64

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala

@@ -1762,6 +1762,71 @@ class BitcoinCoreClientSpec extends TestKitBaseClass with BitcoindService with A
     sender.expectMsg(tx1)
   }
 
+  test("get index information") {
+    val sender = TestProbe()
+    val bitcoinClient = makeBitcoinCoreClient()
+    bitcoinClient.getIndexInfo().pipeTo(sender.ref)
+    val indexInfos = sender.expectMsgType[Map[String, BitcoinCoreClient.IndexInfo]]
+    assert(indexInfos("txindex").synced)
+    assert(indexInfos("txospenderindex").synced)
+  }
+
+  test("find spending transaction of a given output using txospenderindex") {
+    val sender = TestProbe()
+    val bitcoinClient = makeBitcoinCoreClient()
+
+    bitcoinClient.getBlockHeight().pipeTo(sender.ref)
+    val blockHeight = sender.expectMsgType[BlockHeight]
+
+    bitcoinClient.getIndexInfo().pipeTo(sender.ref)
+    val indexInfos = sender.expectMsgType[Map[String, BitcoinCoreClient.IndexInfo]]
+    assert(indexInfos("txindex").synced)
+
+    val address = getNewAddress(sender)
+    val tx1 = sendToAddress(address, 5 btc)
+
+    // Transaction is still in the mempool at that point
+    bitcoinClient.getTxConfirmations(tx1.txid).pipeTo(sender.ref)
+    sender.expectMsg(Some(0))
+    // If we omit the mempool, tx1's input is still considered unspent.
+    bitcoinClient.isTransactionOutputSpendable(tx1.txIn.head.outPoint, includeMempool = false).pipeTo(sender.ref)
+    sender.expectMsg(true)
+    // If we include the mempool, we see that tx1's input is now spent.
+    bitcoinClient.isTransactionOutputSpendable(tx1.txIn.head.outPoint, includeMempool = true).pipeTo(sender.ref)
+    sender.expectMsg(false)
+    // If we omit the mempool, tx1's output is not considered spendable because we can't even find that output.
+    bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = false).pipeTo(sender.ref)
+    sender.expectMsg(false)
+    // If we include the mempool, we see that tx1 produces an output that is still unspent.
+    bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = true).pipeTo(sender.ref)
+    sender.expectMsg(true)
+    // We're able to find the spending transaction in the mempool.
+    bitcoinClient.findSpendingTx(tx1.txIn.head.outPoint).pipeTo(sender.ref)
+    sender.expectMsg(Some(tx1, None))
+
+    // Let's confirm our transaction.
+    generateBlocks(1)
+    bitcoinClient.getBlockHeight().pipeTo(sender.ref)
+    val blockHeight1 = sender.expectMsgType[BlockHeight]
+    assert(blockHeight1 == blockHeight + 1)
+    bitcoinClient.getBlockId(blockHeight1.toInt).pipeTo(sender.ref)
+    val tip = sender.expectMsgType[BlockId]
+    bitcoinClient.findSpendingTx(tx1.txIn.head.outPoint).pipeTo(sender.ref)
+    sender.expectMsg(Some(tx1, Some(tip)))
+
+    bitcoinClient.getTxConfirmations(tx1.txid).pipeTo(sender.ref)
+    sender.expectMsg(Some(1))
+    bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = false).pipeTo(sender.ref)
+    sender.expectMsg(true)
+    bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = true).pipeTo(sender.ref)
+    sender.expectMsg(true)
+
+    // we still fid the spending tx evn if it has been confirmed many times
+    generateBlocks(10)
+    bitcoinClient.findSpendingTx(tx1.txIn.head.outPoint).pipeTo(sender.ref)
+    sender.expectMsg(Some(tx1, Some(tip)))
+  }
+
   test("get pubkey for p2wpkh receive address") {
     val sender = TestProbe()
     val bitcoinClient = makeBitcoinCoreClient()

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala

@@ -62,7 +62,7 @@ trait BitcoindService extends Logging {
 
   val PATH_BITCOIND = sys.env.get("BITCOIND_DIR") match {
     case Some(customBitcoinDir) => new File(customBitcoinDir, "bitcoind")
-    case None => new File(TestUtils.BUILD_DIRECTORY, "bitcoin-30.2/bin/bitcoind")
+    case None => new File(TestUtils.BUILD_DIRECTORY, "bitcoin-31.0/bin/bitcoind")
   }
   logger.info(s"using bitcoind: $PATH_BITCOIND")
   val PATH_BITCOIND_DATADIR = new File(INTEGRATION_TMP_DIR, "datadir-bitcoin")