[kyoto-hardening] jitter on kyoto restart delay

When the trusted bitcoind peer blips, every Kyoto BIP-157 client across
the fleet reconnects on the same fixed 5s timer, creating a thundering
herd that overwhelms the peer and cascades into more disconnects.

Replace the constant 5s sleep with a base + uniform random jitter in
[0, 30s]. Each pod picks an independent restart instant, spreading the
reconnect load across the upstream node.

Author: 0xsiddharthks

crates/hashi/src/btc_monitor/monitor.rs

@@ -9,6 +9,7 @@ use anyhow::Result;
 use kyoto::FeeRate;
 use kyoto::HeaderCheckpoint;
 use kyoto::Warning;
+use rand::Rng;
 use sui_futures::service::Service;
 use tokio::sync::oneshot;
 use tokio::task::JoinSet;
@@ -26,13 +27,27 @@ const FALLBACK_FEE_RATE_SAT_PER_KWU: u64 = 250;
 /// Number of consecutive connection failures before restarting Kyoto.
 const KYOTO_MAX_CONSECUTIVE_FAILURES: u32 = 15;
 
-/// Delay before restarting Kyoto after connectivity loss.
-const KYOTO_RESTART_DELAY: Duration = Duration::from_secs(5);
+/// Base delay before restarting Kyoto after connectivity loss.
+const KYOTO_RESTART_DELAY_BASE: Duration = Duration::from_secs(5);
+
+/// Maximum random jitter added to the base restart delay. Spreads
+/// reconnect attempts across pods so a fleet of BIP-157 clients sharing a
+/// single trusted peer doesn't dog-pile it in lockstep after a blip.
+const KYOTO_RESTART_DELAY_JITTER: Duration = Duration::from_secs(30);
 
 /// How many Bitcoin blocks a deposit observation can go without being
 /// refreshed before it's dropped from the confirmation-metrics cache.
 const STALE_OBSERVATION_BLOCKS: u32 = 10;
 
+/// Pick the next Kyoto restart delay: a fixed base plus uniform random jitter
+/// in `[0, KYOTO_RESTART_DELAY_JITTER]`.
+fn next_restart_delay() -> Duration {
+    let jitter = Duration::from_millis(
+        rand::thread_rng().gen_range(0..=KYOTO_RESTART_DELAY_JITTER.as_millis() as u64),
+    );
+    KYOTO_RESTART_DELAY_BASE + jitter
+}
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum TxStatus {
     Confirmed { confirmations: u32 },
@@ -232,7 +247,9 @@ impl Monitor {
             self.metrics.kyoto_synced.set(0);
             self.metrics.kyoto_consecutive_failures.set(0);
 
-            tokio::time::sleep(KYOTO_RESTART_DELAY).await;
+            let delay = next_restart_delay();
+            debug!("Sleeping {delay:?} before rebuilding Kyoto");
+            tokio::time::sleep(delay).await;
 
             let (new_node, new_client) = Self::build_kyoto_node(&self.config);
             current_node = new_node;
@@ -1237,4 +1254,14 @@ mod tests {
         assert_eq!(cache.len(), 1);
         assert_eq!(bucket(&metrics, "mempool"), 1);
     }
+
+    #[test]
+    fn next_restart_delay_stays_in_range() {
+        let max = KYOTO_RESTART_DELAY_BASE + KYOTO_RESTART_DELAY_JITTER;
+        for _ in 0..1000 {
+            let d = next_restart_delay();
+            assert!(d >= KYOTO_RESTART_DELAY_BASE, "{d:?} < base");
+            assert!(d <= max, "{d:?} > base + jitter");
+        }
+    }
 }