feat(relay): expose script health telemetry

Add a read-only per-deployment health snapshot over the existing relay state so operators can inspect how deployment selection is behaving without changing the scheduler itself.

The snapshot reports masked script IDs, locally observed rolling quota usage, the configured local quota threshold, saturation state, active cooldown seconds, cooldown reason, and timeout strike count. Cooldown reasons are tracked alongside the existing blacklist timestamps and are pruned whenever expired blacklist entries are removed.

Surface the snapshot in the desktop UI as a collapsible Script health table, clear stale rows when the proxy stops or exits, and document that these values are local client observations rather than authoritative Google-side quota counters.

Add focused unit coverage for quota saturation, cooldown reason exposure, timeout strike visibility, and compact duration formatting. The relay routing, quarantine durations, and selection behavior remain unchanged.

Author: maybeknott

docs/guide.fa.md

@@ -227,6 +227,8 @@ HTTP / HTTPS مثل قبل از Apps Script می‌رود (تغییری نمی
 
 بیشتر Deployment = همزمانی بیشتر = تأخیر کمتر هر سشن. انتخاب هر بَچ از بین IDهای تنظیم‌شده با یک ledger محلی rolling 24-hour انجام می‌شود؛ بار پخش می‌شود و کلاینت از Deploymentهایی که همین دستگاه نزدیک سقف request سهمیهٔ رایگان برده دوری می‌کند.
 
+پنل **Script health** در UI دسکتاپ همین وضعیت محلی را فقط به‌صورت read-only نشان می‌دهد: Deployment ID ماسک‌شده، تعداد callهای مشاهده‌شده در پنجرهٔ rolling 24-hour، اینکه threshold محلی free-tier اشباع شده یا نه، cooldown باقی‌مانده، دلیل/کلاس خطایی که آن cooldown را ساخته، و تعداد timeout strikeهای فعلی. این فقط telemetry سمت کلاینت است؛ اگر دستگاه‌های دیگر هم از همان deployment استفاده کنند، Google ممکن است callهای بیشتری شمرده باشد.
+
 **محافظ‌های منابع:**
 - **حداکثر ۵۰ op** در هر بَچ — اگر سشن‌های فعال بیشتر باشند، مالتی‌پلکسر چند بَچ می‌فرستد
 - **سقف payload ۴ مگابایت** در هر بَچ — خیلی کمتر از ۵۰ مگابایت Apps Script

docs/guide.md

@@ -227,6 +227,8 @@ max_concurrent = 30 × number_of_deployment_ids
 
 More deployments = more total concurrency = lower per-session latency. Each batch is selected from the configured IDs with a local rolling 24-hour ledger, spreading load and steering away from deployments this client has already driven near the free-tier request budget.
 
+The desktop **Script health** panel shows this local state without changing routing behavior: masked deployment ID, locally observed calls inside the rolling 24-hour window, whether the local free-tier steering threshold is saturated, any remaining cooldown, the failure class/reason that set that cooldown, and the current timeout-strike count. Treat it as client-side telemetry only; Google may also count requests from other devices using the same deployment.
+
 **Resource guards:**
 - **50 ops max** per batch — if more sessions are active, the mux splits into multiple batches
 - **4 MB payload cap** per batch — well under Apps Script's 50 MB limit

src/bin/ui.rs

@@ -117,6 +117,7 @@ struct UiState {
     running: bool,
     started_at: Option<Instant>,
     last_stats: Option<mhrv_rs::domain_fronter::StatsSnapshot>,
+    last_script_health: Vec<mhrv_rs::domain_fronter::ScriptHealthSnapshot>,
     last_per_site: Vec<(String, mhrv_rs::domain_fronter::HostStat)>,
     log: VecDeque<String>,
     /// Result + timestamp for transient status banners (auto-hide after 10s).
@@ -1156,7 +1157,7 @@ impl eframe::App for App {
             ui.add_space(8.0);
 
             // ── Status + stats card ────────────────────────────────────────
-            let (running, started_at, stats, ca_trusted, last_test_msg, per_site) = {
+            let (running, started_at, stats, ca_trusted, last_test_msg, per_site, script_health) = {
                 let s = self.shared.state.lock().unwrap();
                 (
                     s.running,
@@ -1165,6 +1166,7 @@ impl eframe::App for App {
                     s.ca_trusted,
                     s.last_test_msg.clone(),
                     s.last_per_site.clone(),
+                    s.last_script_health.clone(),
                 )
             };
 
@@ -1318,6 +1320,66 @@ impl eframe::App for App {
                 });
             }
 
+            if !script_health.is_empty() {
+                ui.add_space(2.0);
+                egui::CollapsingHeader::new(format!(
+                    "Script health ({} deployments)",
+                    script_health.len()
+                ))
+                .default_open(false)
+                .show(ui, |ui| {
+                    egui::ScrollArea::vertical()
+                        .max_height(160.0)
+                        .show(ui, |ui| {
+                            egui::Grid::new("script_health")
+                                .num_columns(5)
+                                .spacing([8.0, 2.0])
+                                .striped(true)
+                                .show(ui, |ui| {
+                                    ui.label(egui::RichText::new("script").strong());
+                                    ui.label(egui::RichText::new("quota").strong());
+                                    ui.label(egui::RichText::new("cooldown").strong());
+                                    ui.label(egui::RichText::new("reason").strong());
+                                    ui.label(egui::RichText::new("timeouts").strong());
+                                    ui.end_row();
+
+                                    for st in &script_health {
+                                        let quota = format!(
+                                            "{} / {}{}",
+                                            st.quota_used,
+                                            st.quota_limit,
+                                            if st.quota_saturated { " saturated" } else { "" }
+                                        );
+                                        let cooldown = st
+                                            .cooldown_secs
+                                            .map(fmt_seconds_compact)
+                                            .unwrap_or_else(|| "-".to_string());
+                                        let reason = st
+                                            .cooldown_reason
+                                            .as_deref()
+                                            .unwrap_or("-")
+                                            .to_string();
+                                        ui.label(egui::RichText::new(&st.script_id).monospace());
+                                        ui.label(egui::RichText::new(quota).monospace());
+                                        ui.label(egui::RichText::new(cooldown).monospace());
+                                        ui.label(egui::RichText::new(reason).small());
+                                        ui.label(
+                                            egui::RichText::new(st.timeout_strikes.to_string())
+                                                .monospace(),
+                                        );
+                                        ui.end_row();
+                                    }
+                                });
+                        });
+                    ui.small(
+                        egui::RichText::new(
+                            "Local view only: Google quota can also be consumed by other clients using the same deployment.",
+                        )
+                        .color(egui::Color32::from_gray(130)),
+                    );
+                });
+            }
+
             if !per_site.is_empty() {
                 ui.add_space(2.0);
                 egui::CollapsingHeader::new(format!("Per-site ({} hosts)", per_site.len()))
@@ -1949,6 +2011,16 @@ fn fmt_duration(d: Duration) -> String {
     format!("{:02}:{:02}:{:02}", s / 3600, (s / 60) % 60, s % 60)
 }
 
+fn fmt_seconds_compact(seconds: u64) -> String {
+    if seconds >= 3600 {
+        format!("{}h {}m", seconds / 3600, (seconds / 60) % 60)
+    } else if seconds >= 60 {
+        format!("{}m {}s", seconds / 60, seconds % 60)
+    } else {
+        format!("{}s", seconds)
+    }
+}
+
 fn fmt_bytes(b: u64) -> String {
     const K: u64 = 1024;
     const M: u64 = K * K;
@@ -1986,9 +2058,11 @@ fn background_thread(shared: Arc<Shared>, rx: Receiver<Cmd>) {
                         if let Some(fronter) = f.as_ref() {
                             let s = fronter.snapshot_stats();
                             let per_site = fronter.snapshot_per_site();
+                            let script_health = fronter.snapshot_script_health();
                             let mut st = shared.state.lock().unwrap();
                             st.last_stats = Some(s);
                             st.last_per_site = per_site;
+                            st.last_script_health = script_health;
                         }
                     });
                 }
@@ -2064,6 +2138,7 @@ fn background_thread(shared: Arc<Shared>, rx: Receiver<Cmd>) {
                         // or normal shutdown without Cmd::Stop). The
                         // Stop handler clears this too — either is fine.
                         st.proxy_active = false;
+                        st.last_script_health.clear();
                     }
                     push_log(&shared2, "[ui] proxy stopped");
                 });
@@ -2094,6 +2169,7 @@ fn background_thread(shared: Arc<Shared>, rx: Receiver<Cmd>) {
                     st.running = false;
                     st.started_at = None;
                     st.proxy_active = false;
+                    st.last_script_health.clear();
                 }
             }
 
@@ -2568,3 +2644,15 @@ fn push_log(shared: &Shared, msg: &str) {
         s.log.pop_front();
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::fmt_seconds_compact;
+
+    #[test]
+    fn compact_seconds_formatter_scales_units() {
+        assert_eq!(fmt_seconds_compact(9), "9s");
+        assert_eq!(fmt_seconds_compact(125), "2m 5s");
+        assert_eq!(fmt_seconds_compact(3660), "1h 1m");
+    }
+}

src/domain_fronter.rs

@@ -363,6 +363,10 @@ pub struct DomainFronter {
     inflight: Arc<Mutex<HashMap<String, broadcast::Sender<Vec<u8>>>>>,
     coalesced: AtomicU64,
     blacklist: Arc<std::sync::Mutex<HashMap<String, Instant>>>,
+    /// Human-readable cooldown reason keyed by script ID. Kept separate
+    /// from `blacklist` so the selection hot path still only checks the
+    /// timestamp map while diagnostics can explain the active cooldown.
+    script_cooldown_reasons: Arc<std::sync::Mutex<HashMap<String, String>>>,
     /// Per-deployment local call ledger used by `next_script_id` /
     /// `next_script_ids` to avoid selecting an already saturated deployment
     /// while another configured script still has locally-observed capacity.
@@ -654,6 +658,7 @@ impl DomainFronter {
             inflight: Arc::new(Mutex::new(HashMap::new())),
             coalesced: AtomicU64::new(0),
             blacklist: Arc::new(std::sync::Mutex::new(HashMap::new())),
+            script_cooldown_reasons: Arc::new(std::sync::Mutex::new(HashMap::new())),
             script_quota_ledger: Arc::new(std::sync::Mutex::new(HashMap::new())),
             script_timeouts: Arc::new(std::sync::Mutex::new(HashMap::new())),
             relay_calls: AtomicU64::new(0),
@@ -816,6 +821,43 @@ impl DomainFronter {
         }
     }
 
+    pub fn snapshot_script_health(&self) -> Vec<ScriptHealthSnapshot> {
+        let now = Instant::now();
+        let bl = self.blacklist.lock().unwrap();
+        let reasons = self.script_cooldown_reasons.lock().unwrap();
+        let quota = self.script_quota_ledger.lock().unwrap();
+        let timeouts = self.script_timeouts.lock().unwrap();
+
+        self.script_ids
+            .iter()
+            .map(|sid| {
+                let quota_used = quota
+                    .get(sid)
+                    .map(|calls| {
+                        calls
+                            .iter()
+                            .filter(|at| now.saturating_duration_since(**at) < SCRIPT_QUOTA_WINDOW)
+                            .count()
+                    })
+                    .unwrap_or(0);
+                let cooldown_secs = bl
+                    .get(sid)
+                    .map(|until| until.saturating_duration_since(now).as_secs())
+                    .filter(|secs| *secs > 0);
+                let timeout_strikes = timeouts.get(sid).map(|(_, strikes)| *strikes).unwrap_or(0);
+                ScriptHealthSnapshot {
+                    script_id: mask_script_id(sid),
+                    quota_used,
+                    quota_limit: SCRIPT_QUOTA_FREE_TIER_CALLS,
+                    quota_saturated: quota_used >= SCRIPT_QUOTA_FREE_TIER_CALLS,
+                    cooldown_secs,
+                    cooldown_reason: reasons.get(sid).cloned(),
+                    timeout_strikes,
+                }
+            })
+            .collect()
+    }
+
     pub fn num_scripts(&self) -> usize {
         self.script_ids.len()
     }
@@ -837,6 +879,10 @@ impl DomainFronter {
         let mut bl = self.blacklist.lock().unwrap();
         let now = Instant::now();
         bl.retain(|_, until| *until > now);
+        self.script_cooldown_reasons
+            .lock()
+            .unwrap()
+            .retain(|sid, _| bl.contains_key(sid));
         let mut quota = self.script_quota_ledger.lock().unwrap();
         prune_script_quota_ledger(&mut quota, now);
 
@@ -864,6 +910,7 @@ impl DomainFronter {
         if let Some((sid, _)) = bl.iter().min_by_key(|(_, t)| **t) {
             let sid = sid.clone();
             bl.remove(&sid);
+            self.script_cooldown_reasons.lock().unwrap().remove(&sid);
             record_script_quota_call_locked(&mut quota, &sid, now);
             return sid;
         }
@@ -884,6 +931,10 @@ impl DomainFronter {
         let mut bl = self.blacklist.lock().unwrap();
         let now = Instant::now();
         bl.retain(|_, until| *until > now);
+        self.script_cooldown_reasons
+            .lock()
+            .unwrap()
+            .retain(|sid, _| bl.contains_key(sid));
         let mut quota = self.script_quota_ledger.lock().unwrap();
         prune_script_quota_ledger(&mut quota, now);
 
@@ -928,6 +979,10 @@ impl DomainFronter {
         let until = Instant::now() + cooldown;
         let mut bl = self.blacklist.lock().unwrap();
         bl.insert(script_id.to_string(), until);
+        self.script_cooldown_reasons
+            .lock()
+            .unwrap()
+            .insert(script_id.to_string(), reason.to_string());
         tracing::warn!(
             "blacklisted script {} for {}s: {}",
             mask_script_id(script_id),
@@ -4946,6 +5001,24 @@ pub struct StatsSnapshot {
     pub h2_disabled: bool,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ScriptHealthSnapshot {
+    /// Masked deployment ID (`prefix...suffix`) safe to render in logs/UI.
+    pub script_id: String,
+    /// Locally observed calls inside the rolling 24-hour steering window.
+    pub quota_used: usize,
+    /// Local free-tier steering threshold. This is not an authoritative
+    /// Google-side quota read; it is the client-side selection guard.
+    pub quota_limit: usize,
+    pub quota_saturated: bool,
+    /// Remaining local cooldown, if the deployment is currently sidelined.
+    pub cooldown_secs: Option<u64>,
+    /// Human-readable failure class/reason used when the cooldown was set.
+    pub cooldown_reason: Option<String>,
+    /// Current timeout strikes inside the auto-blacklist rolling window.
+    pub timeout_strikes: u32,
+}
+
 impl StatsSnapshot {
     pub fn hit_rate(&self) -> f64 {
         let total = self.cache_hits + self.cache_misses;
@@ -5183,6 +5256,15 @@ mod tests {
     use std::task::{Context, Poll};
     use tokio::io::{duplex, AsyncRead, AsyncWriteExt, ReadBuf};
 
+    fn find_script_health<'a>(
+        rows: &'a [ScriptHealthSnapshot],
+        masked_script_id: &str,
+    ) -> &'a ScriptHealthSnapshot {
+        rows.iter()
+            .find(|row| row.script_id == masked_script_id)
+            .expect("script health row must exist")
+    }
+
     // Test fixture for ungraceful TLS close: emit a fixed prefix of bytes
     // then return io::ErrorKind::UnexpectedEof on the next read. Mirrors
     // what rustls surfaces when the peer closes TCP without sending a
@@ -6536,6 +6618,51 @@ hello";
         );
     }
 
+    #[test]
+    fn script_health_snapshot_exposes_quota_and_cooldown_state() {
+        let fronter = fronter_for_script_ids(&["SCRIPT_A", "SCRIPT_B"]);
+        let now = Instant::now();
+        seed_script_quota(&fronter, "SCRIPT_A", 3, now);
+        fronter.blacklist_script_for(
+            "SCRIPT_A",
+            Duration::from_secs(600),
+            "transient relay cooldown: HTTP 502",
+        );
+        fronter.record_timeout_strike("SCRIPT_B");
+
+        let rows = fronter.snapshot_script_health();
+        let a = find_script_health(&rows, "SCRI...PT_A");
+        let b = find_script_health(&rows, "SCRI...PT_B");
+
+        assert_eq!(a.quota_used, 3);
+        assert_eq!(a.quota_limit, SCRIPT_QUOTA_FREE_TIER_CALLS);
+        assert!(!a.quota_saturated);
+        assert!(a.cooldown_secs.is_some_and(|secs| secs <= 600 && secs > 0));
+        assert_eq!(
+            a.cooldown_reason.as_deref(),
+            Some("transient relay cooldown: HTTP 502")
+        );
+        assert_eq!(b.timeout_strikes, 1);
+        assert!(b.cooldown_secs.is_none());
+    }
+
+    #[test]
+    fn script_health_snapshot_marks_local_quota_saturation() {
+        let fronter = fronter_for_script_ids(&["SCRIPT_A"]);
+        seed_script_quota(
+            &fronter,
+            "SCRIPT_A",
+            SCRIPT_QUOTA_FREE_TIER_CALLS,
+            Instant::now(),
+        );
+
+        let rows = fronter.snapshot_script_health();
+        let row = find_script_health(&rows, "SCRI...PT_A");
+
+        assert_eq!(row.quota_used, SCRIPT_QUOTA_FREE_TIER_CALLS);
+        assert!(row.quota_saturated);
+    }
+
     #[test]
     fn mask_script_id_hides_middle() {
         assert_eq!(mask_script_id("short"), "***");