Skip to content

Commit f056016

Browse files
committed
fix(quota_tracker): filter global hard stop aggregate to configured IDs, fix 503 response
is_globally_hard_stopped() was summing quota_error_count and requests_used over all persisted buckets, including stale ones from script IDs removed from config. A user rotating away exhausted IDs would still carry their usage in quota_state.json, causing the aggregate check to falsely trip a global hard stop against fresh accounts. Fixed by filtering both sums to self.script_ids only. The all_stopped primary check was already correct (iterates script_ids, not bucket values). Also corrects the hard-stop HTTP response from 502 Bad Gateway to 503 Service Unavailable, which is the accurate status for a deliberately refused request due to resource exhaustion. Regression test: one stale exhausted persisted bucket not in the current config plus one fresh configured bucket must not trigger a global hard stop.
1 parent 957208a commit f056016

2 files changed

Lines changed: 73 additions & 3 deletions

File tree

src/domain_fronter.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1797,7 +1797,7 @@ impl DomainFronter {
17971797
"[quota] global hard stop active — all Apps Script account buckets exhausted"
17981798
);
17991799
return error_response(
1800-
502,
1800+
503,
18011801
"All Apps Script accounts quota exhausted; hard stop active. \
18021802
Quota resets on a rolling 24-hour window per account.",
18031803
);

src/quota_tracker.rs

Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -359,11 +359,20 @@ impl QuotaTracker {
359359
// Secondary check: aggregate remaining < N × safety_buffer
360360
// AND at least one quota error signal has been seen (not just network
361361
// failures or local disconnects).
362-
let total_quota_errors: u64 = st.buckets.values().map(|b| b.quota_error_count).sum();
362+
// Only sum over currently configured script_ids so stale buckets from
363+
// removed accounts don't inflate the used count or error tally and
364+
// falsely trip this check.
365+
let total_quota_errors: u64 = self.script_ids.iter()
366+
.filter_map(|sid| st.buckets.get(sid))
367+
.map(|b| b.quota_error_count)
368+
.sum();
363369
if total_quota_errors == 0 {
364370
return false;
365371
}
366-
let total_used: u64 = st.buckets.values().map(|b| b.requests_used).sum();
372+
let total_used: u64 = self.script_ids.iter()
373+
.filter_map(|sid| st.buckets.get(sid))
374+
.map(|b| b.requests_used)
375+
.sum();
367376
let total_cap = self.daily_limit * self.script_ids.len() as u64;
368377
let total_remaining = total_cap.saturating_sub(total_used);
369378
let aggregate_reserve = self.safety_buffer * self.script_ids.len() as u64;
@@ -559,3 +568,64 @@ impl Drop for QuotaTracker {
559568
self.save();
560569
}
561570
}
571+
572+
#[cfg(test)]
573+
impl QuotaTracker {
574+
fn new_for_test(
575+
script_ids: Vec<String>,
576+
daily_limit: u64,
577+
safety_buffer: u64,
578+
state: QuotaState,
579+
) -> Self {
580+
Self {
581+
state: Mutex::new(state),
582+
script_ids,
583+
daily_limit,
584+
safety_buffer,
585+
dirty_count: AtomicU64::new(0),
586+
state_path: std::path::PathBuf::from("/dev/null"),
587+
}
588+
}
589+
}
590+
591+
#[cfg(test)]
592+
mod tests {
593+
use super::*;
594+
595+
/// A stale exhausted bucket from a removed script ID must not cause a
596+
/// global hard stop when the currently configured ID is fresh and healthy.
597+
#[test]
598+
fn stale_exhausted_bucket_does_not_trigger_global_hard_stop() {
599+
let stale_id = "stale_removed_aaaa1111bbbb2222cccc".to_string();
600+
let active_id = "active_fresh_xxxx9999yyyy8888zzzz".to_string();
601+
602+
let mut state = QuotaState::default();
603+
state.buckets.insert(stale_id.clone(), AccountBucket {
604+
masked_id: mask_id(&stale_id),
605+
requests_used: 19_500,
606+
quota_error_count: 5,
607+
exhausted: true,
608+
hard_stopped: true,
609+
exhaustion_reason: Some("quota exhausted".into()),
610+
..Default::default()
611+
});
612+
state.buckets.insert(active_id.clone(), AccountBucket {
613+
masked_id: mask_id(&active_id),
614+
requests_used: 100,
615+
..Default::default()
616+
});
617+
618+
// Only active_id is in the live config — stale_id was removed.
619+
let tracker = QuotaTracker::new_for_test(
620+
vec![active_id],
621+
20_000,
622+
500,
623+
state,
624+
);
625+
626+
assert!(
627+
!tracker.is_globally_hard_stopped(),
628+
"stale exhausted bucket from a removed script_id should not trigger global hard stop"
629+
);
630+
}
631+
}

0 commit comments

Comments
 (0)