WIP: provider reliability fixes (#369)

- Fail fast on non-retryable client errors (#366)
- Retry-After parsing + per-provider cooldown + fallback skip (#363)
- Paid-subscription 403 as billing with fallback (#364)
- Duplicate API request dump suppression (#367)

Co-authored-by: Hermes Evolution <evolution@hermes.ai>

Author: Lexus2016

agent/agent_runtime_helpers.py

@@ -5,6 +5,8 @@
 rate-limited provider concurrently.
 """
 
+import calendar
+import email.utils
 import random
 import threading
 import time
@@ -47,11 +49,43 @@ def jittered_backoff(
     if exponent >= 63 or base_delay <= 0:
         delay = max_delay
     else:
-        delay = min(base_delay * (2 ** exponent), max_delay)
+        delay = min(base_delay * (2**exponent), max_delay)
 
     # Seed from time + counter for decorrelation even with coarse clocks.
     seed = (time.time_ns() ^ (tick * 0x9E3779B9)) & 0xFFFFFFFF
     rng = random.Random(seed)
     jitter = rng.uniform(0, jitter_ratio * delay)
 
     return delay + jitter
+
+
+def extract_retry_after_seconds(
+    retry_after: str | None, now: float | None = None
+) -> float | None:
+    """Parse a Retry-After value into seconds from now.
+
+    Supports both integer seconds (RFC 7231 §7.1.3) and HTTP-date strings
+    (RFC 7231 §7.1.1.2).  Returns None for malformed/empty values.
+    Caps the result at 300 seconds (5 minutes) so a far-future Retry-After
+    header cannot stall the agent indefinitely.
+    """
+    if not retry_after or not isinstance(retry_after, str):
+        return None
+    retry_after = retry_after.strip()
+    if not retry_after:
+        return None
+
+    # Try integer seconds first.
+    try:
+        return min(float(retry_after), 300.0)
+    except (TypeError, ValueError):
+        pass
+
+    # Try HTTP-date (e.g. "Wed, 21 Oct 2015 07:28:00 GMT").
+    try:
+        parsed = email.utils.parsedate_to_datetime(retry_after)
+        retry_time = calendar.timegm(parsed.utctimetuple())
+        now = now if now is not None else time.time()
+        return max(0.0, min(retry_time - now, 300.0))
+    except Exception:
+        return None

agent/chat_completion_helpers.py

@@ -58,6 +58,9 @@ class TurnRetryState:
     primary_recovery_attempted: bool = False
     has_retried_429: bool = False
 
+    # ── Fail-fast guard for non-retryable client errors ─────────────────
+    fail_fast_attempted: bool = False
+
     # ── Restart signals (read by the outer loop after the attempt) ───────
     restart_with_compressed_messages: bool = False
     restart_with_length_continuation: bool = False