fix(benchmarks): classify Gemini spend cap as terminal

Davincc77 · web-flow · commit 82c1c26c8caa · 2026-05-29T19:30:58.000+02:00
Merge PR #92 after CI success. Treats Gemini monthly spend-cap exhaustion as terminal_billing, prevents wasteful retries, and aborts bundle runs cleanly with manifest/summary metadata. No release/site/npm/PyPI/Zenodo paths altered.
diff --git a/benchmarks/v4.1/providers/__init__.py b/benchmarks/v4.1/providers/__init__.py
@@ -11,8 +11,10 @@
     ProviderConfig,
     ProviderError,
     ProviderResponse,
+    TerminalProviderError,
     TransientProviderError,
     get_provider,
+    is_terminal_billing_error,
     is_transient_error,
     register_provider,
 )
@@ -23,9 +25,11 @@
     "ProviderConfig",
     "ProviderError",
     "ProviderResponse",
+    "TerminalProviderError",
     "TransientProviderError",
     "MockProvider",
     "get_provider",
+    "is_terminal_billing_error",
     "is_transient_error",
     "register_provider",
 ]
diff --git a/benchmarks/v4.1/providers/base.py b/benchmarks/v4.1/providers/base.py
@@ -31,6 +31,44 @@ class TransientProviderError(ProviderError):
     """
 
 
+class TerminalProviderError(ProviderError):
+    """Subclass marking a TERMINAL, run-aborting provider failure.
+
+    Distinct from a plain permanent error: the executor should not only
+    refuse to retry the offending call, it should also stop processing
+    further calls in the current run because every subsequent call to
+    the same provider is guaranteed to fail for the same reason.
+
+    The canonical example is a Gemini ``429 RESOURCE_EXHAUSTED`` whose
+    message says ``Your project has exceeded its monthly spending cap``:
+    until the cap is raised or reset, every other request from the same
+    project will hit the identical billing hard cap, so retrying 6x per
+    item and continuing through 1799 more items just burns wall-clock
+    on a guaranteed-failing run (the symptom observed in workflow run
+    26642239431: 230 errors, 0 raw outputs, 120 min wasted).
+    """
+
+
+# Substrings that identify a *terminal* billing / hard-cap condition.
+# These are matched BEFORE the transient token list so that messages
+# such as ``429 RESOURCE_EXHAUSTED ... monthly spending cap ...`` are
+# not misclassified as a transient rate limit. Match is case-insensitive
+# and substring-based.
+_TERMINAL_BILLING_TOKENS = (
+    "monthly spending cap",
+    "monthly spend cap",
+    "spending cap",
+    "spend cap",
+    "spending limit reached",
+    "billing hard cap",
+    "billing cap exceeded",
+    "exceeded its monthly",
+    "exceeded the monthly",
+    "ai.studio/spend",
+    "ai studio at https://ai.studio/spend",
+)
+
+
 # HTTP status codes and provider error tokens that should be treated as
 # transient by the classifier below.
 _TRANSIENT_HTTP_CODES = (408, 425, 429, 500, 502, 503, 504)
@@ -62,6 +100,25 @@ class TransientProviderError(ProviderError):
 )
 
 
+def is_terminal_billing_error(exc: BaseException) -> bool:
+    """Return True iff ``exc`` looks like a terminal billing hard cap.
+
+    Specifically, a provider error whose human-readable message contains
+    one of :data:`_TERMINAL_BILLING_TOKENS` (e.g. Gemini's
+    ``monthly spending cap``). These conditions are *not* transient: the
+    provider returns the same response for every subsequent call from
+    the affected project until a human raises or resets the cap, so
+    retrying is wasted budget and continuing the run is wasted wall-clock.
+    """
+    if isinstance(exc, TerminalProviderError):
+        return True
+    text = str(exc).lower()
+    for tok in _TERMINAL_BILLING_TOKENS:
+        if tok in text:
+            return True
+    return False
+
+
 def is_transient_error(exc: BaseException) -> bool:
     """Classify an exception as transient (retryable) or not.
 
@@ -72,7 +129,15 @@ def is_transient_error(exc: BaseException) -> bool:
     This is conservative on purpose: anything that does not match a
     known transient signal is treated as non-transient so we do not
     retry auth/permission/quota-permanent errors.
+
+    Terminal billing hard-caps win over the transient signal: a message
+    such as ``429 RESOURCE_EXHAUSTED ... monthly spending cap ...`` is
+    classified non-transient even though it carries both a 429 code and
+    ``RESOURCE_EXHAUSTED``, because the underlying cause is a permanent
+    cap, not a per-second rate limit.
     """
+    if is_terminal_billing_error(exc):
+        return False
     for attr in ("status_code", "http_status", "status"):
         v = getattr(exc, attr, None)
         if isinstance(v, int) and v in _TRANSIENT_HTTP_CODES:
diff --git a/benchmarks/v4.1/providers/gemini_adapter.py b/benchmarks/v4.1/providers/gemini_adapter.py
@@ -19,8 +19,10 @@
     ProviderConfig,
     ProviderError,
     ProviderResponse,
+    TerminalProviderError,
     TransientProviderError,
     env_api_key,
+    is_terminal_billing_error,
     is_transient_error,
 )
 
@@ -79,6 +81,16 @@ def generate(self, system: str, user: str, config: ProviderConfig) -> ProviderRe
                 },
             )
         except Exception as exc:
+            # Terminal billing hard cap (e.g. "monthly spending cap")
+            # comes through Gemini as a 429 RESOURCE_EXHAUSTED, which
+            # would otherwise be misclassified as transient and retried.
+            # Detect it first and surface as TerminalProviderError so the
+            # executor stops the run cleanly instead of burning the whole
+            # 120-min job re-trying every item.
+            if is_terminal_billing_error(exc):
+                raise TerminalProviderError(
+                    f"gemini terminal billing cap: {exc!s}"
+                ) from exc
             # Surface timeouts as transient so the retry loop can recover.
             if _is_timeout_error(exc) or is_transient_error(exc):
                 raise TransientProviderError(
diff --git a/benchmarks/v4.1/runner/executor.py b/benchmarks/v4.1/runner/executor.py
@@ -37,7 +37,9 @@
     ProviderConfig,
     ProviderError,
     ProviderResponse,
+    TerminalProviderError,
     TransientProviderError,
+    is_terminal_billing_error,
     is_transient_error,
 )
 
@@ -116,7 +118,20 @@ def _build_call(
 
 
 def _classify(exc: BaseException) -> str:
-    """Stable short string used in retry logs."""
+    """Stable short string used in retry logs.
+
+    Classes:
+    - ``transient``: retryable (429 rate limit cooldown, 5xx, timeout)
+    - ``terminal_billing``: provider-side billing/spend hard cap. NOT
+      retryable; the entire run should stop because every subsequent
+      call will hit the identical cap.
+    - ``permanent``: auth/config/schema errors. NOT retryable for this
+      call but the rest of the run is unaffected.
+    - ``unhandled``: anything we did not recognise — treated as permanent
+      for safety so we never silently retry an unknown failure mode.
+    """
+    if is_terminal_billing_error(exc):
+        return "terminal_billing"
     if isinstance(exc, TransientProviderError):
         return "transient"
     if isinstance(exc, ProviderError):
diff --git a/benchmarks/v4.1/runner/executor_b_bundles.py b/benchmarks/v4.1/runner/executor_b_bundles.py
@@ -26,6 +26,7 @@
     _utcnow_iso,
     ExecutionPlan,
 )
+from ..providers.base import is_terminal_billing_error
 from ..prompts.test_b_bundles import (
     TEST_B_BUNDLE_CONDITIONS,
     build_test_b_bundle_messages,
@@ -118,14 +119,32 @@ def _log(msg: str) -> None:
         sys.stderr.write(msg + "\n")
         sys.stderr.flush()
 
-    counts = {"ok": 0, "error": 0, "skipped_resumed": len(already_done)}
+    counts = {
+        "ok": 0,
+        "error": 0,
+        "skipped_resumed": len(already_done),
+        "skipped_terminal_abort": 0,
+    }
     latencies: list[int] = []
     by_condition: dict[str, int] = {}
     by_phase: dict[str, int] = {}
     by_bundle: dict[str, int] = {}
     tokens = {"input": 0, "output": 0}
 
+    # Set once a terminal billing / spend-cap condition is observed. Every
+    # subsequent call to the same provider is guaranteed to fail with the
+    # same error, so we stop dispatching new work and record the rest as
+    # ``skipped_terminal_abort`` instead of burning the full 120-min job.
+    abort_reason: dict[str, Any] = {}
+    abort_lock = threading.Lock()
+
     def _do_call(call: dict[str, Any]) -> None:
+        if abort_reason:
+            # A previous call already tripped the terminal-billing trap.
+            # Skip without contacting the provider; the batch loop will
+            # break out cleanly after the current batch drains.
+            counts["skipped_terminal_abort"] += 1
+            return
         t_start = _utcnow_iso()
         with progress_lock:
             progress["started"] += 1
@@ -187,6 +206,22 @@ def _do_call(call: dict[str, Any]) -> None:
                 f"attempts={retried_attempts} "
                 f"wall_ms={dt_ms}"
             )
+            if final_class == "terminal_billing":
+                with abort_lock:
+                    if not abort_reason:
+                        last_err = attempts[-1] if attempts else {}
+                        abort_reason.update({
+                            "first_run_id": call["run_id"],
+                            "error_type": last_err.get("error_type"),
+                            "error": last_err.get("error"),
+                        })
+                        _log(
+                            "[bb] TERMINAL-BILLING tripped: provider has "
+                            "hit a hard billing/spend cap. Aborting the "
+                            "run; remaining calls will be recorded as "
+                            f"skipped_terminal_abort. run_id={call['run_id']} "
+                            f"err={last_err.get('error')}"
+                        )
             return
         raw_writer.write({
             "run_id": call["run_id"],
@@ -286,6 +321,17 @@ def _record_wall_timeout(c: dict[str, Any]) -> None:
 
     try:
         for batch_start in range(0, len(pending), plan.batch_size):
+            if abort_reason:
+                # A prior batch tripped the terminal-billing trap. Stop
+                # dispatching new work entirely; the remaining ``pending``
+                # are intentionally not contacted.
+                _log(
+                    f"[bb] abort: skipping remaining "
+                    f"{len(pending) - batch_start} call(s) after "
+                    f"terminal_billing first observed in "
+                    f"run_id={abort_reason.get('first_run_id')}"
+                )
+                break
             batch = pending[batch_start:batch_start + plan.batch_size]
             if plan.concurrency <= 1:
                 for call in batch:
@@ -357,6 +403,8 @@ def _worker() -> None:
         "test": "test_b_bundles",
         "counts": counts,
         "total_attempted": len(pending),
+        "aborted": bool(abort_reason),
+        "abort_reason": dict(abort_reason) if abort_reason else None,
         "by_condition": by_condition,
         "by_phase": by_phase,
         "by_bundle": by_bundle,
@@ -402,6 +450,8 @@ def _worker() -> None:
         "fixtures": fixtures_manifest,
         "repo_commit": repo_commit,
         "counts": counts,
+        "aborted": bool(abort_reason),
+        "abort_reason": dict(abort_reason) if abort_reason else None,
         "n_run_specs_total": len(pending) + counts["skipped_resumed"],
         "n_already_done_on_start": counts["skipped_resumed"],
         "conditions": list(conditions),
diff --git a/benchmarks/v4.1/tests/test_terminal_billing.py b/benchmarks/v4.1/tests/test_terminal_billing.py
