Skip to content

Commit 5a100f8

Browse files
author
amabito
committed
fix(evaluators): budget R5 -- isfinite guard on estimated cost
R5 finding: Inf pricing rates produced inf cost, permanently locking buckets in exceeded state. max(0.0, inf) = inf. Fix: isfinite + negative check on _estimate_cost return value. Tests: Inf pricing rate test, strengthened negative pricing assertion. 59 budget tests passing.
1 parent 0b41ae9 commit 5a100f8

2 files changed

Lines changed: 20 additions & 2 deletions

File tree

evaluators/builtin/src/agent_control_evaluators/budget/evaluator.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,9 @@ def _estimate_cost(
145145
input_rate = rates.get("input_per_1k", 0.0)
146146
output_rate = rates.get("output_per_1k", 0.0)
147147
cost = (input_tokens * input_rate + output_tokens * output_rate) / 1000.0
148-
return max(0.0, cost) # never return negative cost
148+
if not math.isfinite(cost) or cost < 0:
149+
return 0.0
150+
return cost
149151

150152

151153
def _extract_metadata(data: Any, metadata_paths: dict[str, str]) -> dict[str, str]:

evaluators/builtin/tests/budget/test_budget.py

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -477,4 +477,20 @@ async def test_negative_pricing_does_not_reduce_budget(self) -> None:
477477
for _ in range(10):
478478
await ev.evaluate({"model": "model", "usage": {"input_tokens": 1000, "output_tokens": 1000}})
479479
snap = ev._store.get_snapshot("__global__", "", limit_usd=0.01)
480-
assert snap.spent_usd >= 0.0 # must not go negative
480+
assert snap.spent_usd == pytest.approx(0.0) # negative rates clamped to 0
481+
482+
@pytest.mark.asyncio
483+
async def test_inf_pricing_does_not_cause_inf_cost(self) -> None:
484+
"""Inf pricing rates must not produce inf cost (permanent false positive)."""
485+
import math as _math
486+
from agent_control_evaluators.budget.evaluator import BudgetEvaluator
487+
config = BudgetEvaluatorConfig(
488+
limits=[{"limit_usd": 100.0}],
489+
pricing={"model": {"input_per_1k": float("inf"), "output_per_1k": 0.01}},
490+
model_path="model",
491+
)
492+
ev = BudgetEvaluator(config)
493+
await ev.evaluate({"model": "model", "usage": {"input_tokens": 100, "output_tokens": 100}})
494+
snap = ev._store.get_snapshot("__global__", "", limit_usd=100.0)
495+
assert _math.isfinite(snap.spent_usd)
496+
assert snap.spent_usd == pytest.approx(0.0) # inf rate -> cost clamped to 0

0 commit comments

Comments
 (0)