Skip to content

Commit 0b41ae9

Browse files
author
amabito
committed
fix(evaluators): budget R4 -- clamp negative pricing cost to zero
R4 finding: negative pricing rates in config caused _estimate_cost to return negative cost_usd, which subtracted from spent_usd and disabled USD limit enforcement entirely. Fix: max(0.0, cost) in _estimate_cost return. Test: negative pricing rates produce spent_usd >= 0. 58 budget tests, 288 total evaluator tests passing.
1 parent 4cd08eb commit 0b41ae9

2 files changed

Lines changed: 17 additions & 1 deletion

File tree

evaluators/builtin/src/agent_control_evaluators/budget/evaluator.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,8 @@ def _estimate_cost(
144144
return 0.0
145145
input_rate = rates.get("input_per_1k", 0.0)
146146
output_rate = rates.get("output_per_1k", 0.0)
147-
return (input_tokens * input_rate + output_tokens * output_rate) / 1000.0
147+
cost = (input_tokens * input_rate + output_tokens * output_rate) / 1000.0
148+
return max(0.0, cost) # never return negative cost
148149

149150

150151
def _extract_metadata(data: Any, metadata_paths: dict[str, str]) -> dict[str, str]:

evaluators/builtin/tests/budget/test_budget.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,3 +463,18 @@ def test_extract_cost_rejects_nan_inf(self) -> None:
463463
assert _extract_cost({"c": float("nan")}, "c") is None
464464
assert _extract_cost({"c": float("inf")}, "c") is None
465465
assert _extract_cost({"c": float("-inf")}, "c") is None
466+
467+
@pytest.mark.asyncio
468+
async def test_negative_pricing_does_not_reduce_budget(self) -> None:
469+
"""Negative pricing rates must not produce negative cost (budget credit)."""
470+
from agent_control_evaluators.budget.evaluator import BudgetEvaluator
471+
config = BudgetEvaluatorConfig(
472+
limits=[{"limit_usd": 0.01}],
473+
pricing={"model": {"input_per_1k": -5.0, "output_per_1k": -5.0}},
474+
model_path="model",
475+
)
476+
ev = BudgetEvaluator(config)
477+
for _ in range(10):
478+
await ev.evaluate({"model": "model", "usage": {"input_tokens": 1000, "output_tokens": 1000}})
479+
snap = ev._store.get_snapshot("__global__", "", limit_usd=0.01)
480+
assert snap.spent_usd >= 0.0 # must not go negative

0 commit comments

Comments
 (0)