Skip to content

Commit 7250d91

Browse files
committed
fix(examples): honor disabled sdk cost gate
1 parent dae8692 commit 7250d91

2 files changed

Lines changed: 27 additions & 5 deletions

File tree

examples/optimization/eval_optimize_loop/run_pipeline.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -594,11 +594,12 @@ def _sdk_gate_decision(
594594
f"accept: validation improvement {improvement:.3f} meets threshold {min_improvement:.3f}"
595595
)
596596

597-
if total_cost > max_cost:
598-
accepted = False
599-
reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}")
600-
else:
601-
reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}")
597+
if max_cost is not None:
598+
if total_cost > max_cost:
599+
accepted = False
600+
reasons.append(f"reject: total SDK cost {total_cost:.3f} exceeds budget {max_cost:.3f}")
601+
else:
602+
reasons.append(f"accept: total SDK cost {total_cost:.3f} is within budget {max_cost:.3f}")
602603

603604
if accepted:
604605
reasons.append("accept: SDK aggregate gate passed")

examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN
1515
from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL
1616
from examples.optimization.eval_optimize_loop.run_pipeline import _parse_target_prompt_paths
17+
from examples.optimization.eval_optimize_loop.run_pipeline import _sdk_gate_decision
1718
from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline
1819

1920

@@ -551,6 +552,26 @@ def test_run_pipeline_mode_sdk_custom_gate_rejects_cost_over_budget(tmp_path: Pa
551552
assert any("cost" in reason for reason in decision.reasons)
552553

553554

555+
def test_sdk_gate_decision_skips_disabled_cost_gate_but_keeps_quality_threshold():
556+
decision = _sdk_gate_decision(
557+
candidate_id="sdk_best",
558+
sdk_summary={
559+
"status": "SUCCEEDED",
560+
"pass_rate_improvement": 0.005,
561+
"total_llm_cost": 1000.0,
562+
},
563+
gate_config={
564+
"min_val_score_improvement": 0.01,
565+
"max_total_cost": None,
566+
},
567+
)
568+
569+
assert decision.accepted is False
570+
assert any("validation improvement" in reason for reason in decision.reasons)
571+
assert not any("cost" in reason for reason in decision.reasons)
572+
assert decision.total_run_cost == 1000.0
573+
574+
554575
@pytest.mark.parametrize(
555576
("field_name", "field_value"),
556577
[

0 commit comments

Comments
 (0)