fix: don't only evaluate final input in GT results

Author: andrewklatzke

packages/optimization/src/ldai_optimizer/client.py

@@ -1155,6 +1155,12 @@ async def _run_ground_truth_optimization(
                 sample_passed, optimize_context = self._apply_duration_gate(sample_passed, optimize_context)
                 sample_passed, optimize_context = self._apply_cost_gate(sample_passed, optimize_context)
 
+                # Flush gate scores to the API for this sample. Without this,
+                # the next sample's "generating" event closes out this record
+                # with a status-only PATCH before gate scores are sent, so only
+                # the last sample would ever show latency/cost gate entries.
+                self._safe_status_update("evaluating", optimize_context, linear_iter)
+
                 if not sample_passed:
                     logger.info(
                         "[GT Attempt %d] -> Sample %d/%d FAILED",
@@ -2110,21 +2116,23 @@ def _apply_duration_gate(
         When the gate is active (any acceptance statement implies latency optimization),
         evaluates whether the candidate's duration improved by at least
         _DURATION_TOLERANCE vs the baseline. A synthetic ``_latency_gate`` entry is
-        added to scores with score=1.0 on pass or score=0.0 on fail so the outcome
-        is visible in the API result and UI.
+        always added to scores with score=1.0 on pass or score=0.0 on fail so the
+        outcome is visible in the API result and UI for every iteration.
+
+        The gate score is recorded even when ``passed_so_far`` is False (quality
+        judges already failed) so that latency telemetry is visible on all
+        iterations, not just passing ones. In that case it is informational only
+        and cannot block the iteration further.
 
-        The gate is skipped (no score entry added) when:
-        - No acceptance statement implies latency optimization.
-        - ``passed_so_far`` is already False (a prior check failed the sample).
+        The gate is skipped entirely (no score entry added) only when no acceptance
+        statement implies latency optimization.
 
         :param passed_so_far: Whether all prior checks for this sample passed.
         :param ctx: Current optimization context.
         :return: (passed, updated_ctx) where passed reflects gate outcome.
         """
         if not _acceptance_criteria_implies_duration_optimization(self._options.judges):
             return passed_so_far, ctx
-        if not passed_so_far:
-            return passed_so_far, ctx
         passed = self._evaluate_duration(ctx)
         if passed:
             if self._baseline_duration_ms is not None and ctx.duration_ms is not None:
@@ -2149,9 +2157,13 @@ def _apply_duration_gate(
             score = 0.0
         ctx = dataclasses.replace(
             ctx,
-            scores={**ctx.scores, "_latency_gate": JudgeResult(score=score, rationale=rationale)},
+            scores={**ctx.scores, "_latency_gate": JudgeResult(
+                score=score,
+                rationale=rationale,
+                duration_ms=ctx.duration_ms,
+            )},
         )
-        return passed, ctx
+        return passed_so_far and passed, ctx
 
     def _apply_cost_gate(
         self, passed_so_far: bool, ctx: OptimizationContext
@@ -2160,21 +2172,23 @@ def _apply_cost_gate(
 
         When the gate is active (any acceptance statement implies cost optimization),
         evaluates whether the candidate's estimated cost improved by at least
-        _COST_TOLERANCE vs the baseline. A synthetic ``_cost_gate`` entry is
+        _COST_TOLERANCE vs the baseline. A synthetic ``_cost_gate`` entry is always
         added to scores with score=1.0 on pass or score=0.0 on fail.
 
-        The gate is skipped (no score entry added) when:
-        - No acceptance statement implies cost optimization.
-        - ``passed_so_far`` is already False (a prior check failed the sample).
+        The gate score is recorded even when ``passed_so_far`` is False (quality
+        judges already failed) so that cost telemetry is visible on all iterations,
+        not just passing ones. In that case it is informational only and cannot
+        block the iteration further.
+
+        The gate is skipped entirely (no score entry added) only when no acceptance
+        statement implies cost optimization.
 
         :param passed_so_far: Whether all prior checks for this sample passed.
         :param ctx: Current optimization context.
         :return: (passed, updated_ctx) where passed reflects gate outcome.
         """
         if not _acceptance_criteria_implies_cost_optimization(self._options.judges):
             return passed_so_far, ctx
-        if not passed_so_far:
-            return passed_so_far, ctx
         passed = self._evaluate_cost(ctx)
         if passed:
             if self._baseline_cost_usd is not None and ctx.estimated_cost_usd is not None:
@@ -2199,9 +2213,14 @@ def _apply_cost_gate(
             score = 0.0
         ctx = dataclasses.replace(
             ctx,
-            scores={**ctx.scores, "_cost_gate": JudgeResult(score=score, rationale=rationale)},
+            scores={**ctx.scores, "_cost_gate": JudgeResult(
+                score=score,
+                rationale=rationale,
+                duration_ms=ctx.duration_ms,
+                estimated_cost_usd=ctx.estimated_cost_usd,
+            )},
         )
-        return passed, ctx
+        return passed_so_far and passed, ctx
 
     def _handle_success(
         self, optimize_context: OptimizationContext, iteration: int

packages/optimization/src/ldai_optimizer/dataclasses.py

@@ -43,6 +43,7 @@ class JudgeResult:
     rationale: Optional[str] = None
     duration_ms: Optional[float] = None
     usage: Optional[TokenUsage] = None
+    estimated_cost_usd: Optional[float] = None
 
     def to_json(self) -> Dict[str, Any]:
         """
@@ -61,6 +62,8 @@ def to_json(self) -> Dict[str, Any]:
                 "input": self.usage.input,
                 "output": self.usage.output,
             }
+        if self.estimated_cost_usd is not None:
+            result["estimated_cost_usd"] = self.estimated_cost_usd
         return result
 
 

packages/optimization/tests/test_client.py

@@ -5224,11 +5224,13 @@ def test_no_entry_added_when_gate_not_active(self):
         assert passed is True
         assert "_latency_gate" not in updated.scores
 
-    def test_no_entry_added_when_already_failed(self):
+    def test_gate_recorded_even_when_already_failed(self):
+        # Gate score is always written for telemetry; it cannot block an
+        # iteration that was already failing (passed_so_far=False).
         ctx = self._ctx(1000)
         passed, updated = self.client._apply_duration_gate(False, ctx)
         assert passed is False
-        assert "_latency_gate" not in updated.scores
+        assert "_latency_gate" in updated.scores
 
     def test_gate_pass_adds_score_1(self):
         # baseline=2000ms, threshold=1600ms, candidate=1500ms → pass
@@ -5327,11 +5329,13 @@ def test_no_entry_added_when_gate_not_active(self):
         assert passed is True
         assert "_cost_gate" not in updated.scores
 
-    def test_no_entry_added_when_already_failed(self):
+    def test_gate_recorded_even_when_already_failed(self):
+        # Gate score is always written for telemetry; it cannot block an
+        # iteration that was already failing (passed_so_far=False).
         ctx = self._ctx(0.005)
         passed, updated = self.client._apply_cost_gate(False, ctx)
         assert passed is False
-        assert "_cost_gate" not in updated.scores
+        assert "_cost_gate" in updated.scores
 
     def test_gate_pass_adds_score_1(self):
         # baseline=0.010, threshold=0.009, candidate=0.007 → pass