Skip to content

Commit c3210f5

Browse files
committed
fix(eval_optimize_loop): improve test coverage to 100% for reporting, remove unused import
- Remove unused GateDecision import from test_reporting.py - Add test_write_reports_per_case_delta_all_transitions covering all 4 transition types in per-case delta table: newly_passing, newly_failing, passed (both), failed (both) - Coverage: reporting.py 100% (119/119 lines)
1 parent 984291f commit c3210f5

1 file changed

Lines changed: 73 additions & 1 deletion

File tree

examples/optimization/eval_optimize_loop/tests/test_reporting.py

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
PerCaseResult,
1414
FailureAttribution,
1515
FailureCategory,
16-
GateDecision,
1716
)
1817

1918

@@ -123,3 +122,76 @@ def test_write_reports_reject():
123122
assert "REJECT" in md_content
124123
assert "overfitting" in md_content.lower()
125124
assert "newly failing" in md_content.lower()
125+
126+
127+
def test_write_reports_per_case_delta_all_transitions():
128+
result = PipelineResult(
129+
mode="trace",
130+
gate_decision="MIXED",
131+
gate_reasons=["all 4 transition types covered"],
132+
baseline={
133+
"train": SplitResult(
134+
pass_rate=0.5,
135+
metric_breakdown={"m1": 0.5},
136+
per_case={
137+
"c_newp": PerCaseResult(case_id="c_newp", passed=False, metric_scores={"m1": 0.0}),
138+
"c_newf": PerCaseResult(case_id="c_newf", passed=True, metric_scores={"m1": 1.0}),
139+
"c_bothp": PerCaseResult(case_id="c_bothp", passed=True, metric_scores={"m1": 1.0}),
140+
"c_bothf": PerCaseResult(case_id="c_bothf", passed=False, metric_scores={"m1": 0.0}),
141+
},
142+
),
143+
},
144+
candidate={
145+
"train": SplitResult(
146+
pass_rate=0.5,
147+
metric_breakdown={"m1": 0.5},
148+
per_case={
149+
"c_newp": PerCaseResult(case_id="c_newp", passed=True, metric_scores={"m1": 1.0}),
150+
"c_newf": PerCaseResult(case_id="c_newf", passed=False, metric_scores={"m1": 0.0}),
151+
"c_bothp": PerCaseResult(case_id="c_bothp", passed=True, metric_scores={"m1": 1.0}),
152+
"c_bothf": PerCaseResult(case_id="c_bothf", passed=False, metric_scores={"m1": 0.0}),
153+
},
154+
),
155+
},
156+
delta=SplitDelta(
157+
train=PerCaseDelta(
158+
newly_passing=["c_newp"],
159+
newly_failing=["c_newf"],
160+
score_deltas={"c_newp": {"m1": 1.0}, "c_newf": {"m1": -1.0}},
161+
unchanged=["c_bothp", "c_bothf"],
162+
),
163+
val=PerCaseDelta(
164+
newly_passing=[],
165+
newly_failing=[],
166+
score_deltas={},
167+
unchanged=[],
168+
),
169+
train_pass_rate_delta=0.0,
170+
val_pass_rate_delta=0.0,
171+
),
172+
failure_attribution=FailureAttribution(
173+
total_cases=4,
174+
failed_cases=2,
175+
categories={},
176+
),
177+
duration_seconds=0.5,
178+
seed=42,
179+
started_at="2026-01-01T00:00:00",
180+
finished_at="2026-01-01T00:00:01",
181+
)
182+
183+
with tempfile.TemporaryDirectory() as tmpdir:
184+
write_reports(result, tmpdir)
185+
186+
md_path = os.path.join(tmpdir, "optimization_report.md")
187+
assert os.path.isfile(md_path)
188+
189+
with open(md_path) as f:
190+
md_content = f.read()
191+
192+
assert "## Per-Case Delta" in md_content
193+
assert "newly passing" in md_content
194+
assert "newly failing" in md_content
195+
assert "passed (both)" in md_content
196+
assert "failed (both)" in md_content
197+
assert "MIXED" in md_content

0 commit comments

Comments
 (0)