|
13 | 13 | PerCaseResult, |
14 | 14 | FailureAttribution, |
15 | 15 | FailureCategory, |
16 | | - GateDecision, |
17 | 16 | ) |
18 | 17 |
|
19 | 18 |
|
@@ -123,3 +122,76 @@ def test_write_reports_reject(): |
123 | 122 | assert "REJECT" in md_content |
124 | 123 | assert "overfitting" in md_content.lower() |
125 | 124 | assert "newly failing" in md_content.lower() |
| 125 | + |
| 126 | + |
| 127 | +def test_write_reports_per_case_delta_all_transitions(): |
| 128 | + result = PipelineResult( |
| 129 | + mode="trace", |
| 130 | + gate_decision="MIXED", |
| 131 | + gate_reasons=["all 4 transition types covered"], |
| 132 | + baseline={ |
| 133 | + "train": SplitResult( |
| 134 | + pass_rate=0.5, |
| 135 | + metric_breakdown={"m1": 0.5}, |
| 136 | + per_case={ |
| 137 | + "c_newp": PerCaseResult(case_id="c_newp", passed=False, metric_scores={"m1": 0.0}), |
| 138 | + "c_newf": PerCaseResult(case_id="c_newf", passed=True, metric_scores={"m1": 1.0}), |
| 139 | + "c_bothp": PerCaseResult(case_id="c_bothp", passed=True, metric_scores={"m1": 1.0}), |
| 140 | + "c_bothf": PerCaseResult(case_id="c_bothf", passed=False, metric_scores={"m1": 0.0}), |
| 141 | + }, |
| 142 | + ), |
| 143 | + }, |
| 144 | + candidate={ |
| 145 | + "train": SplitResult( |
| 146 | + pass_rate=0.5, |
| 147 | + metric_breakdown={"m1": 0.5}, |
| 148 | + per_case={ |
| 149 | + "c_newp": PerCaseResult(case_id="c_newp", passed=True, metric_scores={"m1": 1.0}), |
| 150 | + "c_newf": PerCaseResult(case_id="c_newf", passed=False, metric_scores={"m1": 0.0}), |
| 151 | + "c_bothp": PerCaseResult(case_id="c_bothp", passed=True, metric_scores={"m1": 1.0}), |
| 152 | + "c_bothf": PerCaseResult(case_id="c_bothf", passed=False, metric_scores={"m1": 0.0}), |
| 153 | + }, |
| 154 | + ), |
| 155 | + }, |
| 156 | + delta=SplitDelta( |
| 157 | + train=PerCaseDelta( |
| 158 | + newly_passing=["c_newp"], |
| 159 | + newly_failing=["c_newf"], |
| 160 | + score_deltas={"c_newp": {"m1": 1.0}, "c_newf": {"m1": -1.0}}, |
| 161 | + unchanged=["c_bothp", "c_bothf"], |
| 162 | + ), |
| 163 | + val=PerCaseDelta( |
| 164 | + newly_passing=[], |
| 165 | + newly_failing=[], |
| 166 | + score_deltas={}, |
| 167 | + unchanged=[], |
| 168 | + ), |
| 169 | + train_pass_rate_delta=0.0, |
| 170 | + val_pass_rate_delta=0.0, |
| 171 | + ), |
| 172 | + failure_attribution=FailureAttribution( |
| 173 | + total_cases=4, |
| 174 | + failed_cases=2, |
| 175 | + categories={}, |
| 176 | + ), |
| 177 | + duration_seconds=0.5, |
| 178 | + seed=42, |
| 179 | + started_at="2026-01-01T00:00:00", |
| 180 | + finished_at="2026-01-01T00:00:01", |
| 181 | + ) |
| 182 | + |
| 183 | + with tempfile.TemporaryDirectory() as tmpdir: |
| 184 | + write_reports(result, tmpdir) |
| 185 | + |
| 186 | + md_path = os.path.join(tmpdir, "optimization_report.md") |
| 187 | + assert os.path.isfile(md_path) |
| 188 | + |
| 189 | + with open(md_path) as f: |
| 190 | + md_content = f.read() |
| 191 | + |
| 192 | + assert "## Per-Case Delta" in md_content |
| 193 | + assert "newly passing" in md_content |
| 194 | + assert "newly failing" in md_content |
| 195 | + assert "passed (both)" in md_content |
| 196 | + assert "failed (both)" in md_content |
| 197 | + assert "MIXED" in md_content |
0 commit comments