Skip to content

Commit fdbd1e4

Browse files
DNKYrDNKYr
committed
feat(eval_optimize_loop): finalize pipeline with path resolution, example output, gitignore
1 parent 21d2801 commit fdbd1e4

5 files changed

Lines changed: 305 additions & 1 deletion

File tree

‎examples/optimization/eval_optimize_loop/example_output/optimization_report.json‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
{
2+
"schemaVersion": "v1",
3+
"mode": "trace",
4+
"gateDecision": "REJECT",
5+
"gateReasons": [
6+
"newly failing in val not allowed: ['case_val_regression']",
7+
"min_improvement met: val pass_rate delta 0.0000 >= 0.0000",
8+
"duration check passed: 0.01s <= 180s"
9+
],
10+
"baseline": {
11+
"train": {
12+
"passRate": 0.3333333333333333,
13+
"metricBreakdown": {
14+
"final_response_avg_score": 0.3333333333333333
15+
},
16+
"perCase": {
17+
"case_train_regression": {
18+
"caseId": "case_train_regression",
19+
"passed": true,
20+
"metricScores": {
21+
"final_response_avg_score": 1.0
22+
}
23+
},
24+
"case_train_optimizable": {
25+
"caseId": "case_train_optimizable",
26+
"passed": false,
27+
"metricScores": {
28+
"final_response_avg_score": 0.0
29+
}
30+
},
31+
"case_train_always_fail": {
32+
"caseId": "case_train_always_fail",
33+
"passed": false,
34+
"metricScores": {
35+
"final_response_avg_score": 0.0
36+
}
37+
}
38+
}
39+
},
40+
"val": {
41+
"passRate": 0.3333333333333333,
42+
"metricBreakdown": {
43+
"final_response_avg_score": 0.3333333333333333
44+
},
45+
"perCase": {
46+
"case_val_no_change": {
47+
"caseId": "case_val_no_change",
48+
"passed": false,
49+
"metricScores": {
50+
"final_response_avg_score": 0.0
51+
}
52+
},
53+
"case_val_regression": {
54+
"caseId": "case_val_regression",
55+
"passed": true,
56+
"metricScores": {
57+
"final_response_avg_score": 1.0
58+
}
59+
},
60+
"case_val_improves": {
61+
"caseId": "case_val_improves",
62+
"passed": false,
63+
"metricScores": {
64+
"final_response_avg_score": 0.0
65+
}
66+
}
67+
}
68+
}
69+
},
70+
"candidate": {
71+
"train": {
72+
"passRate": 0.3333333333333333,
73+
"metricBreakdown": {
74+
"final_response_avg_score": 0.3333333333333333
75+
},
76+
"perCase": {
77+
"case_train_always_fail": {
78+
"caseId": "case_train_always_fail",
79+
"passed": false,
80+
"metricScores": {
81+
"final_response_avg_score": 0.0
82+
}
83+
},
84+
"case_train_regression": {
85+
"caseId": "case_train_regression",
86+
"passed": false,
87+
"metricScores": {
88+
"final_response_avg_score": 0.0
89+
}
90+
},
91+
"case_train_optimizable": {
92+
"caseId": "case_train_optimizable",
93+
"passed": true,
94+
"metricScores": {
95+
"final_response_avg_score": 1.0
96+
}
97+
}
98+
}
99+
},
100+
"val": {
101+
"passRate": 0.3333333333333333,
102+
"metricBreakdown": {
103+
"final_response_avg_score": 0.3333333333333333
104+
},
105+
"perCase": {
106+
"case_val_no_change": {
107+
"caseId": "case_val_no_change",
108+
"passed": false,
109+
"metricScores": {
110+
"final_response_avg_score": 0.0
111+
}
112+
},
113+
"case_val_regression": {
114+
"caseId": "case_val_regression",
115+
"passed": false,
116+
"metricScores": {
117+
"final_response_avg_score": 0.0
118+
}
119+
},
120+
"case_val_improves": {
121+
"caseId": "case_val_improves",
122+
"passed": true,
123+
"metricScores": {
124+
"final_response_avg_score": 1.0
125+
}
126+
}
127+
}
128+
}
129+
},
130+
"delta": {
131+
"train": {
132+
"newlyPassing": [
133+
"case_train_optimizable"
134+
],
135+
"newlyFailing": [
136+
"case_train_regression"
137+
],
138+
"scoreDeltas": {
139+
"case_train_regression": {
140+
"final_response_avg_score": -1.0
141+
},
142+
"case_train_always_fail": {
143+
"final_response_avg_score": 0.0
144+
},
145+
"case_train_optimizable": {
146+
"final_response_avg_score": 1.0
147+
}
148+
},
149+
"unchanged": [
150+
"case_train_always_fail"
151+
]
152+
},
153+
"val": {
154+
"newlyPassing": [
155+
"case_val_improves"
156+
],
157+
"newlyFailing": [
158+
"case_val_regression"
159+
],
160+
"scoreDeltas": {
161+
"case_val_regression": {
162+
"final_response_avg_score": -1.0
163+
},
164+
"case_val_no_change": {
165+
"final_response_avg_score": 0.0
166+
},
167+
"case_val_improves": {
168+
"final_response_avg_score": 1.0
169+
}
170+
},
171+
"unchanged": [
172+
"case_val_no_change"
173+
]
174+
},
175+
"trainPassRateDelta": 0.0,
176+
"valPassRateDelta": 0.0
177+
},
178+
"failureAttribution": {
179+
"totalCases": 3,
180+
"failedCases": 2,
181+
"categories": {
182+
"final_response_mismatch": {
183+
"count": 2,
184+
"caseIds": [
185+
"case_train_optimizable",
186+
"case_train_always_fail"
187+
]
188+
},
189+
"format_violation": {
190+
"count": 1,
191+
"caseIds": [
192+
"case_train_always_fail"
193+
]
194+
}
195+
}
196+
},
197+
"overfittingWarning": false,
198+
"durationSeconds": 0.010774361000585486,
199+
"costUsd": 0.0,
200+
"seed": 42,
201+
"startedAt": "2026-07-16T03:39:56.304373+00:00",
202+
"finishedAt": "2026-07-16T03:39:56.315185+00:00"
203+
}

‎examples/optimization/eval_optimize_loop/example_output/optimization_report.md‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Optimization Report
2+
3+
## Verdict
4+
5+
**✗ REJECT**
6+
7+
- newly failing in val not allowed: ['case_val_regression']
8+
- min_improvement met: val pass_rate delta 0.0000 >= 0.0000
9+
- duration check passed: 0.01s <= 180s
10+
11+
## Pass Rates
12+
13+
| Split | Baseline | Candidate | Delta |
14+
|---|---|---|---|
15+
| train | 0.3333 | 0.3333 | +0.0000 |
16+
| val | 0.3333 | 0.3333 | +0.0000 |
17+
18+
### Metric Breakdown (val)
19+
20+
| Metric | Baseline | Candidate | Delta |
21+
|---|---|---|---|
22+
| final_response_avg_score | 0.3333 | 0.3333 | +0.0000 |
23+
24+
## Per-Case Delta
25+
26+
### train set
27+
28+
| Case ID | Baseline | Candidate | Status |
29+
|---|---|---|---|
30+
| case_train_always_fail | FAIL | FAIL | failed (both) |
31+
| case_train_optimizable | FAIL | PASS | newly passing |
32+
| case_train_regression | PASS | FAIL | newly failing |
33+
34+
### val set
35+
36+
| Case ID | Baseline | Candidate | Status |
37+
|---|---|---|---|
38+
| case_val_improves | FAIL | PASS | newly passing |
39+
| case_val_no_change | FAIL | FAIL | failed (both) |
40+
| case_val_regression | PASS | FAIL | newly failing |
41+
42+
## Failure Attribution
43+
44+
- Total cases: 3
45+
- Failed (baseline train): 2
46+
47+
| Category | Count | Case IDs |
48+
|---|---|---|
49+
| final_response_mismatch | 2 | case_train_optimizable, case_train_always_fail |
50+
| format_violation | 1 | case_train_always_fail |
51+
52+
## Gate Results
53+
54+
- newly failing in val not allowed: ['case_val_regression']
55+
- min_improvement met: val pass_rate delta 0.0000 >= 0.0000
56+
- duration check passed: 0.01s <= 180s
57+
58+
## Overfitting Check
59+
60+
No overfitting detected.
61+
62+
## Audit
63+
64+
| Field | Value |
65+
|---|---|
66+
| Mode | trace |
67+
| Duration | 0.01s |
68+
| Cost | $0.0000 |
69+
| Seed | 42 |
70+
| Started | 2026-07-16T03:39:56.304373+00:00 |
71+
| Finished | 2026-07-16T03:39:56.315185+00:00 |
72+
| Schema version | v1 |
73+

‎examples/optimization/eval_optimize_loop/outputs/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
optimization_report.*

‎examples/optimization/eval_optimize_loop/pipeline.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,12 @@ def from_config(
4444
call_agent: Optional[CallAgent] = None,
4545
target_prompt: Optional[TargetPrompt] = None,
4646
) -> "EvalOptimizePipeline":
47+
config_dir = os.path.dirname(os.path.abspath(config_path))
4748
with open(config_path, "r", encoding="utf-8") as f:
4849
raw = f.read()
4950

5051
config = PipelineConfig.model_validate_json(raw)
52+
cls._resolve_paths(config, config_dir)
5153

5254
if config.mode == "live" and (call_agent is None or target_prompt is None):
5355
raise ValueError(
@@ -60,6 +62,28 @@ def from_config(
6062
pipeline._live_target_prompt = target_prompt
6163
return pipeline
6264

65+
@staticmethod
66+
def _resolve_paths(config: PipelineConfig, base_dir: str) -> None:
67+
"""Resolve relative paths in config against the config file directory."""
68+
paths = [
69+
("baseline_prompt_path", config.baseline_prompt_path),
70+
("candidate_prompt_path", config.candidate_prompt_path),
71+
("train_baseline_evalset", config.train_baseline_evalset),
72+
("train_candidate_evalset", config.train_candidate_evalset),
73+
("val_baseline_evalset", config.val_baseline_evalset),
74+
("val_candidate_evalset", config.val_candidate_evalset),
75+
("live_train_evalset", config.live_train_evalset),
76+
("live_val_evalset", config.live_val_evalset),
77+
("optimizer_config_path", config.optimizer_config_path),
78+
]
79+
import tempfile
80+
for name, value in paths:
81+
if value is not None and not os.path.isabs(value) and not value.startswith("/tmp"):
82+
setattr(config, name, os.path.normpath(os.path.join(base_dir, value)))
83+
84+
if not os.path.isabs(config.output_dir):
85+
config.output_dir = os.path.normpath(os.path.join(base_dir, config.output_dir))
86+
6387
async def run(self) -> PipelineResult:
6488
started_at = datetime.now(timezone.utc).isoformat()
6589
t0 = time.monotonic()

‎examples/optimization/eval_optimize_loop/run_pipeline.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,10 @@
1111
if str(_REPO_ROOT) not in sys.path:
1212
sys.path.insert(0, str(_REPO_ROOT))
1313

14-
from pipeline import EvalOptimizePipeline # noqa: E402
14+
try:
15+
from .pipeline import EvalOptimizePipeline # noqa: E402
16+
except ImportError:
17+
from pipeline import EvalOptimizePipeline # type: ignore
1518

1619

1720
async def main() -> None:

0 commit comments

Comments
 (0)