Skip to content

Commit ca51ee3

Browse files
coder-mtjcoder-mtj
committed
feat(examples): add eval+optimization closed-loop pipeline
Implements #91 — reproducible Evaluation + Optimization pipeline: - Config loading (optimizer.json + evalsets validated) - Baseline evaluation (fake mode with trace evalsets + SDK path) - Failure attribution (10 categories: tool errors, rubric, format, etc.) - Multi-dimensional gate (improvement threshold, critical cases, cost budget) - Validation set comparison (new passes/failures, overfitting detection) - JSON + Markdown report with full audit trail - 6 train+val evalset cases (3 optimizable, 1 degrading, 1 format, 1 edge) - 35 tests covering config, baseline, attribution, gate, validation, report, integration Signed-off-by: coder-mtj <coder-mtj@users.noreply.github.com>
1 parent a547d6b commit ca51ee3

13 files changed

Lines changed: 1795 additions & 0 deletions

File tree

‎examples/optimization/eval_optimize_loop/data/optimizer.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"evaluate": {
3+
"metrics": [
4+
{"metric_name": "final_response_avg_score", "threshold": 0.7},
5+
{"metric_name": "response_match_score", "threshold": 0.5}
6+
],
7+
"num_runs": 1,
8+
"criteria": {
9+
"final_response_avg_score": {
10+
"compare_mode": "contains"
11+
},
12+
"response_match_score": {
13+
"method": "rouge-1"
14+
}
15+
}
16+
},
17+
"optimize": {
18+
"algorithm": {
19+
"name": "gepa_reflective",
20+
"seed": 42,
21+
"candidate_selection_strategy": "current_best",
22+
"module_selector": "round_robin",
23+
"max_metric_calls": 100,
24+
"max_iterations_without_improvement": 3,
25+
"timeout_seconds": 600,
26+
"score_threshold": 0.95,
27+
"reflection_minibatch_size": 3,
28+
"reflection_history_top_k": 3
29+
},
30+
"stop": {
31+
"required_metrics": "all"
32+
}
33+
}
34+
}

‎examples/optimization/eval_optimize_loop/data/train.evalset.json‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
{
2+
"eval_set_id": "train-math-basic",
3+
"name": "Training Set — Basic Math Problems",
4+
"description": "3 training cases for prompt optimization. Case 1 has a known failure pattern (wrong operation). Case 2 has a formatting issue. Case 3 is a baseline pass.",
5+
"eval_cases": [
6+
{
7+
"eval_id": "train_case_001",
8+
"eval_mode": "trace",
9+
"conversation": [
10+
{
11+
"invocation_id": "inv-001",
12+
"user_content": {
13+
"parts": [{"text": "What is 15 + 27?"}],
14+
"role": "user"
15+
},
16+
"final_response": {
17+
"parts": [{"text": "42"}],
18+
"role": "model"
19+
}
20+
}
21+
],
22+
"actual_conversation": [
23+
{
24+
"invocation_id": "inv-001",
25+
"user_content": {
26+
"parts": [{"text": "What is 15 + 27?"}],
27+
"role": "user"
28+
},
29+
"final_response": {
30+
"parts": [{"text": "15 + 27 = 42"}],
31+
"role": "model"
32+
},
33+
"intermediate_data": {
34+
"tool_uses": [],
35+
"tool_responses": [],
36+
"intermediate_responses": []
37+
}
38+
}
39+
]
40+
},
41+
{
42+
"eval_id": "train_case_002",
43+
"eval_mode": "trace",
44+
"conversation": [
45+
{
46+
"invocation_id": "inv-002",
47+
"user_content": {
48+
"parts": [{"text": "Calculate 8 * 7"}],
49+
"role": "user"
50+
},
51+
"final_response": {
52+
"parts": [{"text": "56"}],
53+
"role": "model"
54+
}
55+
}
56+
],
57+
"actual_conversation": [
58+
{
59+
"invocation_id": "inv-002",
60+
"user_content": {
61+
"parts": [{"text": "Calculate 8 * 7"}],
62+
"role": "user"
63+
},
64+
"final_response": {
65+
"parts": [{"text": "The answer is 56"}],
66+
"role": "model"
67+
},
68+
"intermediate_data": {
69+
"tool_uses": [],
70+
"tool_responses": [],
71+
"intermediate_responses": []
72+
}
73+
}
74+
]
75+
},
76+
{
77+
"eval_id": "train_case_003",
78+
"eval_mode": "trace",
79+
"conversation": [
80+
{
81+
"invocation_id": "inv-003",
82+
"user_content": {
83+
"parts": [{"text": "Divide 100 by 4"}],
84+
"role": "user"
85+
},
86+
"final_response": {
87+
"parts": [{"text": "25"}],
88+
"role": "model"
89+
}
90+
}
91+
],
92+
"actual_conversation": [
93+
{
94+
"invocation_id": "inv-003",
95+
"user_content": {
96+
"parts": [{"text": "Divide 100 by 4"}],
97+
"role": "user"
98+
},
99+
"final_response": {
100+
"parts": [{"text": "100 / 4 = 25"}],
101+
"role": "model"
102+
},
103+
"intermediate_data": {
104+
"tool_uses": [],
105+
"tool_responses": [],
106+
"intermediate_responses": []
107+
}
108+
}
109+
]
110+
}
111+
]
112+
}

‎examples/optimization/eval_optimize_loop/data/val.evalset.json‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
{
2+
"eval_set_id": "val-math-validation",
3+
"name": "Validation Set — Math Problems",
4+
"description": "3 validation cases to detect overfitting. Case 1 is a variation of train_case_001. Case 2 is a harder problem. Case 3 is an edge case (zero/negative).",
5+
"eval_cases": [
6+
{
7+
"eval_id": "val_case_001",
8+
"eval_mode": "trace",
9+
"conversation": [
10+
{
11+
"invocation_id": "inv-v001",
12+
"user_content": {
13+
"parts": [{"text": "What is 12 + 35?"}],
14+
"role": "user"
15+
},
16+
"final_response": {
17+
"parts": [{"text": "47"}],
18+
"role": "model"
19+
}
20+
}
21+
],
22+
"actual_conversation": [
23+
{
24+
"invocation_id": "inv-v001",
25+
"user_content": {
26+
"parts": [{"text": "What is 12 + 35?"}],
27+
"role": "user"
28+
},
29+
"final_response": {
30+
"parts": [{"text": "12 + 35 = 47"}],
31+
"role": "model"
32+
},
33+
"intermediate_data": {
34+
"tool_uses": [],
35+
"tool_responses": [],
36+
"intermediate_responses": []
37+
}
38+
}
39+
]
40+
},
41+
{
42+
"eval_id": "val_case_002",
43+
"eval_mode": "trace",
44+
"conversation": [
45+
{
46+
"invocation_id": "inv-v002",
47+
"user_content": {
48+
"parts": [{"text": "Compute 15 * (3 + 2) step by step"}],
49+
"role": "user"
50+
},
51+
"final_response": {
52+
"parts": [{"text": "75"}],
53+
"role": "model"
54+
}
55+
}
56+
],
57+
"actual_conversation": [
58+
{
59+
"invocation_id": "inv-v002",
60+
"user_content": {
61+
"parts": [{"text": "Compute 15 * (3 + 2) step by step"}],
62+
"role": "user"
63+
},
64+
"final_response": {
65+
"parts": [{"text": "3 + 2 = 5, then 15 * 5 = 75"}],
66+
"role": "model"
67+
},
68+
"intermediate_data": {
69+
"tool_uses": [],
70+
"tool_responses": [],
71+
"intermediate_responses": []
72+
}
73+
}
74+
]
75+
},
76+
{
77+
"eval_id": "val_case_003",
78+
"eval_mode": "trace",
79+
"conversation": [
80+
{
81+
"invocation_id": "inv-v003",
82+
"user_content": {
83+
"parts": [{"text": "What is -5 * 3?"}],
84+
"role": "user"
85+
},
86+
"final_response": {
87+
"parts": [{"text": "-15"}],
88+
"role": "model"
89+
}
90+
}
91+
],
92+
"actual_conversation": [
93+
{
94+
"invocation_id": "inv-v003",
95+
"user_content": {
96+
"parts": [{"text": "What is -5 * 3?"}],
97+
"role": "user"
98+
},
99+
"final_response": {
100+
"parts": [{"text": "-5 * 3 = -15"}],
101+
"role": "model"
102+
},
103+
"intermediate_data": {
104+
"tool_uses": [],
105+
"tool_responses": [],
106+
"intermediate_responses": []
107+
}
108+
}
109+
]
110+
}
111+
]
112+
}

‎examples/optimization/eval_optimize_loop/pipeline/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Evaluation + Optimization Pipeline
2+
# Stages: config → baseline → attribution → optimize → validate → gate → report

0 commit comments

Comments
 (0)