Skip to content

Commit 716d0dd

Browse files
committed
fix(eval_optimize_loop): gate unimplemented real/real-agent at CLI, simplify overfit detection
CLI gate prevents cryptic NotImplementedError when --mode real or --mode real-agent is used. Fake mode remains the only active path with all 99 tests passing. Changes (AI review round 3): - Add explicit CLI rejection for real/real-agent modes - Remove dead real-agent OptimizationRunner branch - Simplify BaselineRunner/ValidationRunner to always use fake - Unify overfit detection to simulated approach (real branch was broken: reused BaselineRunner without optimized prompt)
1 parent 5f5002e commit 716d0dd

1 file changed

Lines changed: 16 additions & 33 deletions

File tree

examples/optimization/eval_optimize_loop/run_pipeline.py

Lines changed: 16 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
#!/usr/bin/env python3
1+
#!/usr/bin/env python3
22
"""Eval-Optimize Loop CLI entry point.
33
44
Usage:
55
python run_pipeline.py # fake mode (fast smoke test)
6-
python run_pipeline.py --mode real # real mode (needs PlateAgent)
76
python run_pipeline.py --max-iter 3 # max optimization iterations
7+
python run_pipeline.py --quiet # minimal output
88
"""
99

1010
import argparse, asyncio, json, os as _os, sys, time
@@ -49,6 +49,12 @@ async def main():
4949
parser.add_argument("--quiet", action="store_true")
5050
args = parser.parse_args()
5151

52+
# Gate unimplemented modes at CLI to prevent cryptic NotImplementedError (AI review round 3)
53+
if args.mode in ("real", "real-agent"):
54+
print("Error: --mode real / real-agent is not yet implemented.", file=sys.stderr)
55+
print("Use --mode fake for the working 6-phase smoke-test pipeline.", file=sys.stderr)
56+
sys.exit(1)
57+
5258
config = load_config()
5359
train_path = Path(args.train) if args.train else BASE_DIR / "config" / "train.evalset.json"
5460
val_path = Path(args.val) if args.val else BASE_DIR / "config" / "val.evalset.json"
@@ -72,8 +78,7 @@ async def main():
7278

7379
# Phase 1: Baseline
7480
if not args.quiet: print("[1/6] Baseline...")
75-
br = BaselineRunner(mode=run_mode if run_mode in ("real",) else "fake",
76-
plate_agent_root=str(BASE_DIR.parent.parent.parent / "plate-agent"))
81+
br = BaselineRunner(mode="fake")
7782
baseline = await br.run(train_path, val_path)
7883
train_bl, val_bl = baseline["train"], baseline["val"]
7984
if not args.quiet:
@@ -89,47 +94,25 @@ async def main():
8994

9095
# Phase 3: Optimization
9196
if not args.quiet: print("[3/6] Optimization...")
92-
use_real_agent = run_mode == "real-agent"
93-
if use_real_agent:
94-
sdk_train = BASE_DIR / "config" / "train.sdk.evalset.json"
95-
sdk_val = BASE_DIR / "config" / "val.sdk.evalset.json"
96-
sdk_opt = BASE_DIR / "config" / "optimizer.sdk.json"
97-
prompt_dir = BASE_DIR / "config" / "prompts"
98-
from src.call_agent import echo_call_agent
99-
opt_runner = OptimizationRunner(
100-
mode="real",
101-
config=config.get("pipeline", {}),
102-
call_agent=echo_call_agent,
103-
train_dataset=str(sdk_train),
104-
validation_dataset=str(sdk_val),
105-
sdk_config_path=str(sdk_opt),
106-
prompt_dir=str(prompt_dir),
107-
output_dir=str(output_dir / "optimizer"),
108-
)
109-
else:
110-
opt_runner = OptimizationRunner(mode="fake", config=config.get("pipeline", {}))
97+
opt_runner = OptimizationRunner(mode="fake", config=config.get("pipeline", {}))
11198
opt_result = opt_runner.run(attr)
11299
if not args.quiet: print(f" candidates: {opt_result.total_iterations}")
113100

114101
# Phase 4: Validation
115102
if not args.quiet: print("[4/6] Validation...")
116-
vr = ValidationRunner(mode=run_mode if run_mode in ("real",) else "fake")
103+
vr = ValidationRunner(mode="fake")
117104
val_result = vr.run(val_bl, opt_result)
118105
if not args.quiet: print(f" delta: {val_result.summary.avg_score_delta:+.3f}")
119106

120107
# Phase 5: Gate
121108
if not args.quiet: print("[5/6] Gate...")
122109
gate = AcceptanceGate(config.get("gate", {}))
123110

124-
# fix: overfit detection — run candidate on train set for real delta
125-
if run_mode == "real":
126-
candidate_train = await br.run_split(train_path, "train_candidate")
127-
candidate_train_scores = candidate_train.score_map
128-
else:
129-
# fake mode: derive candidate train scores from baseline + simulated delta
130-
candidate_train_scores = {
131-
cid: min(1.0, score + 0.05) for cid, score in train_bl.score_map.items()
132-
}
111+
# overfit detection: simulate candidate train scores with +0.05 delta
112+
# (real-mode candidate_train would need optimized-agent re-eval; real mode is not yet implemented)
113+
candidate_train_scores = {
114+
cid: min(1.0, score + 0.05) for cid, score in train_bl.score_map.items()
115+
}
133116

134117
critical_case_ids = _read_critical_case_ids(val_path)
135118

0 commit comments

Comments
 (0)