1- #!/usr/bin/env python3
1+ #!/usr/bin/env python3
22"""Eval-Optimize Loop CLI entry point.
33
44Usage:
55 python run_pipeline.py # fake mode (fast smoke test)
6- python run_pipeline.py --mode real # real mode (needs PlateAgent)
76 python run_pipeline.py --max-iter 3 # max optimization iterations
7+ python run_pipeline.py --quiet # minimal output
88"""
99
1010import argparse , asyncio , json , os as _os , sys , time
@@ -49,6 +49,12 @@ async def main():
4949 parser .add_argument ("--quiet" , action = "store_true" )
5050 args = parser .parse_args ()
5151
52+ # Gate unimplemented modes at CLI to prevent cryptic NotImplementedError (AI review round 3)
53+ if args .mode in ("real" , "real-agent" ):
54+ print ("Error: --mode real / real-agent is not yet implemented." , file = sys .stderr )
55+ print ("Use --mode fake for the working 6-phase smoke-test pipeline." , file = sys .stderr )
56+ sys .exit (1 )
57+
5258 config = load_config ()
5359 train_path = Path (args .train ) if args .train else BASE_DIR / "config" / "train.evalset.json"
5460 val_path = Path (args .val ) if args .val else BASE_DIR / "config" / "val.evalset.json"
@@ -72,8 +78,7 @@ async def main():
7278
7379 # Phase 1: Baseline
7480 if not args .quiet : print ("[1/6] Baseline..." )
75- br = BaselineRunner (mode = run_mode if run_mode in ("real" ,) else "fake" ,
76- plate_agent_root = str (BASE_DIR .parent .parent .parent / "plate-agent" ))
81+ br = BaselineRunner (mode = "fake" )
7782 baseline = await br .run (train_path , val_path )
7883 train_bl , val_bl = baseline ["train" ], baseline ["val" ]
7984 if not args .quiet :
@@ -89,47 +94,25 @@ async def main():
8994
9095 # Phase 3: Optimization
9196 if not args .quiet : print ("[3/6] Optimization..." )
92- use_real_agent = run_mode == "real-agent"
93- if use_real_agent :
94- sdk_train = BASE_DIR / "config" / "train.sdk.evalset.json"
95- sdk_val = BASE_DIR / "config" / "val.sdk.evalset.json"
96- sdk_opt = BASE_DIR / "config" / "optimizer.sdk.json"
97- prompt_dir = BASE_DIR / "config" / "prompts"
98- from src .call_agent import echo_call_agent
99- opt_runner = OptimizationRunner (
100- mode = "real" ,
101- config = config .get ("pipeline" , {}),
102- call_agent = echo_call_agent ,
103- train_dataset = str (sdk_train ),
104- validation_dataset = str (sdk_val ),
105- sdk_config_path = str (sdk_opt ),
106- prompt_dir = str (prompt_dir ),
107- output_dir = str (output_dir / "optimizer" ),
108- )
109- else :
110- opt_runner = OptimizationRunner (mode = "fake" , config = config .get ("pipeline" , {}))
97+ opt_runner = OptimizationRunner (mode = "fake" , config = config .get ("pipeline" , {}))
11198 opt_result = opt_runner .run (attr )
11299 if not args .quiet : print (f" candidates: { opt_result .total_iterations } " )
113100
114101 # Phase 4: Validation
115102 if not args .quiet : print ("[4/6] Validation..." )
116- vr = ValidationRunner (mode = run_mode if run_mode in ( "real" ,) else "fake" )
103+ vr = ValidationRunner (mode = "fake" )
117104 val_result = vr .run (val_bl , opt_result )
118105 if not args .quiet : print (f" delta: { val_result .summary .avg_score_delta :+.3f} " )
119106
120107 # Phase 5: Gate
121108 if not args .quiet : print ("[5/6] Gate..." )
122109 gate = AcceptanceGate (config .get ("gate" , {}))
123110
124- # fix: overfit detection — run candidate on train set for real delta
125- if run_mode == "real" :
126- candidate_train = await br .run_split (train_path , "train_candidate" )
127- candidate_train_scores = candidate_train .score_map
128- else :
129- # fake mode: derive candidate train scores from baseline + simulated delta
130- candidate_train_scores = {
131- cid : min (1.0 , score + 0.05 ) for cid , score in train_bl .score_map .items ()
132- }
111+ # overfit detection: simulate candidate train scores with +0.05 delta
112+ # (real-mode candidate_train would need optimized-agent re-eval; real mode is not yet implemented)
113+ candidate_train_scores = {
114+ cid : min (1.0 , score + 0.05 ) for cid , score in train_bl .score_map .items ()
115+ }
133116
134117 critical_case_ids = _read_critical_case_ids (val_path )
135118
0 commit comments