88from pathlib import Path
99from typing import Any
1010from typing import Iterable
11- from typing import Protocol
1211
12+ from .diffing import make_unified_diff
1313from .evaluator import ExampleEvaluator
1414from .fake_judge import FakeJudge
1515from .fake_model import FakeModel
1919from .schemas import EvalResult
2020
2121
22- class EvalOptimizeBackend (Protocol ):
23- def evaluate (self , * , prompt_id : str , prompt : str , cases : Iterable [EvalCase ], split : str ) -> EvalResult :
24- ...
25-
26- def optimize (
27- self ,
28- * ,
29- baseline_prompt : str ,
30- train_path : str | Path ,
31- val_path : str | Path ,
32- optimizer_config_path : str | Path ,
33- output_dir : str | Path ,
34- ) -> list [CandidatePrompt ]:
35- ...
36-
37-
3822@dataclass
3923class FakeBackend :
4024 seed : int = 91
@@ -61,20 +45,43 @@ def optimize(
6145
6246@dataclass
6347class SDKBackend :
64- """Thin adapter around AgentOptimizer/TargetPrompt for real SDK runs."""
48+ """Thin optimizer adapter around AgentOptimizer/TargetPrompt for SDK runs.
49+
50+ SDK mode relies on AgentOptimizer's internal evaluation loop. It does not
51+ implement the fake per-case ``evaluate`` API.
52+ """
6553
6654 prompt_path : str | Path
6755 call_agent_path : str | None = None
6856 update_source : bool = False
57+ last_result_summary : dict [str , Any ] | None = None
58+ last_artifact_dir : str | None = None
6959
70- def evaluate (self , * , prompt_id : str , prompt : str , cases : Iterable [EvalCase ], split : str ) -> EvalResult :
71- raise ValueError (
72- "sdk mode evaluation expects SDK evalset files and is performed by AgentOptimizer/AgentEvaluator. "
73- "Use --sdk-call-agent module:function and SDK-compatible train/val evalsets; fake EvalCase objects "
74- "cannot be evaluated by the SDK adapter."
60+ def optimize (
61+ self ,
62+ * ,
63+ baseline_prompt : str ,
64+ train_path : str | Path ,
65+ val_path : str | Path ,
66+ optimizer_config_path : str | Path ,
67+ output_dir : str | Path ,
68+ ) -> list [CandidatePrompt ]:
69+ if _has_running_loop ():
70+ raise ValueError (
71+ "SDKBackend.optimize() cannot be called while an event loop is already running; "
72+ "use await SDKBackend.optimize_async(...) instead."
73+ )
74+ return asyncio .run (
75+ self .optimize_async (
76+ baseline_prompt = baseline_prompt ,
77+ train_path = train_path ,
78+ val_path = val_path ,
79+ optimizer_config_path = optimizer_config_path ,
80+ output_dir = output_dir ,
81+ )
7582 )
7683
77- def optimize (
84+ async def optimize_async (
7885 self ,
7986 * ,
8087 baseline_prompt : str ,
@@ -91,35 +98,38 @@ def optimize(
9198 )
9299 call_agent = _load_call_agent (self .call_agent_path )
93100 try :
94- from trpc_agent_sdk .evaluation import AgentEvaluator
95101 from trpc_agent_sdk .evaluation import AgentOptimizer
96102 from trpc_agent_sdk .evaluation import TargetPrompt
97103 except Exception as exc : # pragma: no cover - depends on optional SDK import health
98- raise ValueError (f"sdk mode could not import AgentEvaluator/ AgentOptimizer/TargetPrompt: { exc } ) from exc
104+ raise ValueError (f"sdk mode could not import AgentOptimizer/TargetPrompt: { exc } ) from exc
99105
100- _ = AgentEvaluator
101106 target_prompt = TargetPrompt ().add_path ("system_prompt" , str (self .prompt_path ))
102- result = asyncio .run (
103- AgentOptimizer .optimize (
104- config_path = str (optimizer_config_path ),
105- call_agent = call_agent ,
106- target_prompt = target_prompt ,
107- train_dataset_path = str (train_path ),
108- validation_dataset_path = str (val_path ),
109- output_dir = str (output_dir ),
110- update_source = self .update_source ,
111- verbose = 0 ,
112- )
107+ result = await AgentOptimizer .optimize (
108+ config_path = str (optimizer_config_path ),
109+ call_agent = call_agent ,
110+ target_prompt = target_prompt ,
111+ train_dataset_path = str (train_path ),
112+ validation_dataset_path = str (val_path ),
113+ output_dir = str (output_dir ),
114+ update_source = self .update_source ,
115+ verbose = 0 ,
113116 )
114117 best_prompt = getattr (result , "best_prompts" , {}).get ("system_prompt" )
115118 if not best_prompt :
116119 raise ValueError ("sdk mode completed but OptimizeResult.best_prompts['system_prompt'] was missing" )
120+ self .last_result_summary = _summarize_sdk_result (result )
121+ self .last_artifact_dir = str (output_dir )
117122 return [
118123 CandidatePrompt (
119124 candidate_id = "sdk_best" ,
120125 prompt = best_prompt ,
121126 rationale = "Best prompt returned by AgentOptimizer.optimize." ,
122- prompt_diff = _simple_diff (baseline_prompt , best_prompt ),
127+ prompt_diff = make_unified_diff (
128+ baseline_prompt ,
129+ best_prompt ,
130+ before_name = "baseline_system_prompt.txt" ,
131+ after_name = "sdk_best/system_prompt.txt" ,
132+ ),
123133 )
124134 ]
125135
@@ -128,14 +138,39 @@ def _load_call_agent(path: str):
128138 if ":" not in path :
129139 raise ValueError ("--sdk-call-agent must use module:function format" )
130140 module_name , function_name = path .split (":" , 1 )
131- module = importlib .import_module (module_name )
141+ try :
142+ module = importlib .import_module (module_name )
143+ except Exception as exc :
144+ raise ValueError (f"--sdk-call-agent target { path !r} { module_name !r} { exc } ) from exc
132145 call_agent = getattr (module , function_name , None )
133146 if call_agent is None :
134147 raise ValueError (f"--sdk-call-agent target { path !r} )
135148 return call_agent
136149
137150
138- def _simple_diff (before : str , after : str ) -> str :
139- if before == after :
140- return "# no prompt changes"
141- return "- baseline system_prompt\n + optimized system_prompt"
151+ def _has_running_loop () -> bool :
152+ try :
153+ asyncio .get_running_loop ()
154+ except RuntimeError :
155+ return False
156+ return True
157+
158+
159+ def _summarize_sdk_result (result : Any ) -> dict [str , Any ]:
160+ if hasattr (result , "model_dump" ):
161+ payload = result .model_dump (mode = "json" )
162+ elif hasattr (result , "__dict__" ):
163+ payload = dict (result .__dict__ )
164+ else :
165+ payload = {"repr" : repr (result )}
166+ return _safe_jsonable (payload )
167+
168+
169+ def _safe_jsonable (value : Any ) -> Any :
170+ if isinstance (value , dict ):
171+ return {str (key ): _safe_jsonable (item ) for key , item in value .items ()}
172+ if isinstance (value , (list , tuple )):
173+ return [_safe_jsonable (item ) for item in value ]
174+ if isinstance (value , (str , int , float , bool )) or value is None :
175+ return value
176+ return repr (value )
0 commit comments