Skip to content

Commit 790f3ab

Browse files
committed
fix(eval_optimize_loop): address AI review round 13 — 4 warnings
Warning fixes: - optimizer.py: cross prompt_type accumulation guard — reset prompt_before to base when target_prompt_type changes between iterations (system_prompt -> skill_prompt) - gate.py: cost gate truly skipped when baseline_cost <= 0 — _check_cost returns None, decide() excludes it from checks list (was auto-passing with passed=True contrary to skip comment) - run_pipeline.py: Windows _pid_alive uses OpenProcess first, avoiding os.kill(pid,0) which CPython maps to TerminateProcess on Windows (would kill the lock holder instead of probing) - baseline.py: sys.path restored in finally block on BOTH success and error paths; uses sys.path.remove() by value instead of fragile pop(0) positional removal 99 tests pass, pipeline 6 phases run end-to-end.
1 parent 4b9a432 commit 790f3ab

4 files changed

Lines changed: 41 additions & 38 deletions

File tree

examples/optimization/eval_optimize_loop/run_pipeline.py

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -68,20 +68,17 @@ async def main():
6868
_os.makedirs(str(output_dir), exist_ok=True)
6969

7070
def _pid_alive(pid):
71-
"""Check if a process is running. Cross-platform: signal 0 on Unix,
72-
OpenProcess on Windows. Returns False for dead/invalid PIDs."""
73-
# Unix: os.kill(pid, 0) raises if process doesn't exist
74-
try:
75-
_os.kill(pid, 0)
76-
except ProcessLookupError:
77-
return False
78-
except PermissionError:
79-
return True # exists but we can't signal it
80-
except OSError:
81-
pass # fall through to platform check
82-
83-
# Windows: use kernel32.OpenProcess
71+
"""Check if a process is running. Returns False for dead/invalid PIDs.
72+
73+
Platform strategy:
74+
- Windows: use kernel32.OpenProcess (PROCESS_QUERY_LIMITED_INFORMATION).
75+
We avoid os.kill(pid, 0) on Windows because CPython maps it to
76+
TerminateProcess(handle, 0) which kills the target, not probes it.
77+
- Unix: os.kill(pid, 0) sends signal 0 (null signal) which is a
78+
pure liveness probe per POSIX.
79+
"""
8480
if sys.platform == "win32":
81+
# Windows first: avoid os.kill which terminates the process
8582
try:
8683
import ctypes
8784
h = ctypes.windll.kernel32.OpenProcess(0x0400, False, pid)
@@ -90,10 +87,18 @@ def _pid_alive(pid):
9087
return True
9188
return False
9289
except Exception:
93-
return False # cannot verify liveness; treat as dead so stale lock can be cleaned
94-
# On Unix, if os.kill(pid, 0) succeeded above, process exists
95-
return True
90+
return False # cannot verify; assume dead to allow lock cleanup
9691

92+
# Unix: signal 0 is a pure liveness probe
93+
try:
94+
_os.kill(pid, 0)
95+
except ProcessLookupError:
96+
return False
97+
except PermissionError:
98+
return True # exists but we cannot signal it
99+
except OSError:
100+
return False # cannot verify; assume dead
101+
return True
97102
my_pid = _os.getpid()
98103

99104
# Atomic acquire: O_CREAT|O_EXCL fails if file exists (cross-platform)

examples/optimization/eval_optimize_loop/src/baseline.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,18 +322,22 @@ async def _run_real_split(
322322
)
323323

324324
import sys
325-
sys.path.insert(0, str(Path(plate_agent_root)))
326-
325+
plate_root_str = str(Path(plate_agent_root))
326+
sys.path.insert(0, plate_root_str)
327+
path_restored = False
327328
try:
328329
from agent.session_manager import create_session_service, create_memory_service
329330
from eval.evaluator import PlateEvaluator
330331
except ImportError as e:
331-
sys.path.pop(0) # restore path before raising (fix: round-5 review)
332+
sys.path.remove(plate_root_str) # restore before raising
333+
path_restored = True
332334
raise ImportError(
333335
f"Cannot import PlateAgent modules from {plate_agent_root}. "
334336
f"Ensure trpc_agent_sdk is installed. Error: {e}"
335337
)
336-
338+
finally:
339+
if not path_restored:
340+
sys.path.remove(plate_root_str) # restore on success path too
337341
# 构建 ground_truth.json 格式(临时文件)
338342
# Build ground_truth items with sequential IDs for stable reverse mapping.
339343
# Uses enumerate(start=1) instead of SHA256 hash so that the id->case_id

examples/optimization/eval_optimize_loop/src/gate.py

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -98,9 +98,9 @@ def decide(
9898

9999
# 4. 成本不超预算
100100
if self._rule_enabled("cost_within_budget"):
101-
checks.append(self._check_cost(
102-
baseline_cost, candidate_cost
103-
))
101+
cost_check = self._check_cost(baseline_cost, candidate_cost)
102+
if cost_check is not None:
103+
checks.append(cost_check)
104104

105105
# 5. 过拟合检测
106106
if self._rule_enabled("overfit_detection") and baseline_train_scores and candidate_train_scores:
@@ -132,7 +132,7 @@ def _check_total_improvement(
132132
self,
133133
baseline: dict[str, float],
134134
candidate: dict[str, float],
135-
) -> GateCheck:
135+
) -> "Optional[GateCheck]":
136136
threshold = self.rules["total_score_improvement"].get("threshold", 0.03)
137137
base_avg = sum(baseline.values()) / len(baseline) if baseline else 0
138138
cand_avg = sum(candidate.values()) / len(candidate) if candidate else 0
@@ -149,7 +149,7 @@ def _check_no_new_hard_fail(
149149
self,
150150
baseline: dict[str, float],
151151
candidate: dict[str, float],
152-
) -> GateCheck:
152+
) -> "Optional[GateCheck]":
153153
max_new = self.rules["no_new_hard_fail"].get("max_new_fails", 0)
154154
base_fails = sum(1 for s in baseline.values() if s < PASS_THRESHOLD)
155155
cand_fails = sum(1 for s in candidate.values() if s < PASS_THRESHOLD)
@@ -167,7 +167,7 @@ def _check_critical_cases(
167167
baseline: dict[str, float],
168168
candidate: dict[str, float],
169169
critical_ids: list[str],
170-
) -> GateCheck:
170+
) -> "Optional[GateCheck]":
171171
if not critical_ids:
172172
return GateCheck(
173173
name="critical_case_no_regress",
@@ -193,18 +193,12 @@ def _check_cost(
193193
self,
194194
baseline_cost: float,
195195
candidate_cost: float,
196-
) -> GateCheck:
196+
) -> "Optional[GateCheck]":
197197
max_ratio = self.rules["cost_within_budget"].get("max_cost_ratio", 1.2)
198198
if baseline_cost <= 0:
199-
# Fake mode: costs are simulated placeholders (see run_pipeline.py L173-176).
200-
# Real mode: cost may be 0 if token_tracker is not connected.
201-
# In either case, cost data is absent; skip this gate rather than auto-pass.
202-
return GateCheck(
203-
name="cost_within_budget",
204-
passed=True,
205-
description=f"cost budget skipped (baseline_cost={baseline_cost:.4f} <= 0, tracking inactive)",
206-
detail="cost data unavailable -- gate skipped",
207-
)
199+
# Cost data is absent (fake mode simulated / real mode token_tracker not connected).
200+
# Return None so decide() excludes this gate from the checks list.
201+
return None
208202
else:
209203
ratio = candidate_cost / baseline_cost
210204
passed = ratio <= max_ratio
@@ -221,7 +215,7 @@ def _check_overfit(
221215
candidate_train: dict[str, float],
222216
baseline_val: dict[str, float],
223217
candidate_val: dict[str, float],
224-
) -> GateCheck:
218+
) -> "Optional[GateCheck]":
225219
train_avg_base = sum(baseline_train.values()) / len(baseline_train) if baseline_train else 0
226220
train_avg_cand = sum(candidate_train.values()) / len(candidate_train) if candidate_train else 0
227221
val_avg_base = sum(baseline_val.values()) / len(baseline_val) if baseline_val else 0

examples/optimization/eval_optimize_loop/src/optimizer.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ def optimize(
225225
confidence = target["confidence"]
226226

227227
# prompt_before: use previous iteration's result for cumulative optimization
228-
if candidates:
228+
if candidates and candidates[-1].target_prompt_type == prompt_type:
229229
prompt_before = candidates[-1].prompt_after
230230
else:
231231
prompt_before = self._get_base_prompt(prompt_type)

0 commit comments

Comments
 (0)