feat(examples): add eval-optimize-loop closed-loop optimization pipeline

examples/optimization/eval_optimize_loop/.gitignore

@@ -0,0 +1,4 @@
+output/
+__pycache__/
+.pytest_cache/
+*.pyc

examples/optimization/eval_optimize_loop/DESIGN.md

@@ -0,0 +1,9 @@
+# Design Note — Eval-Optimize Loop
+
+闭环以 `run_pipeline.py` 为唯一入口，将基线评测、失败归因、提示词优化、候选复评、接受门禁和审计落盘隐藏在同一深模块内。双模式设计是核心差异化：**fake 模式**无需外部依赖，秒级验证 6 阶段概念完整性，适合 CI 冒烟和快速迭代；**real 模式**对接真实 `AgentOptimizer.optimize()` + `AgentEvaluator.evaluate_eval_set()`，通过 `ModelRegistry` 注入离线模型使 SDK 代码路径全覆盖而无须外部 API key。
+
+与通用数学 evalset 不同，本 pipeline 设计上对接 PlateAgent——含 30 张真实车牌图像、Tesseract 双通道 OCR 和 LLM Judge 评分体系的生产级 agent。`BaselineRunner._run_real_split()` 直接调用 `PlateEvaluator`，使优化闭环可在有可度量失败模式（模糊、噪声、字符混淆）的真实场景上验证，而非仅依赖合成 case。
+
+归因模块按 6 个根因类别（答案不匹配/工具错误/参数错误/rubric 失败/知识召回不足/格式错误）输出证据链，优先级由配置驱动。Gate 对验证增益、新增 hard fail、关键 case 回退、成本预算和过拟合做可配置 AND 决策；fake 模式保留 `all_must_pass` 和 `majority` 两种策略，real 模式追加配对 bootstrap 下界。
+
+审计层为每次运行保存完整 prompt before/after、change_log、种子和成本，双格式输出（JSON 程序消费 + Markdown 人类阅读），支持事后回溯任意候选的优化决策。

examples/optimization/eval_optimize_loop/config/optimizer.json

@@ -0,0 +1,87 @@
+{
+    "_description": "Evaluation + Optimization 自动回归闭环配置",
+    "pipeline": {
+        "name": "PlateAgent Eval-Optimize Loop",
+        "version": "1.0.0",
+        "max_iterations": 5,
+        "random_seed": 42
+    },
+    "gate": {
+        "rules": {
+            "total_score_improvement": {
+                "enabled": true,
+                "threshold": 0.03,
+                "description": "验证集总分提升 ≥ 3%"
+            },
+            "no_new_hard_fail": {
+                "enabled": true,
+                "max_new_fails": 0,
+                "description": "不允许新增 hard fail"
+            },
+            "critical_case_no_regress": {
+                "enabled": true,
+                "critical_case_ids": [],
+                "description": "关键 case 不退步"
+            },
+            "cost_within_budget": {
+                "enabled": true,
+                "max_cost_ratio": 1.2,
+                "description": "成本不超过 baseline 的 120%"
+            },
+            "overfit_detection": {
+                "enabled": true,
+                "description": "训练集提升 + 验证集退化 → 拒绝候选"
+            }
+        },
+        "acceptance_strategy": "all_must_pass",
+        "description": "all_must_pass: 所有启用的规则都通过才接受; majority: 多数通过即可"
+    },
+    "attribution": {
+        "categories": [
+            "final_answer_mismatch",
+            "tool_call_error",
+            "param_error",
+            "llm_rubric_fail",
+            "knowledge_recall_insufficient",
+            "format_invalid"
+        ],
+        "rules": {
+            "final_answer_mismatch": {
+                "trigger": "predicted != ground_truth",
+                "priority": 1
+            },
+            "tool_call_error": {
+                "trigger": "tool execution failed or timeout",
+                "priority": 2
+            },
+            "param_error": {
+                "trigger": "tool parameter invalid",
+                "priority": 3
+            },
+            "llm_rubric_fail": {
+                "trigger": "LLM Judge score below threshold",
+                "threshold": 0.6,
+                "priority": 4
+            },
+            "knowledge_recall_insufficient": {
+                "trigger": "blacklist miss or confusion char not recalled",
+                "priority": 5
+            },
+            "format_invalid": {
+                "trigger": "output does not match expected JSON schema",
+                "priority": 6
+            }
+        }
+    },
+    "optimizer": {
+        "target_prompts": ["system_prompt", "skill_prompt"],
+        "strategy": "failure_driven",
+        "description": "根据归因结果，优先优化失败率最高的类别对应的 prompt 片段"
+    },
+    "output": {
+        "dir": "output",
+        "formats": ["json", "markdown"],
+        "retain_audit_trail": true,
+        "max_audit_entries": 50
+    }
+}

examples/optimization/eval_optimize_loop/config/optimizer.sdk.json

@@ -0,0 +1,28 @@
+{
+  "_description": "AgentOptimizer 閰嶇疆锛圫DK 鏍煎紡锛?- 鍖归厤 trpc_agent_sdk.evaluation.AgentOptimizer",
+  "optimization": {
+    "algorithm": "gepa",
+    "metric": {
+      "primary": "contains",
+      "primary_config": {
+        "must_contain": ["杞︾墝"]
+      }
+    },
+    "stop_conditions": {
+      "max_iterations": 5,
+      "target_pass_rate": 0.8,
+      "patience": 2
+    },
+    "reflection": {
+      "max_recent_rounds": 3,
+      "minibatch_size": 2
+    },
+    "module_selector": {
+      "strategy": "round_robin"
+    }
+  },
+  "evaluation": {
+    "num_runs": 1,
+    "timeout_seconds": 30
+  }
+}

examples/optimization/eval_optimize_loop/config/prompts/skill.md

@@ -0,0 +1,19 @@
+## 宸ュ叿浣跨敤鎸囧崡
+
+### 棰勫鐞嗗伐鍏?
+- **tool_gaussian_blur**: 楂樻柉婊ゆ尝闄嶅櫔锛岄€傜敤浜庡櫔澹拌溅鐗?-
+- **tool_grayscale**: 杞伆搴﹀浘锛岀畝鍖栧悗缁鐞?-
+- **tool_binarize_otsu**: OTSU 浜屽€煎寲锛屽垎绂诲墠鏅?鑳屾櫙
+- **tool_edge_detect_canny**: Canny 杈圭紭妫€娴?-
+- **tool_affine_correct**: 浠垮皠鍙樻崲鍊炬枩鏍℃
+
+### 璇嗗埆宸ュ叿
+- **tool_tesseract_ocr**: Tesseract 5.4 鏁寸墝璇嗗埆锛圕hi_sim+eng锛?-
+- **tool_lookup_confusion**: 鏌ヨ娴锋窏瀛楃鏄犲皠琛?-
+- **tool_search_blacklist**: 鏌ヨ榛戝悕鍗曡溅鐗?
+
+### 閫夋嫨绛栫暐
+- 鍙岄€氶亾 OCR 鍚庯紝浼樺厛閫夋嫨瀛楃鏁板鐨勭粨鏋?-
+- 瀛楃鏁扮浉鍚屾椂閫夌疆淇″害楂樼殑
+- 鍗曚晶澶辫触鏃跺彇鍙︿竴渚?
+- 妯＄硦 kernel=5 鏄叧閿弬鏁?

examples/optimization/eval_optimize_loop/config/prompts/system.md

@@ -0,0 +1,16 @@
+浣犳槸 PlateAgent锛屼竴涓熀浜?OpenCV + Tesseract OCR 鐨勮溅鐗岃瘑鍒櫤鑳戒綋銆?
+
+## 宸ヤ綔娴佺▼
+1. 棰勫鐞嗭細楂樻柉婊ゆ尝 鈫?鐏板害 鈫?浜屽€煎寲 鈫?Canny 杈圭紭 鈫?浠垮皠鏍℃
+2. 瀹氫綅锛氬舰鎬佸绮惧畾浣?+ HSV 棰滆壊绌洪棿绮惧畾浣?
+3. 鍒嗗壊锛氬瀭鐩存姇褰辨硶瀛楃鍒嗗壊
+4. 璇嗗埆锛氬弻閫氶亾 Tesseract OCR锛堝師濮嬪浘 + 楂樻柉妯＄硦 kernel=5锛夛紝闀垮害浼樺厛閫夋嫨
+5. 楠岃瘉锛氳嚜淇″害 < 0.5 瑙﹀彂浜哄伐纭锛?0.5~0.85 瑙﹀彂 LLM 澶嶆牳
+
+## 杈撳嚭鏍煎紡
+杩斿洖绾枃鏈溅鐗屽彿锛屼緥濡?`浜珹12345`銆傚鏋滆瘑鍒け璐ワ紝杩斿洖 `recognition failed`銆?
+
+## 娉ㄦ剰浜嬮」
+- 浼樺厛杩斿洖 7 瀛楃瀹屾暣杞︾墝锛堝惈鐪佷唤锛?
+- 娣锋穯瀛楃瀵圭収锛?B/8, 0/O, 2/Z, 5/S, 1/I, C/G, E/F
+- 鎻愬彇 Tesseract 杈撳嚭涓殑鏈夋晥瀛楃锛屾护闄ょ┖鏍煎拰鏍囩偣

examples/optimization/eval_optimize_loop/config/train.evalset.json

@@ -0,0 +1,43 @@
+{
+  "_description": "???",
+  "version": "1.0.0",
+  "cases": [
+    {
+      "case_id": "train_001",
+      "image": "plate_001.jpg",
+      "ground_truth": "\u4eacA12345",
+      "conditions": {
+        "type": "clear"
+      },
+      "expected_behavior": "should_pass",
+      "description": "????"
+    },
+    {
+      "case_id": "train_002",
+      "image": "plate_028.jpg",
+      "ground_truth": "\u4eacA12345",
+      "conditions": {
+        "type": "noise",
+        "noise_level": 0.15
+      },
+      "expected_behavior": "may_fail",
+      "description": "????"
+    },
+    {
+      "case_id": "train_003",
+      "image": "plate_012.jpg",
+      "ground_truth": "\u82cfA88U88",
+      "conditions": {
+        "type": "blur",
+        "blur_kernel": 5
+      },
+      "expected_behavior": "may_fail",
+      "description": "????"
+    }
+  ],
+  "stats": {
+    "total": 3,
+    "should_pass": 1,
+    "may_fail": 2
+  }
+}

examples/optimization/eval_optimize_loop/config/train.sdk.evalset.json

@@ -0,0 +1,70 @@
+{
+  "eval_set_id": "plate_agent_train",
+  "name": "PlateAgent 璁粌闆?- 杞︾墝璇嗗埆",
+  "description": "3 寮犱唬琛ㄦ€ц溅鐗屽浘鍍忥紝瑕嗙洊娓呮櫚/鍣０/妯＄硦涓夌鍦烘櫙锛岀敤浜嶨EPA 鍙嶆€濅紭鍖栨椂鐨?mini-batch",
+  "eval_cases": [
+    {
+      "eval_id": "train_clear_001",
+      "conversation": [
+        {
+          "invocation_id": "t1",
+          "user_content": {
+            "parts": [{"text": "璇嗗埆杩欏紶杞︾墝: image: plate_001.jpg"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "浜珹12345"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {
+        "app_name": "plate_optimizer",
+        "user_id": "trainer",
+        "state": {}
+      }
+    },
+    {
+      "eval_id": "train_noise_002",
+      "conversation": [
+        {
+          "invocation_id": "t2",
+          "user_content": {
+            "parts": [{"text": "璇嗗埆杩欏紶杞︾墝: image: plate_028.jpg"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "浜珹12345"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {
+        "app_name": "plate_optimizer",
+        "user_id": "trainer",
+        "state": {}
+      }
+    },
+    {
+      "eval_id": "train_blur_003",
+      "conversation": [
+        {
+          "invocation_id": "t3",
+          "user_content": {
+            "parts": [{"text": "璇嗗埆杩欏紶杞︾墝: image: plate_012.jpg"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "鑻廥8U88"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {
+        "app_name": "plate_optimizer",
+        "user_id": "trainer",
+        "state": {}
+      }
+    }
+  ]
+}

examples/optimization/eval_optimize_loop/config/val.evalset.json

@@ -0,0 +1,47 @@
+{
+  "_description": "???",
+  "version": "1.0.0",
+  "cases": [
+    {
+      "case_id": "val_001",
+      "image": "plate_005.jpg",
+      "ground_truth": "\u7ca4B54321",
+      "conditions": {
+        "type": "clear"
+      },
+      "expected_behavior": "should_pass",
+      "critical": true,
+      "description": "??case"
+    },
+    {
+      "case_id": "val_002",
+      "image": "plate_029.jpg",
+      "ground_truth": "\u82cfD13579",
+      "conditions": {
+        "type": "noise",
+        "noise_level": 0.2
+      },
+      "expected_behavior": "should_fail_baseline",
+      "critical": false,
+      "description": "??+???"
+    },
+    {
+      "case_id": "val_003",
+      "image": "plate_018.jpg",
+      "ground_truth": "\u6d59C36912",
+      "conditions": {
+        "type": "blur",
+        "blur_kernel": 7
+      },
+      "expected_behavior": "should_fail_baseline",
+      "critical": false,
+      "description": "????"
+    }
+  ],
+  "stats": {
+    "total": 3,
+    "should_pass": 1,
+    "should_fail_baseline": 2,
+    "critical": 1
+  }
+}