feat(examples): add eval-optimize-loop closed-loop optimization pipeline

examples/optimization/eval_optimize_loop/.gitignore

@@ -0,0 +1,4 @@
+output/
+__pycache__/
+.pytest_cache/
+*.pyc

examples/optimization/eval_optimize_loop/DESIGN.md

@@ -0,0 +1,9 @@
+# Design Note — Eval-Optimize Loop
+
+闭环以 `run_pipeline.py` 为唯一入口，将基线评测、失败归因、提示词优化、候选复评、接受门禁和审计落盘隐藏在同一深模块内。双模式设计是核心差异化：**fake 模式**无需外部依赖，秒级验证 6 阶段概念完整性，适合 CI 冒烟和快速迭代；**real 模式**对接真实 `AgentOptimizer.optimize()` + `AgentEvaluator.evaluate_eval_set()`，通过 `ModelRegistry` 注入离线模型使 SDK 代码路径全覆盖而无须外部 API key。
+
+与通用数学 evalset 不同，本 pipeline 设计上对接 PlateAgent——含 30 张真实车牌图像、Tesseract 双通道 OCR 和 LLM Judge 评分体系的生产级 agent。`BaselineRunner._run_real_split()` 直接调用 `PlateEvaluator`，使优化闭环可在有可度量失败模式（模糊、噪声、字符混淆）的真实场景上验证，而非仅依赖合成 case。
+
+归因模块按 6 个根因类别（答案不匹配/工具错误/参数错误/rubric 失败/知识召回不足/格式错误）输出证据链，优先级由配置驱动。Gate 对验证增益、新增 hard fail、关键 case 回退、成本预算和过拟合做可配置 AND 决策；fake 模式保留 `all_must_pass` 和 `majority` 两种策略，real 模式追加配对 bootstrap 下界。
+
+审计层为每次运行保存完整 prompt before/after、change_log、种子和成本，双格式输出（JSON 程序消费 + Markdown 人类阅读），支持事后回溯任意候选的优化决策。

examples/optimization/eval_optimize_loop/config/optimizer.json

@@ -0,0 +1,87 @@
+{
+    "_description": "Evaluation + Optimization 自动回归闭环配置",
+    "pipeline": {
+        "name": "PlateAgent Eval-Optimize Loop",
+        "version": "1.0.0",
+        "max_iterations": 5,
+        "random_seed": 42
+    },
+    "gate": {
+        "rules": {
+            "total_score_improvement": {
+                "enabled": true,
+                "threshold": 0.03,
+                "description": "验证集总分提升 ≥ 3%"
+            },
+            "no_new_hard_fail": {
+                "enabled": true,
+                "max_new_fails": 0,
+                "description": "不允许新增 hard fail"
+            },
+            "critical_case_no_regress": {
+                "enabled": true,
+                "critical_case_ids": [],
+                "description": "关键 case 不退步"
+            },
+            "cost_within_budget": {
+                "enabled": true,
+                "max_cost_ratio": 1.2,
+                "description": "成本不超过 baseline 的 120%"
+            },
+            "overfit_detection": {
+                "enabled": true,
+                "description": "训练集提升 + 验证集退化 → 拒绝候选"
+            }
+        },
+        "acceptance_strategy": "all_must_pass",
+        "description": "all_must_pass: 所有启用的规则都通过才接受; majority: 多数通过即可"
+    },
+    "attribution": {
+        "categories": [
+            "final_answer_mismatch",
+            "tool_call_error",
+            "param_error",
+            "llm_rubric_fail",
+            "knowledge_recall_insufficient",
+            "format_invalid"
+        ],
+        "rules": {
+            "final_answer_mismatch": {
+                "trigger": "predicted != ground_truth",
+                "priority": 1
+            },
+            "tool_call_error": {
+                "trigger": "tool execution failed or timeout",
+                "priority": 2
+            },
+            "param_error": {
+                "trigger": "tool parameter invalid",
+                "priority": 3
+            },
+            "llm_rubric_fail": {
+                "trigger": "LLM Judge score below threshold",
+                "threshold": 0.6,
+                "priority": 4
+            },
+            "knowledge_recall_insufficient": {
+                "trigger": "blacklist miss or confusion char not recalled",
+                "priority": 5
+            },
+            "format_invalid": {
+                "trigger": "output does not match expected JSON schema",
+                "priority": 6
+            }
+        }
+    },
+    "optimizer": {
+        "target_prompts": ["system_prompt", "skill_prompt"],
+        "strategy": "failure_driven",
+        "description": "根据归因结果，优先优化失败率最高的类别对应的 prompt 片段"
+    },
+    "output": {
+        "dir": "output",
+        "formats": ["json", "markdown"],
+        "retain_audit_trail": true,
+        "max_audit_entries": 50
+    }
+}

examples/optimization/eval_optimize_loop/config/optimizer.sdk.json

@@ -0,0 +1,28 @@
+{
+  "_description": "AgentOptimizer config (SDK format) - matches trpc_agent_sdk.evaluation.AgentOptimizer",
+  "optimization": {
+    "algorithm": "gepa",
+    "metric": {
+      "primary": "contains",
+      "primary_config": {
+        "must_contain": []
+      }
+    },
+    "stop_conditions": {
+      "max_iterations": 5,
+      "target_pass_rate": 0.8,
+      "patience": 2
+    },
+    "reflection": {
+      "max_recent_rounds": 3,
+      "minibatch_size": 2
+    },
+    "module_selector": {
+      "strategy": "round_robin"
+    }
+  },
+  "evaluation": {
+    "num_runs": 1,
+    "timeout_seconds": 30
+  }
+}

examples/optimization/eval_optimize_loop/config/prompts/skill.md

@@ -0,0 +1,24 @@
+# PlateAgent - Skill Prompt (Tool Usage Guide)
+
+## Preprocess Tools
+- **tool_gaussian_blur**: Gaussian blur denoising, suitable for noisy plates
+- **tool_grayscale**: Convert to grayscale, simplifies downstream processing
+- **tool_binarize_otsu**: OTSU binarization, separates foreground/background
+- **tool_edge_detect_canny**: Canny edge detection
+- **tool_affine_correct**: Affine transform tilt correction
+
+## Recognition Tools
+- **tool_tesseract_ocr**: Tesseract 5.4 whole-plate recognition (chi_sim+eng)
+- **tool_lookup_confusion**: Query confusion character mapping table
+- **tool_search_blacklist**: Query blacklist database
+
+## Selection Strategy
+- After dual-channel OCR, prefer the result with more recognized characters
+- Same character count: prefer higher confidence
+- One channel fails: use the other
+- Gaussian blur kernel=5 is the key parameter
+
+## Output Rules
+- Return ONLY the plate number string, no extra text
+- Failed plates: return "recognition failed"
+- Do NOT include JSON, markdown formatting, or explanations

examples/optimization/eval_optimize_loop/config/prompts/system.md

@@ -0,0 +1,19 @@
+# PlateAgent - System Prompt
+
+You are PlateAgent, a license plate recognition agent based on OpenCV + Tesseract OCR.
+
+## Workflow
+1. Preprocess: Gaussian blur -> Grayscale -> Binarize (OTSU) -> Canny edges -> Affine correction
+2. Locate: Morphology coarse + HSV color-space fine localization
+3. Segment: Vertical projection character segmentation
+4. Recognize: Dual-channel Tesseract OCR (original + GaussianBlur kernel=5), length-priority selection
+5. Verify: confidence < 0.5 triggers human review; 0.5-0.85 triggers LLM re-check
+
+## Output Format
+Return the plain-text plate number, e.g. "京A12345".
+If recognition fails, return "recognition failed".
+
+## Notes
+- Prefer 7-character complete plates (with province prefix)
+- Confusion character mapping: B/8, 0/O, 2/Z, 5/S, 1/I, C/G, E/F
+- Filter valid characters from Tesseract output; strip spaces and punctuation

examples/optimization/eval_optimize_loop/config/train.evalset.json

@@ -0,0 +1,43 @@
+{
+  "_description": "???",
+  "version": "1.0.0",
+  "cases": [
+    {
+      "case_id": "train_001",
+      "image": "plate_001.jpg",
+      "ground_truth": "\u4eacA12345",
+      "conditions": {
+        "type": "clear"
+      },
+      "expected_behavior": "should_pass",
+      "description": "????"
+    },
+    {
+      "case_id": "train_002",
+      "image": "plate_028.jpg",
+      "ground_truth": "\u4eacA12345",
+      "conditions": {
+        "type": "noise",
+        "noise_level": 0.15
+      },
+      "expected_behavior": "may_fail",
+      "description": "????"
+    },
+    {
+      "case_id": "train_003",
+      "image": "plate_012.jpg",
+      "ground_truth": "\u82cfA88U88",
+      "conditions": {
+        "type": "blur",
+        "blur_kernel": 5
+      },
+      "expected_behavior": "may_fail",
+      "description": "????"
+    }
+  ],
+  "stats": {
+    "total": 3,
+    "should_pass": 1,
+    "may_fail": 2
+  }
+}

examples/optimization/eval_optimize_loop/config/train.sdk.evalset.json

@@ -0,0 +1,36 @@
+{
+  "eval_set_id": "plate_agent_train",
+  "name": "PlateAgent Train Set - License Plate Recognition",
+  "description": "3 representative plate images covering clear/noise/blur scenarios for GEPA reflection mini-batch",
+  "eval_cases": [
+    {
+      "eval_id": "train_clear_001",
+      "conversation": [
+        {
+          "invocation_id": "t1",
+          "user_content": {
+            "parts": [
+              {
+                "text": "Identify plate: image: plate_001.jpg"
+              }
+            ],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [
+              {
+                "text": "A12345"
+              }
+            ],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {
+        "app_name": "plate_optimizer",
+        "user_id": "trainer",
+        "state": {}
+      }
+    }
+  ]
+}

examples/optimization/eval_optimize_loop/config/val.evalset.json

@@ -0,0 +1,47 @@
+{
+  "_description": "???",
+  "version": "1.0.0",
+  "cases": [
+    {
+      "case_id": "val_001",
+      "image": "plate_005.jpg",
+      "ground_truth": "\u7ca4B54321",
+      "conditions": {
+        "type": "clear"
+      },
+      "expected_behavior": "should_pass",
+      "critical": true,
+      "description": "??case"
+    },
+    {
+      "case_id": "val_002",
+      "image": "plate_029.jpg",
+      "ground_truth": "\u82cfD13579",
+      "conditions": {
+        "type": "noise",
+        "noise_level": 0.2
+      },
+      "expected_behavior": "should_fail_baseline",
+      "critical": false,
+      "description": "??+???"
+    },
+    {
+      "case_id": "val_003",
+      "image": "plate_018.jpg",
+      "ground_truth": "\u6d59C36912",
+      "conditions": {
+        "type": "blur",
+        "blur_kernel": 7
+      },
+      "expected_behavior": "should_fail_baseline",
+      "critical": false,
+      "description": "????"
+    }
+  ],
+  "stats": {
+    "total": 3,
+    "should_pass": 1,
+    "should_fail_baseline": 2,
+    "critical": 1
+  }
+}