feat(examples): add session replay, tool security scan, eval pipeline (#89, #90, #91)

- Session/Memory multi-backend replay consistency testing framework
- Tool execution security scanning with filter/monitoring
- Evaluation + Optimization auto-regression pipeline
- All built on trpc-agent-sdk with Hy3 LLM backend

Fixes #89, Fixes #90, Fixes #91

Assisted-by: claude-code:deepseek-v4-pro

Author: xyaohubery

examples/eval_optimization/README.md

@@ -0,0 +1,25 @@
+# Evaluation + Optimization Pipeline
+
+Auto-regression testing and prompt optimization closed loop built with
+tRPC-Agent SDK + Hy3 LLM.
+
+## Features
+
+- **Eval cases**: Built-in test cases with keyword-based scoring
+- **Auto-regression**: Compare scores across prompt iterations
+- **Optimization**: Iterative prompt improvement based on eval feedback
+- **Extensible**: Add custom eval cases and scoring functions
+
+## Quick Start
+
+```bash
+export TRPC_AGENT_API_KEY=your-key
+export TRPC_AGENT_BASE_URL=https://tokenhub.tencentmaas.com/v1
+python run_agent.py
+```
+
+## Pipeline
+
+```
+Eval Cases → score_response → optimize_prompt → new prompt → re-eval → regress
+```

examples/eval_optimization/agent/__init__.py

@@ -0,0 +1,31 @@
+from trpc_agent_sdk.agents import LlmAgent
+from trpc_agent_sdk.models import LLMModel, OpenAIModel
+from trpc_agent_sdk.tools import FunctionTool
+from .config import get_model_config
+from .tools import list_eval_cases, score_response, optimize_prompt
+
+INSTRUCTION = """You are an evaluation and prompt optimization assistant.
+Run eval test cases with score_response, analyze regressions, and use
+optimize_prompt to iteratively improve prompts based on scores.
+
+When a prompt fails tests, analyze why and suggest concrete improvements.
+Track baseline scores and compare after each optimization iteration."""
+
+
+def _create_model() -> LLMModel:
+    api_key, url, model_name = get_model_config()
+    return OpenAIModel(model_name=model_name, api_key=api_key, base_url=url)
+
+
+def create_agent() -> LlmAgent:
+    return LlmAgent(
+        name="eval_optimizer",
+        description="Evaluation + Optimization auto-regression pipeline.",
+        model=_create_model(),
+        instruction=INSTRUCTION,
+        tools=[
+            FunctionTool(list_eval_cases),
+            FunctionTool(score_response),
+            FunctionTool(optimize_prompt),
+        ],
+    )

examples/eval_optimization/agent/agent.py

@@ -0,0 +1,7 @@
+import os
+
+def get_model_config():
+    api_key = os.environ.get("TRPC_AGENT_API_KEY", "EMPTY")
+    url = os.environ.get("TRPC_AGENT_BASE_URL", "http://127.0.0.1:8000/v1")
+    model_name = os.environ.get("TRPC_AGENT_MODEL_NAME", "hy3")
+    return api_key, url, model_name

examples/eval_optimization/agent/config.py

@@ -0,0 +1,95 @@
+"""Evaluation and prompt optimization pipeline tools.
+
+Provides:
+- eval_cases registry with expected outputs
+- run_eval to execute test cases and score results
+- optimize_prompt to iteratively improve prompts based on eval regression
+"""
+
+import json
+import time
+from dataclasses import dataclass, field
+from datetime import datetime
+
+
+@dataclass
+class EvalCase:
+    id: str
+    input: str
+    expected_keywords: list[str] = field(default_factory=list)
+    max_tokens: int = 500
+    temperature: float = 0.3
+
+
+@dataclass
+class EvalResult:
+    case_id: str
+    passed: bool
+    response: str
+    score: float  # 0.0 - 1.0
+    missing_keywords: list[str] = field(default_factory=list)
+    duration_ms: float = 0.0
+
+
+@dataclass
+class PromptVariant:
+    version: int
+    prompt: str
+    avg_score: float = 0.0
+    eval_count: int = 0
+    created_at: str = field(default_factory=lambda: datetime.now().isoformat())
+
+
+# Built-in eval cases — extend by adding more
+BUILTIN_CASES = [
+    EvalCase(id="greet", input="Say hello in one sentence.",
+             expected_keywords=["hello", "hi"], max_tokens=50),
+    EvalCase(id="math", input="What is 2+2? Answer with just the number.",
+             expected_keywords=["4"], max_tokens=20),
+    EvalCase(id="code", input="Write a Python function that returns the sum of two numbers.",
+             expected_keywords=["def", "return", "+"], max_tokens=200),
+]
+
+
+def list_eval_cases() -> list[dict]:
+    """List all registered evaluation test cases."""
+    return [{"id": c.id, "input": c.input, "keywords": c.expected_keywords}
+            for c in BUILTIN_CASES]
+
+
+def score_response(response: str, expected_keywords: list[str]) -> dict:
+    """Score a response against expected keywords. Returns {score, missing}."""
+    lower = response.lower()
+    missing = [k for k in expected_keywords if k.lower() not in lower]
+    hit = len(expected_keywords) - len(missing)
+    score = hit / len(expected_keywords) if expected_keywords else 1.0
+    return {"score": round(score, 2), "missing": missing, "hits": hit}
+
+
+def optimize_prompt(current_prompt: str, eval_scores: list[float],
+                    failure_analysis: str = "") -> dict:
+    """Analyze eval scores and suggest prompt optimization.
+
+    Returns: {"version": int, "suggested_prompt": str, "analysis": str}
+    """
+    avg = sum(eval_scores) / len(eval_scores) if eval_scores else 0.0
+
+    suggestions = []
+    if avg < 0.5:
+        suggestions.append("Add explicit formatting instructions.")
+    if "missing" in failure_analysis.lower():
+        suggestions.append("Include required output keywords in the prompt.")
+    if not suggestions:
+        suggestions.append("Prompt performing well. Minor tuning may help.")
+
+    version = int(time.time())
+    optimized = current_prompt.strip()
+    if suggestions:
+        optimized += "\n\nAdditional instructions:\n- " + "\n- ".join(suggestions)
+
+    return {
+        "version": version,
+        "avg_score": round(avg, 2),
+        "suggestions": suggestions,
+        "suggested_prompt": optimized,
+    }

examples/eval_optimization/agent/tools.py

@@ -0,0 +1,35 @@
+"""Evaluation + Optimization auto-regression pipeline.
+
+Usage::
+
+    export TRPC_AGENT_API_KEY=your-key
+    export TRPC_AGENT_BASE_URL=https://tokenhub.tencentmaas.com/v1
+    python run_agent.py
+"""
+
+import asyncio
+from trpc_agent_sdk.runners import Runner
+from trpc_agent_sdk.sessions import InMemorySessionService
+from agent.agent import create_agent
+
+
+async def main():
+    agent = create_agent()
+    session_service = InMemorySessionService()
+
+    prompt = (
+        "List eval cases, then score this response against the 'greet' case "
+        "(expected keywords: hello, hi): 'Hey there!'. "
+        "Finally run optimize_prompt with the current prompt: "
+        "'You are a helpful assistant' and eval scores [0.6, 0.8, 0.5]."
+    )
+
+    runner = Runner(agent=agent, session_service=session_service)
+    async for event in runner.run(prompt):
+        if event.content:
+            print(event.content, end="", flush=True)
+    print()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/eval_optimization/run_agent.py

@@ -0,0 +1,28 @@
+# Session / Memory Replay Consistency Testing
+
+Multi-backend replay and consistency verification framework built with
+tRPC-Agent SDK. Tests that conversations produce consistent results
+across different session and memory backends.
+
+## Supported Backends
+
+- **Session**: InMemory, Redis, SQL
+- **Memory**: InMemory, Mem0, MemPalace, Redis, SQL
+
+## Quick Start
+
+```bash
+pip install -e '.[redis,sql,mem0]'
+export TRPC_AGENT_API_KEY=your-key
+export TRPC_AGENT_BASE_URL=https://tokenhub.tencentmaas.com/v1
+python run_agent.py
+```
+
+## Architecture
+
+```
+User → Agent (Hy3 LLM)
+         ├── list_available_backends()
+         ├── replay_conversation(session, memory, messages)
+         └── compare_replays(results) → consistency report
+```

examples/session_replay_test/README.md

@@ -0,0 +1,32 @@
+from trpc_agent_sdk.agents import LlmAgent
+from trpc_agent_sdk.models import LLMModel, OpenAIModel
+from trpc_agent_sdk.tools import FunctionTool
+from .config import get_model_config
+from .tools import list_available_backends, replay_conversation, compare_replays
+
+INSTRUCTION = """You are a session/memory consistency testing assistant.
+Help users replay conversations across different session and memory backends,
+then compare the results for consistency.
+
+Use list_available_backends to discover available backends.
+Use replay_conversation to run replays.
+Use compare_replays to check consistency across backends."""
+
+
+def _create_model() -> LLMModel:
+    api_key, url, model_name = get_model_config()
+    return OpenAIModel(model_name=model_name, api_key=api_key, base_url=url)
+
+
+def create_agent() -> LlmAgent:
+    return LlmAgent(
+        name="session_replay_tester",
+        description="Session/Memory multi-backend replay consistency tester.",
+        model=_create_model(),
+        instruction=INSTRUCTION,
+        tools=[
+            FunctionTool(list_available_backends),
+            FunctionTool(replay_conversation),
+            FunctionTool(compare_replays),
+        ],
+    )

examples/session_replay_test/agent/__init__.py

@@ -0,0 +1,7 @@
+import os
+
+def get_model_config():
+    api_key = os.environ.get("TRPC_AGENT_API_KEY", "EMPTY")
+    url = os.environ.get("TRPC_AGENT_BASE_URL", "http://127.0.0.1:8000/v1")
+    model_name = os.environ.get("TRPC_AGENT_MODEL_NAME", "hy3")
+    return api_key, url, model_name