fix(examples): parse sdk evalset expectations

Author: YAO-001

examples/optimization/eval_optimize_loop/eval_loop/backends.py

@@ -16,6 +16,7 @@
 from .fake_judge import FakeJudge
 from .fake_model import FakeModel
 from .loader import load_eval_cases
+from .loader import read_json
 from .optimizer import FakeOptimizer
 from .schemas import CandidatePrompt
 from .schemas import CaseResult
@@ -421,7 +422,7 @@ async def evaluate(
             target_paths,
             context=f"cannot evaluate {prompt_id}",
         )
-        expected_cases = load_eval_cases(dataset_path, split=split)
+        expected_cases = _load_sdk_expected_cases(dataset_path, split=split)
         snapshot = snapshot_prompt_files(target_paths)
         result: Any | None = None
         with temporary_prompt_bundle(snapshot, candidate_prompts):
@@ -447,7 +448,7 @@ async def evaluate(
             result,
             prompt_id=prompt_id,
             split=split,
-            expected_cases=expected_cases,
+            expected_cases=expected_cases.values(),
         )
 
     def _load_required_call_agent(self, *, for_evaluation: bool):
@@ -721,6 +722,138 @@ def _safe_jsonable(value: Any) -> Any:
     return repr(value)
 
 
+def _load_sdk_expected_cases(
+    dataset_path: str | Path,
+    *,
+    split: str,
+) -> dict[str, EvalCase]:
+    """Load wrapper metadata from an SDK EvalSet without changing the SDK file."""
+
+    payload = read_json(dataset_path)
+    if "eval_cases" not in payload:
+        if "cases" in payload:
+            return _eval_cases_by_id(
+                load_eval_cases(dataset_path, split=split),
+                context=f"legacy evalset {dataset_path}",
+            )
+        raise ValueError(f"SDK evalset {dataset_path} must contain an eval_cases list")
+
+    eval_set_id = payload.get("eval_set_id")
+    if not isinstance(eval_set_id, str) or not eval_set_id.strip():
+        raise ValueError(f"SDK evalset {dataset_path} is missing non-empty eval_set_id")
+    raw_cases = payload["eval_cases"]
+    if not isinstance(raw_cases, list):
+        raise ValueError(f"SDK evalset {dataset_path} eval_cases must be a list")
+
+    expected_cases: dict[str, EvalCase] = {}
+    for index, raw_case in enumerate(raw_cases):
+        if not isinstance(raw_case, dict):
+            raise ValueError(
+                f"SDK evalset {dataset_path} eval_cases[{index}] must be an object"
+            )
+        eval_id = raw_case.get("eval_id")
+        if not isinstance(eval_id, str) or not eval_id.strip():
+            raise ValueError(
+                f"SDK evalset {dataset_path} eval_cases[{index}] is missing non-empty eval_id"
+            )
+        if eval_id in expected_cases:
+            raise ValueError(f"SDK evalset {dataset_path} contains duplicate eval_id {eval_id!r}")
+        expected_cases[eval_id] = _expected_case_from_sdk_case(
+            raw_case,
+            eval_id=eval_id,
+            split=split,
+            dataset_path=dataset_path,
+        )
+    return expected_cases
+
+
+def _expected_case_from_sdk_case(
+    raw_case: dict[str, Any],
+    *,
+    eval_id: str,
+    split: str,
+    dataset_path: str | Path,
+) -> EvalCase:
+    context = f"SDK evalset {dataset_path} case {eval_id!r}"
+    conversation = raw_case.get("conversation")
+    if not isinstance(conversation, list) or not conversation:
+        raise ValueError(f"{context} must contain a non-empty conversation list")
+
+    input_text = ""
+    for turn_index, invocation in reversed(list(enumerate(conversation))):
+        if not isinstance(invocation, dict):
+            raise ValueError(f"{context} conversation[{turn_index}] must be an object")
+        user_content = invocation.get("user_content")
+        if user_content is None:
+            continue
+        if not isinstance(user_content, dict):
+            raise ValueError(
+                f"{context} conversation[{turn_index}].user_content must be an object"
+            )
+        candidate_text = _content_text(user_content)
+        if candidate_text.strip():
+            input_text = candidate_text
+            break
+    if not input_text:
+        raise ValueError(f"{context} conversation has no user_content text")
+
+    session_input = raw_case.get("session_input")
+    if not isinstance(session_input, dict):
+        raise ValueError(f"{context} must contain a session_input object")
+    state = session_input.get("state")
+    if not isinstance(state, dict):
+        raise ValueError(f"{context} session_input.state must be an object")
+    expectation = state.get("eval_optimize_expectation")
+    if not isinstance(expectation, dict):
+        raise ValueError(
+            f"{context} session_input.state must contain eval_optimize_expectation object"
+        )
+
+    tags = state.get("eval_optimize_tags", [])
+    if not isinstance(tags, list) or any(not isinstance(tag, str) for tag in tags):
+        raise ValueError(f"{context} eval_optimize_tags must be a list of strings")
+    protected = state.get("eval_optimize_protected", False)
+    if not isinstance(protected, bool):
+        raise ValueError(f"{context} eval_optimize_protected must be a boolean")
+
+    expected_failure_category = state.get("eval_optimize_expected_failure_category")
+    if expected_failure_category is None:
+        expected_failure_category = state.get("expected_failure_category")
+    if expected_failure_category is None:
+        expected_failure_category = expectation.get("expected_failure_category")
+    if (
+        expected_failure_category is not None
+        and (
+            not isinstance(expected_failure_category, str)
+            or not expected_failure_category.strip()
+        )
+    ):
+        raise ValueError(f"{context} expected_failure_category must be a non-empty string")
+
+    return EvalCase(
+        case_id=eval_id,
+        split=split,
+        input=input_text,
+        expectation=dict(expectation),
+        tags=list(tags),
+        protected=protected,
+        expected_failure_category=expected_failure_category,
+    )
+
+
+def _eval_cases_by_id(
+    cases: Iterable[EvalCase],
+    *,
+    context: str,
+) -> dict[str, EvalCase]:
+    by_id: dict[str, EvalCase] = {}
+    for case in cases:
+        if case.case_id in by_id:
+            raise ValueError(f"{context} contains duplicate case id {case.case_id!r}")
+        by_id[case.case_id] = case
+    return by_id
+
+
 def _eval_result_from_sdk_result(
     result: Any,
     *,

examples/optimization/eval_optimize_loop/tests/test_sdk_backend.py

@@ -328,24 +328,114 @@ def test_sdk_result_mapping_rejects_non_finite_metric_scores():
         )
 
 
+def test_sdk_expected_cases_parse_standard_evalset_metadata(tmp_path: Path):
+    dataset_path = tmp_path / "validation.evalset.json"
+    case_payload = _sdk_eval_case_payload(
+        "case-1",
+        query="query",
+        expected="expected",
+        tags=["x"],
+        protected=True,
+    )
+    case_payload["conversation"].insert(
+        0,
+        {
+            "invocation_id": "case-1-turn-0",
+            "user_content": {
+                "role": "user",
+                "parts": [{"text": "earlier query"}],
+            },
+            "final_response": {
+                "role": "model",
+                "parts": [{"text": "earlier expected"}],
+            },
+        },
+    )
+    dataset_path.write_text(
+        json.dumps(
+            _sdk_evalset_payload([case_payload])
+        ),
+        encoding="utf-8",
+    )
+
+    expected_cases = backend_module._load_sdk_expected_cases(
+        dataset_path,
+        split="validation",
+    )
+
+    assert set(expected_cases) == {"case-1"}
+    expected_case = expected_cases["case-1"]
+    assert expected_case.input == "query"
+    assert expected_case.expectation == {
+        "type": "exact",
+        "expected": "expected",
+        "expected_failure_category": "final_response_mismatch",
+    }
+    assert expected_case.tags == ["x"]
+    assert expected_case.protected is True
+    assert expected_case.expected_failure_category == "final_response_mismatch"
+    assert expected_case.split == "validation"
+
+
+def test_sdk_expected_cases_reject_duplicate_eval_ids(tmp_path: Path):
+    dataset_path = tmp_path / "duplicate.evalset.json"
+    dataset_path.write_text(
+        json.dumps(
+            _sdk_evalset_payload([
+                _sdk_eval_case_payload("case-1"),
+                _sdk_eval_case_payload("case-1"),
+            ])
+        ),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="duplicate.*case-1"):
+        backend_module._load_sdk_expected_cases(dataset_path, split="validation")
+
+
+def test_sdk_expected_cases_require_expectation_metadata(tmp_path: Path):
+    payload = _sdk_evalset_payload([_sdk_eval_case_payload("case-1")])
+    del payload["eval_cases"][0]["session_input"]["state"]["eval_optimize_expectation"]
+    dataset_path = tmp_path / "missing_expectation.evalset.json"
+    dataset_path.write_text(json.dumps(payload), encoding="utf-8")
+
+    with pytest.raises(ValueError, match="case-1.*eval_optimize_expectation"):
+        backend_module._load_sdk_expected_cases(dataset_path, split="validation")
+
+
+def test_sdk_expected_cases_reject_invalid_eval_cases_shape(tmp_path: Path):
+    dataset_path = tmp_path / "invalid_shape.evalset.json"
+    dataset_path.write_text(
+        json.dumps({"eval_set_id": "set", "eval_cases": {}}),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="eval_cases.*list"):
+        backend_module._load_sdk_expected_cases(dataset_path, split="validation")
+
+
 @pytest.mark.asyncio
 async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_bytes(
     tmp_path: Path,
     monkeypatch,
 ):
     dataset_path = tmp_path / "validation.evalset.json"
     dataset_path.write_text(
-        json.dumps({
-            "split": "validation",
-            "cases": [{
-                "case_id": "case_a",
-                "input": "question",
-                "expectation": {"answer": "answer"},
-                "expected_failure_category": "format_violation",
-            }],
-        }),
+        json.dumps(
+            _sdk_evalset_payload([
+                _sdk_eval_case_payload(
+                    "case_a",
+                    query="question",
+                    expected="answer",
+                    expected_failure_category="format_violation",
+                )
+            ])
+        ),
         encoding="utf-8",
     )
+    from trpc_agent_sdk.evaluation import EvalSet
+
+    assert EvalSet.model_validate_json(dataset_path.read_text(encoding="utf-8")).eval_cases[0].eval_id == "case_a"
     prompt_path = tmp_path / "prompt.txt"
     original_bytes = b"original prompt\r\n"
     prompt_path.write_bytes(original_bytes)
@@ -385,6 +475,7 @@ async def test_sdk_backend_evaluate_temporarily_installs_and_restores_prompt_byt
     assert calls["eval_result_output_dir"] == str(tmp_path / "sdk_eval")
     assert prompt_path.read_bytes() == original_bytes
     assert mapped.cases[0].trace_available is True
+    assert mapped.cases[0].expected_failure_category == "format_violation"
 
 
 def test_sdk_backend_requires_call_agent_path(tmp_path: Path):
@@ -1397,6 +1488,51 @@ def _sdk_evaluate_result(runs_by_case_id: dict[str, list[object]]):
     )
 
 
+def _sdk_evalset_payload(eval_cases: list[dict[str, object]]) -> dict[str, object]:
+    return {
+        "eval_set_id": "set",
+        "eval_cases": eval_cases,
+    }
+
+
+def _sdk_eval_case_payload(
+    eval_id: str,
+    *,
+    query: str = "query",
+    expected: str = "expected",
+    expected_failure_category: str = "final_response_mismatch",
+    tags: list[str] | None = None,
+    protected: bool = False,
+) -> dict[str, object]:
+    return {
+        "eval_id": eval_id,
+        "conversation": [{
+            "invocation_id": f"{eval_id}-turn-1",
+            "user_content": {
+                "role": "user",
+                "parts": [{"text": query}],
+            },
+            "final_response": {
+                "role": "model",
+                "parts": [{"text": expected}],
+            },
+        }],
+        "session_input": {
+            "app_name": "eval_optimize_loop",
+            "user_id": "test-user",
+            "state": {
+                "eval_optimize_expectation": {
+                    "type": "exact",
+                    "expected": expected,
+                    "expected_failure_category": expected_failure_category,
+                },
+                "eval_optimize_tags": list(tags or []),
+                "eval_optimize_protected": protected,
+            },
+        },
+    }
+
+
 def _install_fake_agent_evaluator(
     monkeypatch,
     *,