fix(evals): normalize multi-turn agent responses

Author: PSR94

agentplatform/_genai/_evals_common.py

@@ -1958,6 +1958,53 @@ def _is_multi_turn_agent_simulation(
     )
 
 
+def _normalize_agent_event(event: Any) -> Any:
+    """Normalizes raw agent event dictionaries for AgentEvent validation."""
+    if not isinstance(event, dict):
+        return event
+    return {
+        key: value
+        for key, value in event.items()
+        if key in {"author", "content", "event_time", "state_delta", "active_tools"}
+    }
+
+
+def _normalize_conversation_turns(resp_item: Any) -> Any:
+    """Normalizes raw multi-turn responses for ConversationTurn validation."""
+    if not isinstance(resp_item, list):
+        return resp_item
+
+    turns = []
+    for turn_index, turn in enumerate(resp_item):
+        if not isinstance(turn, dict):
+            turns.append(turn)
+            continue
+
+        normalized_turn = {
+            key: value
+            for key, value in turn.items()
+            if key in {"turn_index", "turn_id", "events"}
+        }
+        if normalized_turn.get("turn_index") is None:
+            normalized_turn["turn_index"] = turn_index
+        if normalized_turn.get("turn_id") is None:
+            turn_id = turn.get("turn_id") or turn.get("id") or turn.get("invocation_id")
+            if turn_id is not None:
+                normalized_turn["turn_id"] = turn_id
+
+        if "events" in normalized_turn and normalized_turn["events"] is not None:
+            normalized_turn["events"] = [
+                _normalize_agent_event(event) for event in normalized_turn["events"]
+            ]
+        else:
+            event = _normalize_agent_event(turn)
+            if event:
+                normalized_turn["events"] = [event]
+
+        turns.append(normalized_turn)
+    return turns
+
+
 def _process_multi_turn_agent_response(
     resp_item: Any,
     agent_data_agents: Optional[dict[str, Any]],
@@ -1966,7 +2013,7 @@ def _process_multi_turn_agent_response(
     if isinstance(resp_item, dict) and "error" in resp_item:
         return json.dumps(resp_item)
     return types.evals.AgentData(
-        turns=resp_item,
+        turns=_normalize_conversation_turns(resp_item),
         agents=agent_data_agents,
     ).model_dump(exclude_unset=True)
 

tests/unit/agentplatform/genai/test_evals.py

@@ -4438,6 +4438,61 @@ def test_run_agent_internal_multi_turn_success(self, mock_run_agent):
             {"turn_index": 1, "turn_id": "t2", "events": []},
         ]
 
+    @mock.patch.object(_evals_common, "_run_agent")
+    def test_run_agent_internal_multi_turn_normalizes_raw_events(self, mock_run_agent):
+        mock_run_agent.return_value = [
+            [
+                {
+                    "id": "event-1",
+                    "author": "agent",
+                    "content": {"parts": [{"text": "hello"}]},
+                    "model_version": "gemini-2.5-flash",
+                    "timestamp": "2026-01-01T00:00:00Z",
+                },
+                {
+                    "invocation_id": "event-2",
+                    "author": "agent",
+                    "content": {"parts": [{"text": "bye"}]},
+                    "actions": {"state_delta": {}},
+                    "usage_metadata": {"total_token_count": 12},
+                },
+            ]
+        ]
+        prompt_dataset = pd.DataFrame({"prompt": ["p1"], "conversation_plan": ["plan"]})
+        mock_agent_engine = mock.Mock()
+        mock_api_client = mock.Mock()
+
+        result_df = _evals_common._run_agent_internal(
+            api_client=mock_api_client,
+            agent_engine=mock_agent_engine,
+            agent=None,
+            prompt_dataset=prompt_dataset,
+        )
+
+        agent_data = result_df["agent_data"][0]
+        assert agent_data["turns"] == [
+            {
+                "turn_index": 0,
+                "turn_id": "event-1",
+                "events": [
+                    {
+                        "author": "agent",
+                        "content": {"parts": [{"text": "hello"}]},
+                    }
+                ],
+            },
+            {
+                "turn_index": 1,
+                "turn_id": "event-2",
+                "events": [
+                    {
+                        "author": "agent",
+                        "content": {"parts": [{"text": "bye"}]},
+                    }
+                ],
+            },
+        ]
+
     @mock.patch.object(_evals_common, "_run_agent")
     def test_run_agent_internal_multi_turn_with_agent(self, mock_run_agent):
         mock_run_agent.return_value = [
@@ -6578,9 +6633,7 @@ def test_groundedness_aliases_grounding_v1(self):
         assert lazy_metric._get_api_metric_spec_name() == "grounding_v1"
 
     def test_groundedness_logs_field_difference_warning(self, caplog):
-        loader_logger = (
-            "agentplatform._genai._evals_metric_loaders"
-        )
+        loader_logger = "agentplatform._genai._evals_metric_loaders"
         with caplog.at_level("WARNING", logger=loader_logger):
             _ = agentplatform_genai_types.RubricMetric.GROUNDEDNESS
         messages = [r.getMessage() for r in caplog.records if r.name == loader_logger]

vertexai/_genai/_evals_common.py

@@ -1939,6 +1939,53 @@ def _is_multi_turn_agent_simulation(
     )
 
 
+def _normalize_agent_event(event: Any) -> Any:
+    """Normalizes raw agent event dictionaries for AgentEvent validation."""
+    if not isinstance(event, dict):
+        return event
+    return {
+        key: value
+        for key, value in event.items()
+        if key in {"author", "content", "event_time", "state_delta", "active_tools"}
+    }
+
+
+def _normalize_conversation_turns(resp_item: Any) -> Any:
+    """Normalizes raw multi-turn responses for ConversationTurn validation."""
+    if not isinstance(resp_item, list):
+        return resp_item
+
+    turns = []
+    for turn_index, turn in enumerate(resp_item):
+        if not isinstance(turn, dict):
+            turns.append(turn)
+            continue
+
+        normalized_turn = {
+            key: value
+            for key, value in turn.items()
+            if key in {"turn_index", "turn_id", "events"}
+        }
+        if normalized_turn.get("turn_index") is None:
+            normalized_turn["turn_index"] = turn_index
+        if normalized_turn.get("turn_id") is None:
+            turn_id = turn.get("turn_id") or turn.get("id") or turn.get("invocation_id")
+            if turn_id is not None:
+                normalized_turn["turn_id"] = turn_id
+
+        if "events" in normalized_turn and normalized_turn["events"] is not None:
+            normalized_turn["events"] = [
+                _normalize_agent_event(event) for event in normalized_turn["events"]
+            ]
+        else:
+            event = _normalize_agent_event(turn)
+            if event:
+                normalized_turn["events"] = [event]
+
+        turns.append(normalized_turn)
+    return turns
+
+
 def _process_multi_turn_agent_response(
     resp_item: Any,
     agent_data_agents: Optional[dict[str, Any]],
@@ -1947,7 +1994,7 @@ def _process_multi_turn_agent_response(
     if isinstance(resp_item, dict) and "error" in resp_item:
         return json.dumps(resp_item)
     return types.evals.AgentData(
-        turns=resp_item,
+        turns=_normalize_conversation_turns(resp_item),
         agents=agent_data_agents,
     ).model_dump(exclude_unset=True)