feat: Support creating evaluation runs from pre-existing Interactions API data

PiperOrigin-RevId: 944601748

Author: vertex-sdk-bot

agentplatform/_genai/_evals_common.py

@@ -420,6 +420,96 @@ def _extract_response_from_completed_trace(
     return event_dicts
 
 
+def _has_interactions_data_source(
+    eval_cases: list[types.EvalCase],
+) -> bool:
+    """Returns True if any EvalCase has interactions_data_source set."""
+    return any(
+        getattr(case, "interactions_data_source", None) is not None
+        for case in eval_cases
+    )
+
+
+def _resolve_interactions_to_eval_cases(
+    api_client: BaseApiClient,
+    eval_cases: list[types.EvalCase],
+) -> list[types.EvalCase]:
+    """Resolves EvalCases with interactions_data_source to agent_data.
+
+    For each EvalCase that has interactions_data_source set, fetches the
+    Interaction via the SDK's interactions.get() API, converts the steps
+    to AgentData, and returns a new EvalCase with agent_data populated.
+
+    Args:
+        api_client: The API client (must have an interactions module).
+        eval_cases: EvalCases with interactions_data_source set.
+
+    Returns:
+        New list of EvalCases with agent_data populated from resolved
+        interactions.
+
+    Raises:
+        ValueError: If eval_cases have missing interaction references.
+    """
+    # Validate all cases up front before making any API calls.
+    for case in eval_cases:
+        ids = case.interactions_data_source
+        if ids is None:
+            raise ValueError(
+                "All eval_cases must have interactions_data_source set when"
+                " using interaction resolution. Found a case without it. Do"
+                " not mix interaction-based and prompt-based eval cases."
+            )
+        if not ids.interaction:
+            raise ValueError(
+                "interactions_data_source.interaction is required. Each"
+                " EvalCase must reference an existing Interaction resource."
+            )
+
+    resolved_cases = []
+
+    for case in eval_cases:
+        ids = case.interactions_data_source
+
+        # Extract the interaction ID from the resource name.
+        # Format: projects/{p}/locations/{l}/interactions/{id}
+        parts = ids.interaction.split("/")
+        if len(parts) >= 6 and parts[4] == "interactions":
+            interaction_id = parts[5]
+        else:
+            interaction_id = ids.interaction
+
+        logger.info("Fetching interaction: %s", ids.interaction)
+        path = f"interactions/{interaction_id}"
+        response = api_client.request("get", path, {}, None)
+        interaction_dict = (
+            {} if not response.body else json.loads(response.body)
+        )
+
+        agent_data = _interaction_dict_to_agent_data(interaction_dict)
+
+        # Best-effort: fetch the agent config (instruction, tools,
+        # description) from the Agent API so the display can render
+        # the System Topology section.
+        gemini_cfg = getattr(ids, "gemini_agent_config", None)
+        agent_name = (
+            getattr(gemini_cfg, "gemini_agent", None)
+            if gemini_cfg else None
+        )
+        agent_config = _fetch_agent_config_dict(
+            api_client, agent_name or ""
+        )
+        agent_data.agents = {agent_config.agent_id: agent_config}
+
+        # Merge consecutive text events and parts so multi-paragraph
+        # responses render as a single block in the trace display.
+        _merge_text_parts_in_agent_data(agent_data)
+
+        resolved_cases.append(types.EvalCase(agent_data=agent_data))
+
+    return resolved_cases
+
+
 def _resolve_dataset(
     api_client: BaseApiClient,
     dataset: Union[types.EvaluationRunDataSource, types.EvaluationDataset],
@@ -428,6 +518,15 @@ def _resolve_dataset(
 ) -> types.EvaluationRunDataSource:
     """Resolves dataset for the evaluation run."""
     if isinstance(dataset, types.EvaluationDataset):
+        # Resolve EvalCases with interactions_data_source by fetching
+        # each interaction and converting it to agent_data, then flowing
+        # through the normal DataFrame/GCS pipeline.
+        if dataset.eval_cases and _has_interactions_data_source(dataset.eval_cases):
+            resolved_cases = _resolve_interactions_to_eval_cases(
+                api_client, dataset.eval_cases
+            )
+            dataset = types.EvaluationDataset(eval_cases=resolved_cases)
+
         candidate_name = _get_candidate_name(dataset, parsed_agent_info)
         eval_df = dataset.eval_dataset_df
         if eval_df is None and dataset.eval_cases:

tests/unit/agentplatform/genai/test_evals.py

@@ -9563,6 +9563,133 @@ def test_resolve_dataset_preserves_conversation_history(
         assert "conversation_history" in ptd_values
 
 
+class TestResolveDatasetWithInteractions:
+    """Tests for resolving interactions_data_source in _resolve_dataset."""
+
+    def setup_method(self):
+        self.mock_api_client = mock.Mock()
+        self.mock_api_client.project = "test-project"
+        self.mock_api_client.location = "us-central1"
+
+    def test_has_interactions_data_source_true(self):
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig(
+                        gemini_agent="projects/p/locations/l/agents/a"
+                    ),
+                    interaction="projects/p/locations/l/interactions/i1",
+                )
+            )
+        ]
+        assert _evals_common._has_interactions_data_source(cases)
+
+    def test_has_interactions_data_source_false(self):
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                prompt=genai_types.Content(
+                    parts=[genai_types.Part(text="test")]
+                ),
+            )
+        ]
+        assert not _evals_common._has_interactions_data_source(cases)
+
+    def test_resolve_rejects_mixed_cases(self):
+        """Mixing interaction-based and prompt-based cases raises ValueError."""
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    interaction="projects/p/locations/l/interactions/i1",
+                )
+            ),
+            agentplatform_genai_types.EvalCase(
+                prompt=genai_types.Content(
+                    parts=[genai_types.Part(text="test")]
+                ),
+            ),
+        ]
+        with pytest.raises(ValueError, match="interactions_data_source"):
+            _evals_common._resolve_interactions_to_eval_cases(
+                self.mock_api_client, cases
+            )
+
+    def test_resolve_rejects_missing_interaction(self):
+        """EvalCase with interactions_data_source but no interaction raises."""
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig(
+                        gemini_agent="projects/p/locations/l/agents/a"
+                    ),
+                )
+            ),
+        ]
+        with pytest.raises(ValueError, match="interaction is required"):
+            _evals_common._resolve_interactions_to_eval_cases(
+                self.mock_api_client, cases
+            )
+
+    def test_interaction_dict_to_agent_data_text_conversation(self):
+        """Converts user_input + model_output steps to agent_data."""
+        interaction_dict = {
+            "steps": [
+                {
+                    "type": "user_input",
+                    "content": [{"type": "text", "text": "Hello agent"}],
+                },
+                {
+                    "type": "model_output",
+                    "content": [
+                        {"type": "text", "text": "Hello! How can I help?"}
+                    ],
+                },
+            ]
+        }
+
+        result = _evals_common._interaction_dict_to_agent_data(
+            interaction_dict
+        )
+
+        assert len(result.turns) == 1
+        events = result.turns[0].events
+        assert len(events) == 2
+        assert events[0].author == "user"
+        assert events[0].content.parts[0].text == "Hello agent"
+        assert events[1].author == "agent"
+        assert events[1].content.parts[0].text == "Hello! How can I help?"
+
+    def test_interaction_dict_to_agent_data_with_tool_calls(self):
+        """Converts function_call + function_result steps."""
+        interaction_dict = {
+            "steps": [
+                {
+                    "type": "function_call",
+                    "name": "get_weather",
+                    "arguments": {"city": "NYC"},
+                    "id": "call_1",
+                },
+                {
+                    "type": "function_result",
+                    "name": "get_weather",
+                    "callId": "call_1",
+                    "result": {"temp": "72F"},
+                },
+            ]
+        }
+
+        result = _evals_common._interaction_dict_to_agent_data(
+            interaction_dict
+        )
+
+        events = result.turns[0].events
+        assert len(events) == 2
+        fc_event = events[0]
+        assert fc_event.author == "agent"
+        assert fc_event.content.parts[0].function_call.name == "get_weather"
+        fr_event = events[1]
+        assert fr_event.content.parts[0].function_response.id == "call_1"
+
+
 class TestRateLimiter:
     """Tests for the RateLimiter class in _evals_utils."""