feat: Support creating evaluation runs from pre-existing Interactions API data

PiperOrigin-RevId: 944601748

Author: vertex-sdk-bot

agentplatform/_genai/_evals_common.py

@@ -433,6 +433,15 @@ def _resolve_dataset(
 ) -> types.EvaluationRunDataSource:
     """Resolves dataset for the evaluation run."""
     if isinstance(dataset, types.EvaluationDataset):
+        # Resolve EvalCases with interactions_data_source by fetching
+        # each interaction and converting it to agent_data, then flowing
+        # through the normal DataFrame/GCS pipeline.
+        if dataset.eval_cases and _has_interactions_data_source(dataset.eval_cases):
+            resolved_cases = _resolve_interactions_to_eval_cases(
+                api_client, dataset.eval_cases
+            )
+            dataset = types.EvaluationDataset(eval_cases=resolved_cases)
+
         candidate_name = _get_candidate_name(dataset, parsed_agent_info)
         eval_df = dataset.eval_dataset_df
         if eval_df is None and dataset.eval_cases:
@@ -923,6 +932,92 @@ def _fetch_agent_config_dict(
     )
 
 
+def _has_interactions_data_source(
+    eval_cases: list[types.EvalCase],
+) -> bool:
+    """Returns True if any EvalCase has interactions_data_source set."""
+    return any(case.interactions_data_source is not None for case in eval_cases)
+
+
+def _resolve_interactions_to_eval_cases(
+    api_client: BaseApiClient,
+    eval_cases: list[types.EvalCase],
+) -> list[types.EvalCase]:
+    """Resolves EvalCases with interactions_data_source to agent_data.
+
+    For each EvalCase that has interactions_data_source set, fetches the
+    Interaction via the SDK's interactions.get() API, converts the steps
+    to AgentData, and returns a new EvalCase with agent_data populated.
+
+    Args:
+        api_client: The API client (must have an interactions module).
+        eval_cases: EvalCases with interactions_data_source set.
+
+    Returns:
+        New list of EvalCases with agent_data populated from resolved
+        interactions.
+
+    Raises:
+        ValueError: If eval_cases have missing interaction references.
+    """
+    # Validate all cases up front before making any API calls.
+    for case in eval_cases:
+        ids = case.interactions_data_source
+        if ids is None:
+            raise ValueError(
+                "All eval_cases must have interactions_data_source set when"
+                " using interaction resolution. Found a case without it. Do"
+                " not mix interaction-based and prompt-based eval cases."
+            )
+        if not ids.interaction:
+            raise ValueError(
+                "interactions_data_source.interaction is required. Each"
+                " EvalCase must reference an existing Interaction resource."
+            )
+
+    resolved_cases = []
+
+    for case in eval_cases:
+        ids = case.interactions_data_source
+
+        # Extract the interaction short ID from the resource name.
+        # Handles both full resource names (projects/.../interactions/{id})
+        # and bare IDs by always taking the last path component.
+        interaction_id = ids.interaction.split("/")[-1]
+
+        logger.info("Fetching interaction: %s", ids.interaction)
+        path = f"interactions/{interaction_id}"
+        response = api_client.request("get", path, {}, None)
+        interaction_dict = {} if not response.body else json.loads(response.body)
+
+        agent_data = _interaction_dict_to_agent_data(interaction_dict)
+
+        # Best-effort: fetch the agent config (instruction, tools,
+        # description) from the Agent API so the display can render
+        # the System Topology section.
+        gemini_cfg = ids.gemini_agent_config
+        agent_name = gemini_cfg.gemini_agent if gemini_cfg else None
+        agent_config = _fetch_agent_config_dict(api_client, agent_name or "")
+        agent_data.agents = {agent_config.agent_id: agent_config}
+
+        # Merge consecutive text events and parts so multi-paragraph
+        # responses render as a single block in the trace display.
+        _merge_text_parts_in_agent_data(agent_data)
+
+        # Preserve all original EvalCase fields; only update agent_data
+        # and clear the now-resolved interactions_data_source.
+        resolved_cases.append(
+            case.model_copy(
+                update={
+                    "agent_data": agent_data,
+                    "interactions_data_source": None,
+                }
+            )
+        )
+
+    return resolved_cases
+
+
 def _add_evaluation_run_labels(
     labels: Optional[dict[str, str]] = None,
     agent: Optional[str] = None,

tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py

@@ -709,6 +709,42 @@ def test_create_eval_run_with_metric_resource_name(mock_uuid4, client):
 #             == INPUT_DF_WITH_CONTEXT_AND_HISTORY.iloc[i]["response"]
 #         )
 #     assert evaluation_run.error is None
+@mock.patch("uuid.uuid4")
+def test_create_eval_run_with_interactions_data_source(mock_uuid4, client):
+    """Tests create_evaluation_run() with EvalCases using interactions_data_source.
+
+    The SDK resolves each interaction client-side (GET interactions/{id} and
+    GET agents/{id}) before uploading the EvalCases to the evaluation pipeline.
+    """
+    mock_uuid4.return_value = uuid.UUID("a1b2c3d4-e5f6-7890-abcd-ef1234567890")
+    client._api_client._http_options.api_version = "v1beta1"
+    interaction_id = "ChA2YzllYzk0MjY1NjZjODM5EAgaATAqBG1haW4"
+    gemini_agent = (
+        "projects/model-evaluation-dev/locations/global/agents/test-agent-eval"
+    )
+    eval_case = types.EvalCase(
+        interactions_data_source=types.InteractionsDataSource(
+            interaction=(
+                f"projects/model-evaluation-dev/locations/global"
+                f"/interactions/{interaction_id}"
+            ),
+            gemini_agent_config=types.GeminiAgentConfig(
+                gemini_agent=gemini_agent,
+            ),
+        )
+    )
+    evaluation_run = client.evals.create_evaluation_run(
+        name="test_interactions_data_source",
+        display_name="test_interactions_data_source",
+        dataset=types.EvaluationDataset(eval_cases=[eval_case]),
+        dest=GCS_DEST,
+        metrics=[GENERAL_QUALITY_METRIC],
+    )
+    assert isinstance(evaluation_run, types.EvaluationRun)
+    assert evaluation_run.state == types.EvaluationRunState.PENDING
+    assert evaluation_run.error is None
+
+
 def test_create_eval_run_with_red_teaming_config(client):
     """Tests that create_evaluation_run() with red_teaming_config sends analysisConfigs."""
     evaluation_run = client.evals.create_evaluation_run(

tests/unit/agentplatform/genai/test_evals.py

@@ -9563,6 +9563,230 @@ def test_resolve_dataset_preserves_conversation_history(
         assert "conversation_history" in ptd_values
 
 
+class TestResolveDatasetWithInteractions:
+    """Tests for resolving interactions_data_source in _resolve_dataset."""
+
+    def setup_method(self):
+        self.mock_api_client = mock.Mock()
+        self.mock_api_client.project = "test-project"
+        self.mock_api_client.location = "us-central1"
+
+    def test_has_interactions_data_source_true(self):
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig(
+                        gemini_agent="projects/p/locations/l/agents/a"
+                    ),
+                    interaction="projects/p/locations/l/interactions/i1",
+                )
+            )
+        ]
+        assert _evals_common._has_interactions_data_source(cases)
+
+    def test_has_interactions_data_source_false(self):
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                prompt=genai_types.Content(
+                    parts=[genai_types.Part(text="test")]
+                ),
+            )
+        ]
+        assert not _evals_common._has_interactions_data_source(cases)
+
+    def test_resolve_rejects_mixed_cases(self):
+        """Mixing interaction-based and prompt-based cases raises ValueError."""
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    interaction="projects/p/locations/l/interactions/i1",
+                )
+            ),
+            agentplatform_genai_types.EvalCase(
+                prompt=genai_types.Content(
+                    parts=[genai_types.Part(text="test")]
+                ),
+            ),
+        ]
+        with pytest.raises(ValueError, match="interactions_data_source"):
+            _evals_common._resolve_interactions_to_eval_cases(
+                self.mock_api_client, cases
+            )
+
+    def test_resolve_rejects_missing_interaction(self):
+        """EvalCase with interactions_data_source but no interaction raises."""
+        cases = [
+            agentplatform_genai_types.EvalCase(
+                interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                    gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig(
+                        gemini_agent="projects/p/locations/l/agents/a"
+                    ),
+                )
+            ),
+        ]
+        with pytest.raises(ValueError, match="interaction is required"):
+            _evals_common._resolve_interactions_to_eval_cases(
+                self.mock_api_client, cases
+            )
+
+    def test_interaction_dict_to_agent_data_text_conversation(self):
+        """Converts user_input + model_output steps to agent_data."""
+        interaction_dict = {
+            "status": "completed",
+            "steps": [
+                {
+                    "type": "user_input",
+                    "content": [{"type": "text", "text": "Hello agent"}],
+                },
+                {
+                    "type": "model_output",
+                    "content": [
+                        {"type": "text", "text": "Hello! How can I help?"}
+                    ],
+                },
+            ]
+        }
+
+        result = _evals_common._interaction_dict_to_agent_data(
+            interaction_dict
+        )
+
+        assert len(result.turns) == 1
+        events = result.turns[0].events
+        assert len(events) == 2
+        assert events[0].author == "user"
+        assert events[0].content.parts[0].text == "Hello agent"
+        assert events[1].author == "agent"
+        assert events[1].content.parts[0].text == "Hello! How can I help?"
+
+    def test_interaction_dict_to_agent_data_with_tool_calls(self):
+        """Converts function_call + function_result steps."""
+        interaction_dict = {
+            "status": "completed",
+            "steps": [
+                {
+                    "type": "function_call",
+                    "name": "get_weather",
+                    "arguments": {"city": "NYC"},
+                    "id": "call_1",
+                },
+                {
+                    "type": "function_result",
+                    "name": "get_weather",
+                    "call_id": "call_1",
+                    "result": {"temp": "72F"},
+                },
+            ]
+        }
+
+        result = _evals_common._interaction_dict_to_agent_data(
+            interaction_dict
+        )
+
+        events = result.turns[0].events
+        assert len(events) == 2
+        fc_event = events[0]
+        assert fc_event.author == "agent"
+        assert fc_event.content.parts[0].function_call.name == "get_weather"
+        fr_event = events[1]
+        assert fr_event.content.parts[0].function_response.id == "call_1"
+
+    def test_resolve_interactions_to_eval_cases_happy_path(self):
+        """Fetches an interaction and populates agent_data on the EvalCase."""
+        interaction_body = json.dumps({
+            "status": "completed",
+            "steps": [
+                {
+                    "type": "user_input",
+                    "content": [{"type": "text", "text": "What is the weather?"}],
+                },
+                {
+                    "type": "model_output",
+                    "content": [{"type": "text", "text": "It is sunny."}],
+                },
+            ]
+        })
+        agent_body = json.dumps({
+            "system_instruction": "You are helpful.",
+            "base_agent": "gemini-2.0-flash",
+        })
+
+        def mock_request(method, path, *args, **kwargs):
+            resp = mock.MagicMock()
+            if path.startswith("interactions/"):
+                resp.body = interaction_body
+            elif path.startswith("agents/"):
+                resp.body = agent_body
+            else:
+                resp.body = None
+            return resp
+
+        self.mock_api_client.request.side_effect = mock_request
+
+        original_case = agentplatform_genai_types.EvalCase(
+            interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig(
+                    gemini_agent="projects/p/locations/l/agents/my-agent",
+                ),
+                interaction="projects/p/locations/l/interactions/i1",
+            ),
+        )
+
+        resolved = _evals_common._resolve_interactions_to_eval_cases(
+            self.mock_api_client, [original_case]
+        )
+
+        assert len(resolved) == 1
+        result_case = resolved[0]
+
+        # interactions_data_source should be cleared after resolution.
+        assert result_case.interactions_data_source is None
+
+        # agent_data should be populated with the interaction content.
+        agent_data = result_case.agent_data
+        assert agent_data is not None
+        assert len(agent_data.turns) == 1
+        events = agent_data.turns[0].events
+        assert len(events) == 2
+        assert events[0].author == "user"
+        assert events[0].content.parts[0].text == "What is the weather?"
+        assert events[1].author == "agent"
+        assert events[1].content.parts[0].text == "It is sunny."
+
+        # agents map should be populated with config from the Agent API.
+        assert "my-agent" in agent_data.agents
+        agent_config = agent_data.agents["my-agent"]
+        assert agent_config.agent_id == "my-agent"
+        assert agent_config.instruction == "You are helpful."
+        assert agent_config.agent_type == "gemini-2.0-flash"
+
+    def test_resolve_preserves_original_eval_case_fields(self):
+        """Fields other than agent_data are preserved on the resolved EvalCase."""
+        interaction_body = json.dumps({"status": "completed", "steps": []})
+
+        def mock_request(method, path, *args, **kwargs):
+            resp = mock.MagicMock()
+            resp.body = interaction_body
+            return resp
+
+        self.mock_api_client.request.side_effect = mock_request
+
+        original_case = agentplatform_genai_types.EvalCase(
+            interactions_data_source=agentplatform_genai_types.InteractionsDataSource(
+                interaction="projects/p/locations/l/interactions/i1",
+            ),
+        )
+
+        resolved = _evals_common._resolve_interactions_to_eval_cases(
+            self.mock_api_client, [original_case]
+        )
+
+        assert len(resolved) == 1
+        # agent_data is populated and interactions_data_source cleared.
+        assert resolved[0].agent_data is not None
+        assert resolved[0].interactions_data_source is None
+
+
 class TestRateLimiter:
     """Tests for the RateLimiter class in _evals_utils."""