feat: Support creating evaluation runs from pre-existing Interactions API data

PiperOrigin-RevId: 944601748

Author: vertex-sdk-bot

agentplatform/_genai/_evals_common.py

@@ -54,6 +54,8 @@
 
 logger = logging.getLogger(__name__)
 
+_MAX_INTERACTION_CHAIN_DEPTH = 10
+
 try:
     import litellm
 except ImportError:
@@ -433,6 +435,15 @@ def _resolve_dataset(
 ) -> types.EvaluationRunDataSource:
     """Resolves dataset for the evaluation run."""
     if isinstance(dataset, types.EvaluationDataset):
+        # Resolve EvalCases with interactions_data_source by fetching
+        # each interaction and converting it to agent_data, then flowing
+        # through the normal DataFrame/GCS pipeline.
+        if dataset.eval_cases and _has_interactions_data_source(dataset.eval_cases):
+            resolved_cases = _resolve_interactions_to_eval_cases(
+                api_client, dataset.eval_cases
+            )
+            dataset = types.EvaluationDataset(eval_cases=resolved_cases)
+
         candidate_name = _get_candidate_name(dataset, parsed_agent_info)
         eval_df = dataset.eval_dataset_df
         if eval_df is None and dataset.eval_cases:
@@ -1007,6 +1018,127 @@ def _build_interaction_id_dataset(
     return types.EvaluationDataset(eval_cases=eval_cases)
 
 
+def _has_interactions_data_source(
+    eval_cases: list[types.EvalCase],
+) -> bool:
+    """Returns True if any EvalCase has interactions_data_source set."""
+    return any(case.interactions_data_source is not None for case in eval_cases)
+
+
+def _resolve_interactions_to_eval_cases(
+    api_client: BaseApiClient,
+    eval_cases: list[types.EvalCase],
+) -> list[types.EvalCase]:
+    """Resolves EvalCases with interactions_data_source to agent_data.
+
+    For each EvalCase that has interactions_data_source set, fetches the
+    Interaction via the SDK's interactions.get() API, converts the steps
+    to AgentData, and returns a new EvalCase with agent_data populated.
+
+    Args:
+        api_client: The API client (must have an interactions module).
+        eval_cases: EvalCases with interactions_data_source set.
+
+    Returns:
+        New list of EvalCases with agent_data populated from resolved
+        interactions.
+
+    Raises:
+        ValueError: If eval_cases have missing interaction references.
+    """
+    # Validate all cases up front before making any API calls.
+    for case in eval_cases:
+        ids = case.interactions_data_source
+        if ids is None:
+            raise ValueError(
+                "All eval_cases must have interactions_data_source set when"
+                " using interaction resolution. Found a case without it. Do"
+                " not mix interaction-based and prompt-based eval cases."
+            )
+        if not ids.interaction:
+            raise ValueError(
+                "interactions_data_source.interaction is required. Each"
+                " EvalCase must reference an existing Interaction resource."
+            )
+
+    resolved_cases = []
+
+    for case in eval_cases:
+        ids = case.interactions_data_source
+
+        # Extract the interaction short ID from the resource name.
+        # Handles both full resource names (projects/.../interactions/{id})
+        # and bare IDs by always taking the last path component.
+        interaction_id = ids.interaction.split("/")[-1]
+
+        logger.info("Fetching interaction: %s", ids.interaction)
+
+        current_interaction_id = interaction_id
+        interactions = []
+        seen_ids = set()
+        for _ in range(_MAX_INTERACTION_CHAIN_DEPTH):
+            if current_interaction_id in seen_ids:
+                break
+            seen_ids.add(current_interaction_id)
+            path = f"interactions/{current_interaction_id}"
+            response = api_client.request("get", path, {}, None)
+            if not response.body:
+                if not interactions:
+                    logger.warning(
+                        "Empty response fetching interaction %s.",
+                        ids.interaction,
+                    )
+                break
+            interaction_dict = json.loads(response.body)
+            try:
+                typed_interaction = interaction_types.Interaction.model_validate(interaction_dict)
+            except Exception as e:
+                logger.warning("Failed to validate interaction model: %s", e)
+                break
+
+            interactions.append(typed_interaction)
+            if not typed_interaction.previous_interaction_id:
+                break
+            current_interaction_id = typed_interaction.previous_interaction_id.split("/")[-1]
+
+        if not interactions:
+            agent_data = types.evals.AgentData(turns=[]) # Fallback
+        else:
+            interactions.reverse() # chronological order
+            all_steps = []
+            for i_typed in interactions:
+                all_steps.extend(i_typed.steps or [])
+
+            combined_interaction = interactions[-1].model_dump()
+            combined_interaction["steps"] = all_steps
+            agent_data = _interaction_dict_to_agent_data(combined_interaction)
+
+        # Best-effort: fetch the agent config (instruction, tools,
+        # description) from the Agent API so the display can render
+        # the System Topology section.
+        gemini_cfg = ids.gemini_agent_config
+        agent_name = gemini_cfg.gemini_agent if gemini_cfg else None
+        agent_config = _fetch_agent_config_dict(api_client, agent_name or "")
+        agent_data.agents = {agent_config.agent_id: agent_config}
+
+        # Merge consecutive text events and parts so multi-paragraph
+        # responses render as a single block in the trace display.
+        _merge_text_parts_in_agent_data(agent_data)
+
+        # Preserve all original EvalCase fields; only update agent_data
+        # and clear the now-resolved interactions_data_source.
+        resolved_cases.append(
+            case.model_copy(
+                update={
+                    "agent_data": agent_data,
+                    "interactions_data_source": None,
+                }
+            )
+        )
+
+    return resolved_cases
+
+
 def _add_evaluation_run_labels(
     labels: Optional[dict[str, str]] = None,
     agent: Optional[str] = None,

tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py

@@ -709,6 +709,42 @@ def test_create_eval_run_with_metric_resource_name(mock_uuid4, client):
 #             == INPUT_DF_WITH_CONTEXT_AND_HISTORY.iloc[i]["response"]
 #         )
 #     assert evaluation_run.error is None
+@mock.patch("uuid.uuid4")
+def test_create_eval_run_with_interactions_data_source(mock_uuid4, client):
+    """Tests create_evaluation_run() with EvalCases using interactions_data_source.
+
+    The SDK resolves each interaction client-side (GET interactions/{id} and
+    GET agents/{id}) before uploading the EvalCases to the evaluation pipeline.
+    """
+    mock_uuid4.return_value = uuid.UUID("a1b2c3d4-e5f6-7890-abcd-ef1234567890")
+    client._api_client._http_options.api_version = "v1beta1"
+    interaction_id = "ChA2YzllYzk0MjY1NjZjODM5EAgaATAqBG1haW4"
+    gemini_agent = (
+        "projects/model-evaluation-dev/locations/global/agents/test-agent-eval"
+    )
+    eval_case = types.EvalCase(
+        interactions_data_source=types.InteractionsDataSource(
+            interaction=(
+                f"projects/model-evaluation-dev/locations/global"
+                f"/interactions/{interaction_id}"
+            ),
+            gemini_agent_config=types.GeminiAgentConfig(
+                gemini_agent=gemini_agent,
+            ),
+        )
+    )
+    evaluation_run = client.evals.create_evaluation_run(
+        name="test_interactions_data_source",
+        display_name="test_interactions_data_source",
+        dataset=types.EvaluationDataset(eval_cases=[eval_case]),
+        dest=GCS_DEST,
+        metrics=[GENERAL_QUALITY_METRIC],
+    )
+    assert isinstance(evaluation_run, types.EvaluationRun)
+    assert evaluation_run.state == types.EvaluationRunState.PENDING
+    assert evaluation_run.error is None
+
+
 def test_create_eval_run_with_red_teaming_config(client):
     """Tests that create_evaluation_run() with red_teaming_config sends analysisConfigs."""
     evaluation_run = client.evals.create_evaluation_run(