feat: Resolve Interactions API data for agent trace display in eval results.

PiperOrigin-RevId: 944658614

Author: vertex-sdk-bot

agentplatform/_genai/_evals_common.py

@@ -48,6 +48,8 @@
 from . import _evals_metric_loaders
 from . import _evals_utils
 from . import _gcs_utils
+
+_MAX_INTERACTION_CHAIN_DEPTH = 10
 from . import evals
 from . import types
 from . import _transformers as t
@@ -720,7 +722,9 @@ def _interaction_dict_to_agent_data(
     Returns:
         An AgentData object with one or more ConversationTurns.
     """
-    typed_interaction = interaction_types.Interaction.model_validate(interaction)
+    typed_interaction = interaction_types.Interaction.model_validate(
+        interaction
+    )
     all_events = _interaction_steps_to_events(typed_interaction.steps or [])
 
     # Group events into turns. Each UserInputStep starts a new turn.
@@ -1007,6 +1011,154 @@ def _build_interaction_id_dataset(
     return types.EvaluationDataset(eval_cases=eval_cases)
 
 
+def _has_interactions_data_source(
+    eval_cases: list[types.EvalCase],
+) -> bool:
+    """Returns True if any EvalCase has interactions_data_source set."""
+    return any(case.interactions_data_source is not None for case in eval_cases)
+
+def _resolve_interactions_to_eval_cases(
+    api_client: BaseApiClient,
+    eval_cases: list[types.EvalCase],
+) -> list[types.EvalCase]:
+    """Resolves EvalCases with interactions_data_source to agent_data.
+
+    For each EvalCase that has interactions_data_source set, fetches the
+    Interaction via the SDK's interactions.get() API, converts the steps
+    to AgentData, and returns a new EvalCase with agent_data populated.
+
+    Args:
+        api_client: The API client (must have an interactions module).
+        eval_cases: EvalCases with interactions_data_source set.
+
+    Returns:
+        New list of EvalCases with agent_data populated from resolved
+        interactions.
+
+    Raises:
+        ValueError: If eval_cases have missing interaction references.
+    """
+    # Validate all cases up front before making any API calls.
+    for case in eval_cases:
+        ids = case.interactions_data_source
+        if ids is None:
+            raise ValueError(
+                "All eval_cases must have interactions_data_source set when"
+                " using interaction resolution. Found a case without it. Do"
+                " not mix interaction-based and prompt-based eval cases."
+            )
+        if not ids.interaction:
+            raise ValueError(
+                "interactions_data_source.interaction is required. Each"
+                " EvalCase must reference an existing Interaction resource."
+            )
+
+    resolved_cases = []
+
+    for case in eval_cases:
+        if case.agent_data:
+            resolved_cases.append(case)
+            continue
+        ids = case.interactions_data_source
+
+        # Extract the interaction short ID from the resource name.
+        # Handles both full resource names (projects/.../interactions/{id})
+        # and bare IDs by always taking the last path component.
+        interaction_id = ids.interaction.split("/")[-1]
+
+        logger.info("Fetching interaction: %s", ids.interaction)
+
+        current_interaction_id = interaction_id
+        interactions = []
+        seen_ids = set()
+        for _ in range(_MAX_INTERACTION_CHAIN_DEPTH):
+            if current_interaction_id in seen_ids:
+                break
+            seen_ids.add(current_interaction_id)
+            path = f"interactions/{current_interaction_id}"
+            response = api_client.request("get", path, {}, None)
+            if not response.body:
+                if not interactions:
+                    logger.warning(
+                        "Empty response fetching interaction %s.",
+                        ids.interaction,
+                    )
+                break
+            interaction_dict = json.loads(response.body)
+            try:
+                typed_interaction = interaction_types.Interaction.model_validate(interaction_dict)
+            except Exception as e:
+                logger.warning("Failed to validate interaction model: %s", e)
+                break
+
+            interactions.append(typed_interaction)
+            if not typed_interaction.previous_interaction_id:
+                break
+            current_interaction_id = typed_interaction.previous_interaction_id.split(
+                "/"
+            )[-1]
+
+        if not interactions:
+            agent_data = types.evals.AgentData(turns=[])  # Fallback
+        else:
+            interactions.reverse()  # chronological order
+            all_steps = []
+            for i_typed in interactions:
+                all_steps.extend(i_typed.steps or [])
+
+            combined_interaction = interactions[-1].model_dump()
+            combined_interaction["steps"] = all_steps
+            agent_data = _interaction_dict_to_agent_data(combined_interaction)
+
+        # Best-effort: fetch the agent config (instruction, tools,
+        # description) from the Agent API so the display can render
+        # the System Topology section.
+        gemini_cfg = ids.gemini_agent_config
+        agent_name = gemini_cfg.gemini_agent if gemini_cfg else None
+        agent_config = _fetch_agent_config_dict(api_client, agent_name or "")
+        agent_data.agents = {agent_config.agent_id: agent_config}
+
+        # Merge consecutive text events and parts so multi-paragraph
+        # responses render as a single block in the trace display.
+        _merge_text_parts_in_agent_data(agent_data)
+
+        # Preserve all original EvalCase fields; only update agent_data
+        # and clear the now-resolved interactions_data_source.
+        resolved_cases.append(
+            case.model_copy(
+                update={
+                    "agent_data": agent_data,
+                    "interactions_data_source": None,
+                }
+            )
+        )
+
+    return resolved_cases
+
+
+def _resolve_interactions_for_display(
+    api_client: BaseApiClient,
+    dataset_list: list[types.EvaluationDataset],
+) -> list[types.EvaluationDataset]:
+    """Resolves Interaction traces for visualization."""
+    resolved_datasets = []
+    for dataset in dataset_list:
+        if dataset.eval_cases and _has_interactions_data_source(dataset.eval_cases):
+            try:
+                resolved_cases = _resolve_interactions_to_eval_cases(
+                    api_client, dataset.eval_cases
+                )
+                resolved_datasets.append(
+                    dataset.model_copy(update={"eval_cases": resolved_cases})
+                )
+            except Exception as e:
+                logger.warning("Failed to resolve interactions for display: %s", e)
+                resolved_datasets.append(dataset)
+        else:
+            resolved_datasets.append(dataset)
+    return resolved_datasets
+
+
 def _add_evaluation_run_labels(
     labels: Optional[dict[str, str]] = None,
     agent: Optional[str] = None,
@@ -2349,6 +2501,11 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
     t2 = time.perf_counter()
     logger.info("Evaluation took: %f seconds", t2 - t1)
 
+    # Resolve interactions_data_source to agent_data for display.
+    # This fetches Interaction trace data client-side so that show() can
+    # render the System Topology and Conversation Trace sections.
+    dataset_list = _resolve_interactions_for_display(api_client, dataset_list)
+
     evaluation_result.evaluation_dataset = dataset_list
     evaluation_result.agent_info = validated_agent_info
 

tests/unit/agentplatform/genai/replays/test_evaluate.py

@@ -609,6 +609,57 @@ def test_evaluation_with_interaction_id(client):
     assert len(evaluation_result.eval_case_results) == 1
 
 
+def test_evaluation_with_interaction_resolution(client):
+    """Tests evaluate() resolves interaction trace + agent config for display."""
+    interaction_id = "ChAxMDE5YTgzOGU3Mjg4NDE4EAgaATAqBG1haW4"
+    agent = (
+        "projects/model-evaluation-dev/locations/global/agents/"
+        "async-scrape-test-fe111cc6"
+    )
+    eval_dataset = types.EvaluationDataset(
+        eval_cases=[
+            types.EvalCase(
+                interactions_data_source=types.InteractionsDataSource(
+                    interaction=(
+                        "projects/model-evaluation-dev/locations/global"
+                        f"/interactions/{interaction_id}"
+                    ),
+                    gemini_agent_config=types.GeminiAgentConfig(
+                        gemini_agent=agent,
+                    ),
+                ),
+            )
+        ]
+    )
+
+    evaluation_result = client.evals.evaluate(
+        dataset=eval_dataset,
+        metrics=[types.RubricMetric.MULTI_TURN_TASK_SUCCESS],
+    )
+
+    assert isinstance(evaluation_result, types.EvaluationResult)
+    assert evaluation_result.summary_metrics is not None
+    assert len(evaluation_result.summary_metrics) > 0
+
+    assert evaluation_result.eval_case_results is not None
+    assert len(evaluation_result.eval_case_results) == 1
+
+    # Verify the interaction was resolved for display: agent_data should
+    # be populated with conversation trace and agent topology.
+    eval_dataset_list = evaluation_result.evaluation_dataset
+    assert eval_dataset_list is not None
+    assert len(eval_dataset_list) > 0
+    resolved_case = eval_dataset_list[0].eval_cases[0]
+    assert resolved_case.agent_data is not None
+    # Conversation trace: at least one turn with events.
+    assert resolved_case.agent_data.turns is not None
+    assert len(resolved_case.agent_data.turns) > 0
+    assert len(resolved_case.agent_data.turns[0].events) > 0
+    # Agent topology: agents map populated with agent config.
+    assert resolved_case.agent_data.agents is not None
+    assert len(resolved_case.agent_data.agents) > 0
+
+
 pytestmark = pytest_helper.setup(
     file=__file__,
     globals_for_file=globals(),