feat: GenAI Client(evals) - evaluate() agent parameter for interaction-id datasets

PiperOrigin-RevId: 947203638

Author: jsondai

agentplatform/_genai/_evals_common.py

@@ -923,6 +923,90 @@ def _fetch_agent_config_dict(
     )
 
 
+def _get_resolved_location(api_client: Any) -> Optional[str]:
+    """Returns the location configured on the API client."""
+    return getattr(api_client, "location", None)
+
+
+def _normalize_interaction_resource(
+    interaction: str, agent: str, location: Optional[str]
+) -> str:
+    """Normalizes an interaction id into a full resource name.
+
+    A bare interaction id is expanded to
+    `projects/{project}/locations/{location}/interactions/{id}` using the
+    project and location parsed from the agent resource name. Fully-qualified
+    interaction resource names are returned unchanged.
+    """
+    if interaction.startswith("projects/"):
+        return interaction
+    parts = agent.split("/")
+    project = parts[1]
+    agent_location = parts[3] if len(parts) > 3 else (location or "global")
+    return f"projects/{project}/locations/{agent_location}/interactions/{interaction}"
+
+
+def _build_interaction_id_dataset(
+    loaded_data: list[dict[str, Any]],
+    agent: Optional[str],
+    location: Optional[str],
+) -> Optional[types.EvaluationDataset]:
+    """Builds an EvaluationDataset from rows that carry an `interaction_id`.
+
+    When the dataset contains an `interaction_id` column, each row is turned
+    into an EvalCase whose `interactions_data_source` references the interaction
+    and the Gemini agent. The backend resolves the interaction trace and agent
+    config; no client-side prompt/response is required. Returns None if the
+    data does not contain interaction ids.
+    """
+    has_interaction_id = bool(loaded_data) and any(
+        _evals_constant.INTERACTION_ID in row for row in loaded_data
+    )
+    if not has_interaction_id:
+        if agent:
+            raise ValueError(
+                "An `agent` was provided but the dataset does not contain an"
+                " `interaction_id` column. The `agent` argument is only used to"
+                " resolve an `interaction_id` dataset column (so the backend can"
+                " fetch the interaction trace and Agent config). To evaluate"
+                " with an agent, provide a dataset with an `interaction_id`"
+                " column; otherwise omit `agent`."
+            )
+        return None
+
+    if not agent:
+        raise ValueError(
+            "An `agent` resource name is required when the dataset contains an"
+            " `interaction_id` column, so the backend can resolve the Agent"
+            " config for each interaction."
+        )
+    if not _is_gemini_agent_resource(agent):
+        raise ValueError(
+            "`agent` must be a Gemini Agents API resource name of the form"
+            " projects/{project}/locations/{location}/agents/{agent} when"
+            f" evaluating interaction ids. Got: {agent}"
+        )
+
+    gemini_agent_config = types.GeminiAgentConfig(gemini_agent=agent)
+    eval_cases = []
+    for i, row in enumerate(loaded_data):
+        interaction = row.get(_evals_constant.INTERACTION_ID)
+        if not interaction:
+            raise ValueError(f"Missing `interaction_id` value for row {i}.")
+        eval_cases.append(
+            types.EvalCase(
+                eval_case_id=f"eval_case_{i}",
+                interactions_data_source=types.InteractionsDataSource(
+                    interaction=_normalize_interaction_resource(
+                        str(interaction), agent, location
+                    ),
+                    gemini_agent_config=gemini_agent_config,
+                ),
+            )
+        )
+    return types.EvaluationDataset(eval_cases=eval_cases)
+
+
 def _add_evaluation_run_labels(
     labels: Optional[dict[str, str]] = None,
     agent: Optional[str] = None,
@@ -1939,6 +2023,8 @@ def _resolve_dataset_inputs(
     dataset_schema: Optional[Literal["GEMINI", "FLATTEN", "OPENAI"]],
     loader: "_evals_utils.EvalDatasetLoader",
     agent_info: Optional[types.evals.AgentInfo] = None,
+    agent: Optional[str] = None,
+    api_client: Any = None,
 ) -> tuple[types.EvaluationDataset, int]:
     """Loads and processes single or multiple datasets for evaluation.
 
@@ -1988,6 +2074,21 @@ def _resolve_dataset_inputs(
         ds_source_for_loader = _get_dataset_source(ds_item)
         current_loaded_data = loader.load(ds_source_for_loader)
 
+        interaction_dataset = _build_interaction_id_dataset(
+            current_loaded_data, agent, _get_resolved_location(api_client)
+        )
+        if interaction_dataset is not None:
+            if dataset_schema:
+                raise ValueError(
+                    "`dataset_schema` is not supported for datasets with an"
+                    " `interaction_id` column. The interaction trace and agent"
+                    " config are resolved by the backend, so no client-side"
+                    " schema conversion is applied. Omit `dataset_schema` when"
+                    " evaluating an interaction_id dataset."
+                )
+            parsed_evaluation_datasets.append(interaction_dataset)
+            continue
+
         if dataset_schema:
             current_schema = _evals_data_converters.EvalDatasetSchema(dataset_schema)
         else:
@@ -2145,6 +2246,7 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
     api_client: Any,
     dataset: Union[types.EvaluationDataset, list[types.EvaluationDataset]],
     metrics: list[types.Metric],
+    agent: Optional[str] = None,
     dataset_schema: Optional[Literal["GEMINI", "FLATTEN", "OPENAI"]] = None,
     dest: Optional[str] = None,
     location: Optional[str] = None,
@@ -2225,6 +2327,8 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
         dataset_schema=dataset_schema,
         loader=loader,
         agent_info=validated_agent_info,
+        agent=agent,
+        api_client=api_client,
     )
 
     resolved_metrics = _resolve_metrics(metrics, api_client)

agentplatform/_genai/_evals_constant.py

@@ -57,6 +57,7 @@
 PARTS = "parts"
 USER_AUTHOR = "user"
 AGENT_DATA = "agent_data"
+INTERACTION_ID = "interaction_id"
 STARTING_PROMPT = "starting_prompt"
 CONVERSATION_PLAN = "conversation_plan"
 HISTORY = "history"
@@ -74,5 +75,6 @@
         STARTING_PROMPT,
         CONVERSATION_PLAN,
         AGENT_DATA,
+        INTERACTION_ID,
     }
 )

agentplatform/_genai/evals.py

@@ -2214,6 +2214,7 @@ def evaluate(
             list[types.EvaluationDatasetOrDict],
         ],
         metrics: Optional[list[types.MetricOrDict]] = None,
+        agent: Optional[str] = None,
         location: Optional[str] = None,
         config: Optional[types.EvaluateMethodConfigOrDict] = None,
         **kwargs: Any,
@@ -2222,8 +2223,15 @@ def evaluate(
 
         Args:
           dataset: The dataset(s) to evaluate. Can be a pandas DataFrame, a single
-            `types.EvaluationDataset` or a list of `types.EvaluationDataset`.
+            `types.EvaluationDataset` or a list of `types.EvaluationDataset`. To
+            evaluate existing interactions, provide a dataset with an
+            `interaction_id` column; each interaction is resolved by the backend
+            using `agent` to populate the agent data for evaluation.
           metrics: The list of metrics to use for evaluation.
+          agent: Optional Gemini Agents API agent resource name
+            (`projects/{project}/locations/{location}/agents/{agent}`). Required
+            when the dataset contains an `interaction_id` column: the backend uses
+            it to resolve the Agent config for each referenced interaction.
           location: The location to use for the evaluation service. If not specified,
              the location configured in the client will be used. If specified,
              this will override the location set in `agentplatform.Client` only for
@@ -2274,6 +2282,7 @@ def evaluate(
             api_client=self._api_client,
             dataset=dataset,
             metrics=metrics,
+            agent=agent,
             dataset_schema=config.dataset_schema,
             dest=config.dest,
             location=location,

tests/unit/agentplatform/genai/replays/test_evaluate.py

@@ -582,6 +582,33 @@ def test_evaluation_single_turn_agent_data(client):
     assert len(evaluation_result.eval_case_results) == 1
 
 
+def test_evaluation_with_interaction_id(client):
+    """Tests evaluate() an interaction_id dataset with the `agent` parameter."""
+    client._api_client._http_options.api_version = "v1beta1"
+    eval_dataset = types.EvaluationDataset(
+        eval_dataset_df=pd.DataFrame(
+            {"interaction_id": ["ChA5YTc2MWEzZmIxNWQyY2Y2EAgaATAqBG1haW4"]}
+        )
+    )
+
+    evaluation_result = client.evals.evaluate(
+        dataset=eval_dataset,
+        agent=("projects/model-evaluation-dev/locations/global/agents/test-agent-eval"),
+        metrics=[types.RubricMetric.MULTI_TURN_TASK_SUCCESS],
+    )
+
+    assert isinstance(evaluation_result, types.EvaluationResult)
+    assert evaluation_result.summary_metrics is not None
+    assert len(evaluation_result.summary_metrics) > 0
+    for summary in evaluation_result.summary_metrics:
+        assert isinstance(summary, types.AggregatedMetricResult)
+        assert summary.metric_name is not None
+        assert summary.mean_score is not None
+
+    assert evaluation_result.eval_case_results is not None
+    assert len(evaluation_result.eval_case_results) == 1
+
+
 pytestmark = pytest_helper.setup(
     file=__file__,
     globals_for_file=globals(),

tests/unit/agentplatform/genai/test_evals.py

@@ -9905,6 +9905,101 @@ def test_evaluate_instances_sends_interactions_data_source(self):
         assert data_source["gemini_agent_config"]["gemini_agent"] == _TEST_GEMINI_AGENT
 
 
+class TestEvaluateInteractionIdDataset:
+    """CUJ1 via evaluate(): interaction_id column + agent -> data source."""
+
+    def test_build_interaction_id_dataset_from_column(self):
+        loaded = [
+            {"interaction_id": "abc123"},
+            {"interaction_id": ("projects/p/locations/global/interactions/def456")},
+        ]
+        dataset = _evals_common._build_interaction_id_dataset(
+            loaded, _TEST_GEMINI_AGENT, "global"
+        )
+
+        assert dataset is not None
+        assert len(dataset.eval_cases) == 2
+        ds0 = dataset.eval_cases[0].interactions_data_source
+        assert ds0.gemini_agent_config.gemini_agent == _TEST_GEMINI_AGENT
+        assert ds0.interaction == (
+            "projects/test-project/locations/us-central1/interactions/abc123"
+        )
+        assert (
+            dataset.eval_cases[1].interactions_data_source.interaction
+            == "projects/p/locations/global/interactions/def456"
+        )
+        assert dataset.eval_cases[0].agent_data is None
+
+    def test_build_interaction_id_dataset_requires_agent(self):
+        with pytest.raises(ValueError, match="agent.*required"):
+            _evals_common._build_interaction_id_dataset(
+                [{"interaction_id": "abc123"}], None, "global"
+            )
+
+    def test_build_interaction_id_dataset_rejects_non_gemini_agent(self):
+        with pytest.raises(ValueError, match="Gemini Agents API resource name"):
+            _evals_common._build_interaction_id_dataset(
+                [{"interaction_id": "abc123"}],
+                "projects/p/locations/us-central1/reasoningEngines/123",
+                "global",
+            )
+
+    def test_build_interaction_id_dataset_none_without_column_and_no_agent(self):
+        assert (
+            _evals_common._build_interaction_id_dataset(
+                [{"prompt": "hi", "response": "yo"}], None, "global"
+            )
+            is None
+        )
+
+    def test_build_interaction_id_dataset_agent_without_column_raises(self):
+        with pytest.raises(ValueError, match="interaction_id"):
+            _evals_common._build_interaction_id_dataset(
+                [{"prompt": "hi", "response": "yo"}], _TEST_GEMINI_AGENT, "global"
+            )
+
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
+    def test_evaluate_agent_without_interaction_id_column_raises(
+        self, mock_eval_dataset_loader
+    ):
+        agentplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
+        client = agentplatform.Client(project=_TEST_PROJECT, location=_TEST_LOCATION)
+        mock_df = pd.DataFrame([{"prompt": "p1", "response": "r1"}])
+        mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict(
+            orient="records"
+        )
+        dataset = agentplatform_genai_types.EvaluationDataset(eval_dataset_df=mock_df)
+
+        with pytest.raises(ValueError, match="interaction_id"):
+            client.evals.evaluate(
+                dataset=dataset,
+                metrics=[agentplatform_genai_types.Metric(name="exact_match")],
+                agent=_TEST_GEMINI_AGENT,
+            )
+
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
+    def test_evaluate_interaction_id_with_dataset_schema_raises(
+        self, mock_eval_dataset_loader
+    ):
+        agentplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
+        client = agentplatform.Client(project=_TEST_PROJECT, location=_TEST_LOCATION)
+        mock_df = pd.DataFrame([{"interaction_id": "i1"}])
+        mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict(
+            orient="records"
+        )
+        dataset = agentplatform_genai_types.EvaluationDataset(eval_dataset_df=mock_df)
+
+        with pytest.raises(ValueError, match="dataset_schema"):
+            client.evals.evaluate(
+                dataset=dataset,
+                metrics=[agentplatform_genai_types.Metric(name="exact_match")],
+                agent=_TEST_GEMINI_AGENT,
+                config=agentplatform_genai_types.EvaluateMethodConfig(
+                    dataset_schema="GEMINI"
+                ),
+            )
+
+
 class TestCreateEvaluationRunGeminiAgent:
     """CUJ2: scrape a Gemini agent via create_evaluation_run."""