feat: Make agent_info optional for running inference with create_evaluation_run

PiperOrigin-RevId: 951079941

Author: vertex-sdk-bot

agentplatform/_genai/_evals_common.py

@@ -88,6 +88,7 @@
 PARTS = _evals_constant.PARTS
 USER_AUTHOR = _evals_constant.USER_AUTHOR
 AGENT_DATA = _evals_constant.AGENT_DATA
+_DEFAULT_CANDIDATE_NAME = _evals_constant.DEFAULT_CANDIDATE_NAME
 
 
 @contextlib.contextmanager
@@ -528,11 +529,11 @@ def _resolve_inference_configs(
         if inference_configs is None:
             inference_configs = {}
 
-        # We might have used "candidate-1" as a placeholder key in the caller,
-        # let's migrate it to the agent name, or if it doesn't exist, just create it.
-        if "candidate-1" in inference_configs:
+        # We might have used the default candidate name as a placeholder key
+        # in the caller; migrate it to the agent name.
+        if _DEFAULT_CANDIDATE_NAME in inference_configs:
             inference_configs[parsed_agent_info.name] = inference_configs.pop(
-                "candidate-1"
+                _DEFAULT_CANDIDATE_NAME
             )
 
         if parsed_agent_info.name not in inference_configs:

agentplatform/_genai/_evals_constant.py

@@ -63,6 +63,7 @@
 CONVERSATION_PLAN = "conversation_plan"
 HISTORY = "history"
 CONVERSATION_HISTORY = "conversation_history"
+DEFAULT_CANDIDATE_NAME = "candidate-1"
 
 COMMON_DATASET_COLUMNS = frozenset(
     {

agentplatform/_genai/evals.py

@@ -2746,7 +2746,25 @@ def create_evaluation_run(
         if isinstance(config, dict):
             config = types.CreateEvaluationRunConfig.model_validate(config)
 
-        if agent_info and not inference_configs:
+        # Auto-construct inference_configs when agent_info is explicitly
+        # provided (existing behavior) OR an agent resource is provided
+        # (allows omitting agent_info for both Gemini Agents and Agent
+        # Engine). The server skips inference per-item when a
+        # CandidateResponse with a matching candidate name already exists,
+        # so it is safe to always send inference_configs.
+        _should_auto_infer = not inference_configs and (agent_info or agent)
+        if _should_auto_infer:
+            if not parsed_agent_info.name:
+                # Prefer the dataset's candidate_name (set by run_inference)
+                # so the inference_configs key matches the CandidateResponse
+                # and the server correctly skips already-completed items.
+                if (
+                    isinstance(dataset, types.EvaluationDataset)
+                    and dataset.candidate_name
+                ):
+                    parsed_agent_info.name = dataset.candidate_name
+                else:
+                    parsed_agent_info.name = _evals_common._DEFAULT_CANDIDATE_NAME
             parsed_user_simulator_config = (
                 evals_types.UserSimulatorConfig.model_validate(user_simulator_config)
                 if isinstance(user_simulator_config, dict)
@@ -2755,7 +2773,9 @@ def create_evaluation_run(
             if getattr(parsed_user_simulator_config, "max_turn", None) is None:
                 parsed_user_simulator_config.max_turn = 5
 
-            candidate_name = parsed_agent_info.name or "candidate-1"
+            candidate_name = (
+                parsed_agent_info.name or _evals_common._DEFAULT_CANDIDATE_NAME
+            )
             if agent and _evals_common._is_gemini_agent_resource(agent):
                 agent_run_config = types.AgentRunConfig(
                     gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent),
@@ -4534,7 +4554,25 @@ async def create_evaluation_run(
         if isinstance(config, dict):
             config = types.CreateEvaluationRunConfig.model_validate(config)
 
-        if agent_info and not inference_configs:
+        # Auto-construct inference_configs when agent_info is explicitly
+        # provided (existing behavior) OR an agent resource is provided
+        # (allows omitting agent_info for both Gemini Agents and Agent
+        # Engine). The server skips inference per-item when a
+        # CandidateResponse with a matching candidate name already exists,
+        # so it is safe to always send inference_configs.
+        _should_auto_infer = not inference_configs and (agent_info or agent)
+        if _should_auto_infer:
+            if not parsed_agent_info.name:
+                # Prefer the dataset's candidate_name (set by run_inference)
+                # so the inference_configs key matches the CandidateResponse
+                # and the server correctly skips already-completed items.
+                if (
+                    isinstance(dataset, types.EvaluationDataset)
+                    and dataset.candidate_name
+                ):
+                    parsed_agent_info.name = dataset.candidate_name
+                else:
+                    parsed_agent_info.name = _evals_common._DEFAULT_CANDIDATE_NAME
             parsed_user_simulator_config = (
                 evals_types.UserSimulatorConfig.model_validate(user_simulator_config)
                 if isinstance(user_simulator_config, dict)
@@ -4543,7 +4581,9 @@ async def create_evaluation_run(
             if getattr(parsed_user_simulator_config, "max_turn", None) is None:
                 parsed_user_simulator_config.max_turn = 5
 
-            candidate_name = parsed_agent_info.name or "candidate-1"
+            candidate_name = (
+                parsed_agent_info.name or _evals_common._DEFAULT_CANDIDATE_NAME
+            )
             if agent and _evals_common._is_gemini_agent_resource(agent):
                 agent_run_config = types.AgentRunConfig(
                     gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent),

tests/unit/agentplatform/genai/test_evals.py

@@ -10761,6 +10761,10 @@ def test_create_evaluation_run_builds_gemini_agent_config(self):
             == _TEST_GEMINI_AGENT
         )
         assert "agent_engine" not in agent_run_config
+        # agent_info.name overrides the default candidate name.
+        inference_configs = request_body["inferenceConfigs"]
+        assert "gemini-agent" in inference_configs
+        assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs
 
     def test_create_evaluation_run_agent_engine_does_not_set_gemini(self):
         evals_module = evals.Evals(api_client_=self.mock_api_client)
@@ -10788,6 +10792,140 @@ def test_create_evaluation_run_agent_engine_does_not_set_gemini(self):
         agent_run_config = self._agent_run_config(request_body)
         assert "gemini_agent_config" not in agent_run_config
         assert agent_run_config["agent_engine"] == _TEST_AGENT_ENGINE
+        # agent_info.name overrides the default candidate name.
+        inference_configs = request_body["inferenceConfigs"]
+        assert "ae-agent" in inference_configs
+        assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs
+
+    def test_create_evaluation_run_gemini_agent_without_agent_info(self):
+        """Gemini agent resource alone triggers inference_configs auto-construction."""
+        evals_module = evals.Evals(api_client_=self.mock_api_client)
+
+        evals_module.create_evaluation_run(
+            dataset=agentplatform_genai_types.EvaluationRunDataSource(
+                evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
+            ),
+            metrics=[
+                agentplatform_genai_types.EvaluationRunMetric(
+                    metric="multi_turn_task_success_v1",
+                    metric_config=agentplatform_genai_types.UnifiedMetric(
+                        predefined_metric_spec=genai_types.PredefinedMetricSpec(
+                            metric_spec_name="multi_turn_task_success_v1",
+                        )
+                    ),
+                )
+            ],
+            dest="gs://test-bucket/output",
+            agent=_TEST_GEMINI_AGENT,
+            # No agent_info provided.
+        )
+
+        request_body = self._get_create_run_body()
+        agent_run_config = self._agent_run_config(request_body)
+        assert (
+            agent_run_config["gemini_agent_config"]["gemini_agent"]
+            == _TEST_GEMINI_AGENT
+        )
+        assert "agent_engine" not in agent_run_config
+        # Default candidate name should match the constant.
+        inference_configs = request_body["inferenceConfigs"]
+        assert _evals_common._DEFAULT_CANDIDATE_NAME in inference_configs
+
+    def test_create_evaluation_run_no_agent_no_agent_info_no_inference(self):
+        """Without agent or agent_info, no inference_configs are auto-constructed."""
+        evals_module = evals.Evals(api_client_=self.mock_api_client)
+
+        evals_module.create_evaluation_run(
+            dataset=agentplatform_genai_types.EvaluationRunDataSource(
+                evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
+            ),
+            metrics=[
+                agentplatform_genai_types.EvaluationRunMetric(
+                    metric="multi_turn_task_success_v1",
+                    metric_config=agentplatform_genai_types.UnifiedMetric(
+                        predefined_metric_spec=genai_types.PredefinedMetricSpec(
+                            metric_spec_name="multi_turn_task_success_v1",
+                        )
+                    ),
+                )
+            ],
+            dest="gs://test-bucket/output",
+            # No agent, no agent_info.
+        )
+
+        request_body = self._get_create_run_body()
+        assert "inferenceConfigs" not in request_body or not request_body.get(
+            "inferenceConfigs"
+        )
+
+    def test_create_evaluation_run_agent_engine_without_agent_info(self):
+        """Agent Engine resource alone triggers inference_configs auto-construction."""
+        evals_module = evals.Evals(api_client_=self.mock_api_client)
+
+        evals_module.create_evaluation_run(
+            dataset=agentplatform_genai_types.EvaluationRunDataSource(
+                evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
+            ),
+            metrics=[
+                agentplatform_genai_types.EvaluationRunMetric(
+                    metric="multi_turn_task_success_v1",
+                    metric_config=agentplatform_genai_types.UnifiedMetric(
+                        predefined_metric_spec=genai_types.PredefinedMetricSpec(
+                            metric_spec_name="multi_turn_task_success_v1",
+                        )
+                    ),
+                )
+            ],
+            dest="gs://test-bucket/output",
+            agent=_TEST_AGENT_ENGINE,
+            # No agent_info provided.
+        )
+
+        request_body = self._get_create_run_body()
+        agent_run_config = self._agent_run_config(request_body)
+        assert agent_run_config["agent_engine"] == _TEST_AGENT_ENGINE
+        assert "gemini_agent_config" not in agent_run_config
+        # Default candidate name should match the constant.
+        inference_configs = request_body["inferenceConfigs"]
+        assert _evals_common._DEFAULT_CANDIDATE_NAME in inference_configs
+
+    @mock.patch.object(_evals_common, "_resolve_dataset")
+    def test_create_evaluation_run_uses_dataset_candidate_name(
+        self, mock_resolve_dataset
+    ):
+        """When dataset.candidate_name is set (e.g. from run_inference), the
+        inference_configs key should match it instead of using the default."""
+        mock_resolve_dataset.return_value = (
+            agentplatform_genai_types.EvaluationRunDataSource(
+                evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
+            )
+        )
+        evals_module = evals.Evals(api_client_=self.mock_api_client)
+
+        evals_module.create_evaluation_run(
+            dataset=agentplatform_genai_types.EvaluationDataset(
+                eval_dataset_df=pd.DataFrame({"prompt": ["hello"]}),
+                candidate_name="my-agent-v2",
+            ),
+            metrics=[
+                agentplatform_genai_types.EvaluationRunMetric(
+                    metric="multi_turn_task_success_v1",
+                    metric_config=agentplatform_genai_types.UnifiedMetric(
+                        predefined_metric_spec=genai_types.PredefinedMetricSpec(
+                            metric_spec_name="multi_turn_task_success_v1",
+                        )
+                    ),
+                )
+            ],
+            dest="gs://test-bucket/output",
+            agent=_TEST_GEMINI_AGENT,
+            # No agent_info -- candidate name should come from dataset.
+        )
+
+        request_body = self._get_create_run_body()
+        inference_configs = request_body["inferenceConfigs"]
+        assert "my-agent-v2" in inference_configs
+        assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs
 
 
 class TestResolveInteractionsForDisplay: