feat: GenAI Client(evals) - async auto-create EvaluationExperiment parity

PiperOrigin-RevId: 955977255

Author: jsondai

agentplatform/_genai/evals.py

@@ -5504,6 +5504,7 @@ async def create_evaluation_run(
         metrics: list[types.EvaluationRunMetricOrDict],
         name: Optional[str] = None,
         display_name: Optional[str] = None,
+        evaluation_experiment: Optional[str] = None,
         agent_info: Optional[evals_types.AgentInfo] = None,
         agent: Optional[str] = None,
         user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None,
@@ -5524,6 +5525,11 @@ async def create_evaluation_run(
           metrics: The list of metrics to evaluate.
           name: The name of the evaluation run.
           display_name: The display name of the evaluation run.
+          evaluation_experiment: The resource name of an existing
+              EvaluationExperiment to group this run under. If omitted, a new
+              EvaluationExperiment is created automatically so the run is visible in
+              the Agent Platform UI. Pass an existing experiment name to group
+              multiple runs together.
           agent_info: The agent info to evaluate. Mutually exclusive with
               `inference_configs`.
           agent: The agent resource name in str type. Accepts either an Agent
@@ -5682,11 +5688,23 @@ async def create_evaluation_run(
             self._api_client, resolved_dataset, inference_configs, parsed_agent_info
         )
         resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent)
-        resolved_name = name or f"evaluation_run_{uuid.uuid4()}"
+        resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}"
+        resolved_experiment = evaluation_experiment
+        if resolved_experiment is None:
+            experiment_display_name = (
+                display_name or f"SDK Experiment {_evals_common._local_timestamp()}"
+            )
+            created_experiment = await self.create_evaluation_experiment(
+                display_name=experiment_display_name
+            )
+            resolved_experiment = created_experiment.name
 
+        # The backend rejects a caller-supplied `name` for runs that belong to an
+        # EvaluationExperiment and assigns the resource ID itself, so `name` is
+        # only used as a display_name fallback.
         result = await self._create_evaluation_run(
-            name=resolved_name,
-            display_name=display_name or resolved_name,
+            display_name=resolved_display_name,
+            evaluation_experiment=resolved_experiment,
             data_source=resolved_dataset,
             evaluation_config=evaluation_config,
             inference_configs=resolved_inference_configs,

tests/unit/agentplatform/genai/test_evals.py

@@ -10635,25 +10635,33 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self)
             return_value=self.mock_response
         )
         async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client)
-
-        await async_evals_module.create_evaluation_run(
-            dataset=agentplatform_genai_types.EvaluationRunDataSource(
-                evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
-            ),
-            metrics=[
-                agentplatform_genai_types.EvaluationRunMetric(
-                    metric="general_quality_v1",
-                    metric_config=agentplatform_genai_types.UnifiedMetric(
-                        predefined_metric_spec=genai_types.PredefinedMetricSpec(
-                            metric_spec_name="general_quality_v1",
-                        )
-                    ),
-                )
-            ],
-            dest="gs://test-bucket/output",
-            config={"allow_cross_region_model": True},
+        experiment = agentplatform_genai_types.EvaluationExperiment(
+            name="projects/123/locations/us-central1/evaluationExperiments/e1"
         )
 
+        with mock.patch.object(
+            async_evals_module,
+            "create_evaluation_experiment",
+            new=mock.AsyncMock(return_value=experiment),
+        ):
+            await async_evals_module.create_evaluation_run(
+                dataset=agentplatform_genai_types.EvaluationRunDataSource(
+                    evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
+                ),
+                metrics=[
+                    agentplatform_genai_types.EvaluationRunMetric(
+                        metric="general_quality_v1",
+                        metric_config=agentplatform_genai_types.UnifiedMetric(
+                            predefined_metric_spec=genai_types.PredefinedMetricSpec(
+                                metric_spec_name="general_quality_v1",
+                            )
+                        ),
+                    )
+                ],
+                dest="gs://test-bucket/output",
+                config={"allow_cross_region_model": True},
+            )
+
         self.mock_api_client.async_request.assert_called_once()
         call_args = self.mock_api_client.async_request.call_args
         request_body = call_args[0][2]  # Third positional arg is the request dict
@@ -12414,3 +12422,86 @@ def test_uses_provided_experiment_without_creating(self):
             request_body.get("evaluationExperiment")
             == "projects/123/locations/us-central1/evaluationExperiments/existing"
         )
+
+
+class TestAsyncCreateEvaluationRunAutoExperiment:
+
+    def setup_method(self, method):
+        self.mock_api_client = mock.MagicMock()
+        self.mock_api_client.vertexai = True
+        self.mock_response = mock.MagicMock()
+        self.mock_response.body = json.dumps(
+            {
+                "name": "projects/123/locations/us-central1/evaluationRuns/456",
+                "displayName": "test_run",
+                "state": "PENDING",
+                "evaluationExperiment": (
+                    "projects/123/locations/us-central1/evaluationExperiments/e1"
+                ),
+            }
+        )
+        self.mock_api_client.async_request = mock.AsyncMock(
+            return_value=self.mock_response
+        )
+        self.dataset = agentplatform_genai_types.EvaluationRunDataSource(
+            evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
+        )
+        self.metrics = [
+            agentplatform_genai_types.EvaluationRunMetric(
+                metric="general_quality_v1",
+                metric_config=agentplatform_genai_types.UnifiedMetric(
+                    predefined_metric_spec=genai_types.PredefinedMetricSpec(
+                        metric_spec_name="general_quality_v1",
+                    )
+                ),
+            )
+        ]
+
+    @pytest.mark.asyncio
+    async def test_async_auto_creates_experiment_when_not_provided(self):
+        async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client)
+        experiment = agentplatform_genai_types.EvaluationExperiment(
+            name="projects/123/locations/us-central1/evaluationExperiments/e1"
+        )
+        with mock.patch.object(
+            async_evals_module,
+            "create_evaluation_experiment",
+            new=mock.AsyncMock(return_value=experiment),
+        ) as mock_create_exp:
+            await async_evals_module.create_evaluation_run(
+                dataset=self.dataset,
+                metrics=self.metrics,
+                dest="gs://test-bucket/output",
+                display_name="my_run",
+            )
+
+        mock_create_exp.assert_awaited_once()
+        request_body = self.mock_api_client.async_request.call_args[0][2]
+        assert (
+            request_body.get("evaluationExperiment")
+            == "projects/123/locations/us-central1/evaluationExperiments/e1"
+        )
+
+    @pytest.mark.asyncio
+    async def test_async_uses_provided_experiment_without_creating(self):
+        async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client)
+        with mock.patch.object(
+            async_evals_module,
+            "create_evaluation_experiment",
+            new=mock.AsyncMock(),
+        ) as mock_create_exp:
+            await async_evals_module.create_evaluation_run(
+                dataset=self.dataset,
+                metrics=self.metrics,
+                dest="gs://test-bucket/output",
+                evaluation_experiment=(
+                    "projects/123/locations/us-central1/evaluationExperiments/existing"
+                ),
+            )
+
+        mock_create_exp.assert_not_awaited()
+        request_body = self.mock_api_client.async_request.call_args[0][2]
+        assert (
+            request_body.get("evaluationExperiment")
+            == "projects/123/locations/us-central1/evaluationExperiments/existing"
+        )

vertexai/_genai/evals.py

@@ -4949,6 +4949,7 @@ async def create_evaluation_run(
         metrics: list[types.EvaluationRunMetricOrDict],
         name: Optional[str] = None,
         display_name: Optional[str] = None,
+        evaluation_experiment: Optional[str] = None,
         agent_info: Optional[evals_types.AgentInfo] = None,
         agent: Optional[str] = None,
         user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None,
@@ -5095,11 +5096,23 @@ async def create_evaluation_run(
             self._api_client, resolved_dataset, inference_configs, parsed_agent_info
         )
         resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent)
-        resolved_name = name or f"evaluation_run_{uuid.uuid4()}"
+        resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}"
+        resolved_experiment = evaluation_experiment
+        if resolved_experiment is None:
+            experiment_display_name = (
+                display_name or f"SDK Experiment {_evals_common._local_timestamp()}"
+            )
+            created_experiment = await self.create_evaluation_experiment(
+                display_name=experiment_display_name
+            )
+            resolved_experiment = created_experiment.name
 
+        # The backend rejects a caller-supplied `name` for runs that belong to an
+        # EvaluationExperiment and assigns the resource ID itself, so `name` is
+        # only used as a display_name fallback.
         result = await self._create_evaluation_run(
-            name=resolved_name,
-            display_name=display_name or resolved_name,
+            display_name=resolved_display_name,
+            evaluation_experiment=resolved_experiment,
             data_source=resolved_dataset,
             evaluation_config=evaluation_config,
             inference_configs=resolved_inference_configs,