fix: Add a warning that the autorater config is not applicable for predefined metrics in SDK

PiperOrigin-RevId: 910847685

Author: vertex-sdk-bot

agentplatform/_genai/_evals_metric_handlers.py

@@ -972,6 +972,17 @@ def __init__(self, module: "evals.Evals", metric: types.Metric):
             raise ValueError(
                 f"Metric '{self.metric.name}' is not a supported predefined metric."
             )
+        if (
+            self.metric.judge_model
+            or self.metric.judge_model_generation_config
+            or self.metric.judge_model_sampling_count
+        ):
+            logger.warning(
+                "Autorater config settings (judge_model, "
+                "judge_model_generation_config, judge_model_sampling_count) "
+                "are ignored for predefined metric '%s'.",
+                self.metric.name,
+            )
 
     def _build_request_payload(
         self, eval_case: types.EvalCase, response_index: int

agentplatform/_genai/types/common.py

@@ -2424,7 +2424,8 @@ class EvaluationRunConfig(_common.BaseModel):
         default=None, description="""The output config for the evaluation run."""
     )
     autorater_config: Optional[genai_types.AutoraterConfig] = Field(
-        default=None, description="""The autorater config for the evaluation run."""
+        default=None,
+        description="""The autorater config for the evaluation run. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""",
     )
     prompt_template: Optional[EvaluationRunPromptTemplate] = Field(
         default=None, description="""The prompt template used for inference."""
@@ -2454,7 +2455,7 @@ class EvaluationRunConfigDict(TypedDict, total=False):
     """The output config for the evaluation run."""
 
     autorater_config: Optional[genai_types.AutoraterConfigDict]
-    """The autorater config for the evaluation run."""
+    """The autorater config for the evaluation run. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored."""
 
     prompt_template: Optional[EvaluationRunPromptTemplateDict]
     """The prompt template used for inference."""
@@ -4761,7 +4762,8 @@ class _EvaluateInstancesRequestParameters(_common.BaseModel):
         default=None, description=""""""
     )
     autorater_config: Optional[genai_types.AutoraterConfig] = Field(
-        default=None, description=""""""
+        default=None,
+        description="""Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""",
     )
     metrics: Optional[list[Metric]] = Field(
         default=None,
@@ -4812,7 +4814,7 @@ class _EvaluateInstancesRequestParametersDict(TypedDict, total=False):
     """"""
 
     autorater_config: Optional[genai_types.AutoraterConfigDict]
-    """"""
+    """Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored."""
 
     metrics: Optional[list[MetricDict]]
     """The metrics used for evaluation.
@@ -19043,7 +19045,8 @@ class EvaluateDatasetRequestParameters(_common.BaseModel):
         default=None, description=""""""
     )
     autorater_config: Optional[genai_types.AutoraterConfig] = Field(
-        default=None, description=""""""
+        default=None,
+        description="""Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""",
     )
     config: Optional[EvaluateDatasetConfig] = Field(default=None, description="""""")
 
@@ -19061,7 +19064,7 @@ class EvaluateDatasetRequestParametersDict(TypedDict, total=False):
     """"""
 
     autorater_config: Optional[genai_types.AutoraterConfigDict]
-    """"""
+    """Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored."""
 
     config: Optional[EvaluateDatasetConfigDict]
     """"""

tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py

@@ -106,6 +106,39 @@ def test_evaluation_result(client):
 #         assert case_result.response_candidate_results is not None
 
 
+def test_predefined_metric_with_judge_model_ignores_autorater_config(client):
+    """Tests that autorater_config is ignored for predefined metrics in replays."""
+    prompts_df = pd.DataFrame(
+        {
+            "prompt": ["Explain the concept of machine learning in simple terms."],
+            "response": [
+                "Machine learning is a type of artificial intelligence that allows"
+                " computers to learn from data without being explicitly programmed."
+            ],
+        }
+    )
+
+    eval_dataset = types.EvaluationDataset(
+        eval_dataset_df=prompts_df,
+        candidate_name="gemini-2.5-flash",
+    )
+
+    # Set judge_model, which should be ignored for predefined metrics
+    metric = types.Metric(
+        name="safety_v1",
+        judge_model="projects/model-evaluation-dev/locations/us-central1/publishers/google/models/gemini-2.5-flash"
+    )
+
+    evaluation_result = client.evals.evaluate(
+        dataset=eval_dataset,
+        metrics=[metric],
+    )
+
+    assert isinstance(evaluation_result, types.EvaluationResult)
+    assert evaluation_result.summary_metrics is not None
+    assert evaluation_result.summary_metrics[0].metric_name == "safety_v1"
+
+
 def test_multi_turn_predefined_metric(client):
     """Tests that evaluate works with multi-turn predefined metrics."""
     prompts_data = {

vertexai/_genai/_evals_metric_handlers.py

@@ -972,6 +972,17 @@ def __init__(self, module: "evals.Evals", metric: types.Metric):
             raise ValueError(
                 f"Metric '{self.metric.name}' is not a supported predefined metric."
             )
+        if (
+            self.metric.judge_model
+            or self.metric.judge_model_generation_config
+            or self.metric.judge_model_sampling_count
+        ):
+            logger.warning(
+                "Autorater config settings (judge_model, "
+                "judge_model_generation_config, judge_model_sampling_count) "
+                "are ignored for predefined metric '%s'.",
+                self.metric.name,
+            )
 
     def _build_request_payload(
         self, eval_case: types.EvalCase, response_index: int

vertexai/_genai/types/common.py

@@ -2409,7 +2409,8 @@ class EvaluationRunConfig(_common.BaseModel):
         default=None, description="""The output config for the evaluation run."""
     )
     autorater_config: Optional[genai_types.AutoraterConfig] = Field(
-        default=None, description="""The autorater config for the evaluation run."""
+        default=None,
+        description="""The autorater config for the evaluation run. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""",
     )
     prompt_template: Optional[EvaluationRunPromptTemplate] = Field(
         default=None, description="""The prompt template used for inference."""
@@ -2439,7 +2440,7 @@ class EvaluationRunConfigDict(TypedDict, total=False):
     """The output config for the evaluation run."""
 
     autorater_config: Optional[genai_types.AutoraterConfigDict]
-    """The autorater config for the evaluation run."""
+    """The autorater config for the evaluation run. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored."""
 
     prompt_template: Optional[EvaluationRunPromptTemplateDict]
     """The prompt template used for inference."""
@@ -4746,7 +4747,8 @@ class _EvaluateInstancesRequestParameters(_common.BaseModel):
         default=None, description=""""""
     )
     autorater_config: Optional[genai_types.AutoraterConfig] = Field(
-        default=None, description=""""""
+        default=None,
+        description="""Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""",
     )
     metrics: Optional[list[Metric]] = Field(
         default=None,
@@ -4797,7 +4799,7 @@ class _EvaluateInstancesRequestParametersDict(TypedDict, total=False):
     """"""
 
     autorater_config: Optional[genai_types.AutoraterConfigDict]
-    """"""
+    """Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored."""
 
     metrics: Optional[list[MetricDict]]
     """The metrics used for evaluation.
@@ -19028,7 +19030,8 @@ class EvaluateDatasetRequestParameters(_common.BaseModel):
         default=None, description=""""""
     )
     autorater_config: Optional[genai_types.AutoraterConfig] = Field(
-        default=None, description=""""""
+        default=None,
+        description="""Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""",
     )
     config: Optional[EvaluateDatasetConfig] = Field(default=None, description="""""")
 
@@ -19046,7 +19049,7 @@ class EvaluateDatasetRequestParametersDict(TypedDict, total=False):
     """"""
 
     autorater_config: Optional[genai_types.AutoraterConfigDict]
-    """"""
+    """Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored."""
 
     config: Optional[EvaluateDatasetConfigDict]
     """"""