chore: GenAI Client(evals) - normalize inference model names in create_evaluation_run

PiperOrigin-RevId: 926988958

Author: jsondai

agentplatform/_genai/_evals_common.py

@@ -513,9 +513,21 @@ def _resolve_inference_configs(
             else:
                 config.agent_configs = parsed_agent_info.agents
 
-    # Resolve prompt template data
     if inference_configs:
         for inference_config in inference_configs.values():
+            model_val = (
+                inference_config.get("model")
+                if isinstance(inference_config, dict)
+                else inference_config.model
+            )
+            if model_val:
+                normalized_model = _normalize_inference_model_name(
+                    model_val, api_client
+                )
+                if isinstance(inference_config, dict):
+                    inference_config["model"] = normalized_model
+                else:
+                    inference_config.model = normalized_model
             prompt_template_val = (
                 inference_config.get("prompt_template")
                 if isinstance(inference_config, dict)
@@ -912,6 +924,62 @@ def _is_gemini_model(model: str) -> bool:
     )
 
 
+def _normalize_inference_model_name(model: str, api_client: BaseApiClient) -> str:
+    """Expands a model name to a fully-qualified resource name for inference.
+
+    A short or location-less model name has no serving location for the
+    Evaluation Service to route on, so it is expanded using the client's
+    project and location. Already fully-qualified names pass through. Raises
+    ValueError if the client is missing a project or location, or if the model
+    name is not a recognized Vertex form.
+    """
+    if not model:
+        return model
+
+    if model.startswith("projects/"):
+        return model
+
+    project = getattr(api_client, "project", None)
+    location = getattr(api_client, "location", None)
+    prefix = f"projects/{project}/locations/{location}/"
+
+    def _require_project_location() -> None:
+        if not project or not location:
+            raise ValueError(
+                f"Cannot expand model name '{model}' to a fully-qualified"
+                " resource name because the client is missing a project or"
+                " location. Set project and location on the client, or pass a"
+                " fully-qualified"
+                " 'projects/{project}/locations/{location}/publishers/google/models/{model}'"
+                " resource name."
+            )
+
+    if (
+        model.startswith("publishers/")
+        or model.startswith("endpoints/")
+        or model.startswith("tunedModels/")
+    ):
+        _require_project_location()
+        return f"{prefix}{model}"
+
+    if model.startswith("models/"):
+        _require_project_location()
+        return f"{prefix}publishers/google/{model}"
+
+    if "/" not in model and _is_gemini_model(model):
+        _require_project_location()
+        return f"{prefix}publishers/google/models/{model}"
+
+    raise ValueError(
+        f"Unrecognized model name '{model}'. Provide a Gemini model name (e.g."
+        " 'gemini-2.5-flash'), or a fully-qualified publisher-model or endpoint"
+        " resource name (e.g."
+        " 'projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash'"
+        " or"
+        " 'projects/{project}/locations/{location}/endpoints/{endpoint}')."
+    )
+
+
 def _run_inference_internal(
     api_client: BaseApiClient,
     model: Union[Callable[[Any], Any], str],

agentplatform/_genai/evals.py

@@ -2659,6 +2659,11 @@ def create_evaluation_run(
               The key is the candidate name, and the value is the inference config.
               If provided, `agent_info` must be None. If omitted and `agent_info` is provided,
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
+              The `model` field of an inference config accepts a short Gemini model
+              name (e.g. `gemini-2.5-flash`), which is automatically expanded to a
+              fully-qualified resource name using the client's project and location,
+              or an already fully-qualified publisher-model or endpoint resource
+              name.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
           labels: The labels to apply to the evaluation run.
@@ -4448,6 +4453,11 @@ async def create_evaluation_run(
               The key is the candidate name, and the value is the inference config.
               If provided, `agent_info` must be None. If omitted and `agent_info` is provided,
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
+              The `model` field of an inference config accepts a short Gemini model
+              name (e.g. `gemini-2.5-flash`), which is automatically expanded to a
+              fully-qualified resource name using the client's project and location,
+              or an already fully-qualified publisher-model or endpoint resource
+              name.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
           red_teaming_config: This field is experimental and may change in future

agentplatform/_genai/types/common.py

@@ -2586,7 +2586,7 @@ class EvaluationRunInferenceConfig(_common.BaseModel):
     )
     model: Optional[str] = Field(
         default=None,
-        description="""The fully qualified name of the publisher model or endpoint to use for inference.""",
+        description="""The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`).""",
     )
     prompt_template: Optional[EvaluationRunPromptTemplate] = Field(
         default=None, description="""The prompt template used for inference."""
@@ -2611,7 +2611,7 @@ class EvaluationRunInferenceConfigDict(TypedDict, total=False):
     """The agent config."""
 
     model: Optional[str]
-    """The fully qualified name of the publisher model or endpoint to use for inference."""
+    """The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`)."""
 
     prompt_template: Optional[EvaluationRunPromptTemplateDict]
     """The prompt template used for inference."""

tests/unit/agentplatform/genai/test_evals.py

@@ -228,6 +228,81 @@ def test_get_api_client_with_none_location(
         mock_agentplatform_client.assert_not_called()
 
 
+class TestNormalizeInferenceModelName:
+    _FQ_PREFIX = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/"
+
+    def test_short_gemini_name_expanded(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "gemini-2.5-flash", mock_api_client_fixture
+        )
+        assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash")
+
+    def test_location_less_publisher_path_prepended(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "publishers/google/models/gemini-2.5-flash", mock_api_client_fixture
+        )
+        assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash")
+
+    def test_third_party_publisher_path_prepended(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "publishers/anthropic/models/claude-sonnet", mock_api_client_fixture
+        )
+        assert result == (f"{self._FQ_PREFIX}publishers/anthropic/models/claude-sonnet")
+
+    def test_endpoint_path_prepended(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "endpoints/123", mock_api_client_fixture
+        )
+        assert result == f"{self._FQ_PREFIX}endpoints/123"
+
+    def test_models_path_canonicalized(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "models/gemini-2.5-flash", mock_api_client_fixture
+        )
+        assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash")
+
+    def test_fully_qualified_name_unchanged(self, mock_api_client_fixture):
+        fq = f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash"
+        assert (
+            _evals_common._normalize_inference_model_name(fq, mock_api_client_fixture)
+            == fq
+        )
+
+    def test_empty_model_unchanged(self, mock_api_client_fixture):
+        assert (
+            _evals_common._normalize_inference_model_name("", mock_api_client_fixture)
+            == ""
+        )
+
+    def test_missing_project_or_location_raises(self, mock_api_client_fixture):
+        mock_api_client_fixture.location = None
+        with pytest.raises(ValueError, match="missing a project or location"):
+            _evals_common._normalize_inference_model_name(
+                "gemini-2.5-flash", mock_api_client_fixture
+            )
+
+    def test_unrecognized_model_raises(self, mock_api_client_fixture):
+        with pytest.raises(ValueError, match="Unrecognized model name"):
+            _evals_common._normalize_inference_model_name(
+                "not/a/valid/model", mock_api_client_fixture
+            )
+
+    def test_resolve_inference_configs_normalizes_model(self, mock_api_client_fixture):
+        inference_configs = {
+            "candidate-1": agentplatform_genai_types.EvaluationRunInferenceConfig(
+                model="gemini-2.5-flash"
+            )
+        }
+        result = _evals_common._resolve_inference_configs(
+            mock_api_client_fixture,
+            common_types.EvaluationRunDataSource(),
+            inference_configs,
+        )
+        assert result["candidate-1"].model == (
+            f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash"
+        )
+
+
 class TestTransformers:
     """Unit tests for transformers."""
 

vertexai/_genai/_evals_common.py

@@ -513,9 +513,21 @@ def _resolve_inference_configs(
             else:
                 config.agent_configs = parsed_agent_info.agents
 
-    # Resolve prompt template data
     if inference_configs:
         for inference_config in inference_configs.values():
+            model_val = (
+                inference_config.get("model")
+                if isinstance(inference_config, dict)
+                else inference_config.model
+            )
+            if model_val:
+                normalized_model = _normalize_inference_model_name(
+                    model_val, api_client
+                )
+                if isinstance(inference_config, dict):
+                    inference_config["model"] = normalized_model
+                else:
+                    inference_config.model = normalized_model
             prompt_template_val = (
                 inference_config.get("prompt_template")
                 if isinstance(inference_config, dict)
@@ -912,6 +924,62 @@ def _is_gemini_model(model: str) -> bool:
     )
 
 
+def _normalize_inference_model_name(model: str, api_client: BaseApiClient) -> str:
+    """Expands a model name to a fully-qualified resource name for inference.
+
+    A short or location-less model name has no serving location for the
+    Evaluation Service to route on, so it is expanded using the client's
+    project and location. Already fully-qualified names pass through. Raises
+    ValueError if the client is missing a project or location, or if the model
+    name is not a recognized Vertex form.
+    """
+    if not model:
+        return model
+
+    if model.startswith("projects/"):
+        return model
+
+    project = getattr(api_client, "project", None)
+    location = getattr(api_client, "location", None)
+    prefix = f"projects/{project}/locations/{location}/"
+
+    def _require_project_location() -> None:
+        if not project or not location:
+            raise ValueError(
+                f"Cannot expand model name '{model}' to a fully-qualified"
+                " resource name because the client is missing a project or"
+                " location. Set project and location on the client, or pass a"
+                " fully-qualified"
+                " 'projects/{project}/locations/{location}/publishers/google/models/{model}'"
+                " resource name."
+            )
+
+    if (
+        model.startswith("publishers/")
+        or model.startswith("endpoints/")
+        or model.startswith("tunedModels/")
+    ):
+        _require_project_location()
+        return f"{prefix}{model}"
+
+    if model.startswith("models/"):
+        _require_project_location()
+        return f"{prefix}publishers/google/{model}"
+
+    if "/" not in model and _is_gemini_model(model):
+        _require_project_location()
+        return f"{prefix}publishers/google/models/{model}"
+
+    raise ValueError(
+        f"Unrecognized model name '{model}'. Provide a Gemini model name (e.g."
+        " 'gemini-2.5-flash'), or a fully-qualified publisher-model or endpoint"
+        " resource name (e.g."
+        " 'projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash'"
+        " or"
+        " 'projects/{project}/locations/{location}/endpoints/{endpoint}')."
+    )
+
+
 def _run_inference_internal(
     api_client: BaseApiClient,
     model: Union[Callable[[Any], Any], str],

vertexai/_genai/evals.py

@@ -2659,6 +2659,11 @@ def create_evaluation_run(
               The key is the candidate name, and the value is the inference config.
               If provided, `agent_info` must be None. If omitted and `agent_info` is provided,
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
+              The `model` field of an inference config accepts a short Gemini model
+              name (e.g. `gemini-2.5-flash`), which is automatically expanded to a
+              fully-qualified resource name using the client's project and location,
+              or an already fully-qualified publisher-model or endpoint resource
+              name.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
           labels: The labels to apply to the evaluation run.
@@ -4448,6 +4453,11 @@ async def create_evaluation_run(
               The key is the candidate name, and the value is the inference config.
               If provided, `agent_info` must be None. If omitted and `agent_info` is provided,
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
+              The `model` field of an inference config accepts a short Gemini model
+              name (e.g. `gemini-2.5-flash`), which is automatically expanded to a
+              fully-qualified resource name using the client's project and location,
+              or an already fully-qualified publisher-model or endpoint resource
+              name.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
           red_teaming_config: This field is experimental and may change in future

vertexai/_genai/types/common.py

@@ -2549,7 +2549,7 @@ class EvaluationRunInferenceConfig(_common.BaseModel):
     )
     model: Optional[str] = Field(
         default=None,
-        description="""The fully qualified name of the publisher model or endpoint to use for inference.""",
+        description="""The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`).""",
     )
     prompt_template: Optional[EvaluationRunPromptTemplate] = Field(
         default=None, description="""The prompt template used for inference."""
@@ -2574,7 +2574,7 @@ class EvaluationRunInferenceConfigDict(TypedDict, total=False):
     """The agent config."""
 
     model: Optional[str]
-    """The fully qualified name of the publisher model or endpoint to use for inference."""
+    """The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`)."""
 
     prompt_template: Optional[EvaluationRunPromptTemplateDict]
     """The prompt template used for inference."""