chore: GenAI Client(evals) - normalize inference model names in create_evaluation_run

The Evaluation Service routes server-side inference by parsing the
EvaluationRunInferenceConfig.model string into a publisher-model or
endpoint resource name, from which it extracts the serving location.
Names that are not fully qualified fail server-side in two ways:

- A short name (e.g. `gemini-2.5-flash`) is rejected by API validation
  with a clear 400 INVALID_ARGUMENT error.
- A location-less path (e.g. `publishers/google/models/gemini-2.5-flash`)
  passes validation but has no location for the server to route on, so it
  fails later with an opaque `INTERNAL: Internal error occurred.` that is
  impossible to debug from the client.

The SDK previously passed `model` through verbatim. This change makes the
SDK guard the input: model names in inference_configs are normalized to a
fully-qualified `projects/{project}/locations/{location}/...` resource
name using the client's project and location before the request is sent.
Short Gemini names, location-less publisher/endpoint/tuned-model paths,
and `models/{id}` are expanded; already fully-qualified names pass
through. Unrecognized inputs and a client missing project/location raise
a clear client-side ValueError.

Also corrects the create_evaluation_run docstring and the
EvaluationRunInferenceConfig.model field doc, which incorrectly showed a
short model name as a working example.

PiperOrigin-RevId: 926248584

Author: jsondai

agentplatform/_genai/_evals_common.py

@@ -513,9 +513,22 @@ def _resolve_inference_configs(
             else:
                 config.agent_configs = parsed_agent_info.agents
 
-    # Resolve prompt template data
+    # Normalize model names and resolve prompt template data.
     if inference_configs:
         for inference_config in inference_configs.values():
+            model_val = (
+                inference_config.get("model")
+                if isinstance(inference_config, dict)
+                else inference_config.model
+            )
+            if model_val:
+                normalized_model = _normalize_inference_model_name(
+                    model_val, api_client
+                )
+                if isinstance(inference_config, dict):
+                    inference_config["model"] = normalized_model
+                else:
+                    inference_config.model = normalized_model
             prompt_template_val = (
                 inference_config.get("prompt_template")
                 if isinstance(inference_config, dict)
@@ -912,6 +925,92 @@ def _is_gemini_model(model: str) -> bool:
     )
 
 
+def _normalize_inference_model_name(model: str, api_client: BaseApiClient) -> str:
+    """Returns a fully-qualified model resource name for server-side inference.
+
+    The Evaluation Service routes inference by parsing the model string into a
+    publisher-model or endpoint resource name, from which it extracts the
+    serving location. Names that are not fully qualified fail server-side in two
+    ways:
+
+    - A short name (e.g. ``gemini-2.5-flash``) is rejected by API validation
+      with a clear 400 ``INVALID_ARGUMENT`` error.
+    - A location-less path (e.g. ``publishers/google/models/gemini-2.5-flash``)
+      passes validation but has no location for the server to route on, so it
+      fails later with an opaque ``INTERNAL`` error that is impossible to debug
+      from the client.
+
+    This helper expands both forms to the fully-qualified resource name using
+    the client's project and location so the value sent to the server is always
+    routable.
+
+    Args:
+        model: The model name from an ``EvaluationRunInferenceConfig``. May be a
+          short Gemini name, a location-less publisher/endpoint/tuned-model
+          path, or an already fully-qualified ``projects/.../locations/...``
+          resource name.
+        api_client: The API client, used to source the project and location for
+          expansion.
+
+    Returns:
+        A fully-qualified model resource name.
+
+    Raises:
+        ValueError: If ``model`` needs expansion but the client is missing a
+          project or location, or if ``model`` is not a recognized Vertex model
+          form.
+    """
+    if not model:
+        return model
+
+    # Already fully-qualified; nothing to do.
+    if model.startswith("projects/"):
+        return model
+
+    project = getattr(api_client, "project", None)
+    location = getattr(api_client, "location", None)
+    prefix = f"projects/{project}/locations/{location}/"
+
+    def _require_project_location() -> None:
+        if not project or not location:
+            raise ValueError(
+                f"Cannot expand model name '{model}' to a fully-qualified"
+                " resource name because the client is missing a project or"
+                " location. Set project and location on the client, or pass a"
+                " fully-qualified"
+                " 'projects/{project}/locations/{location}/publishers/google/models/{model}'"
+                " resource name."
+            )
+
+    # Location-less Vertex resource paths: prepend project/location.
+    if (
+        model.startswith("publishers/")
+        or model.startswith("endpoints/")
+        or model.startswith("tunedModels/")
+    ):
+        _require_project_location()
+        return f"{prefix}{model}"
+
+    # Bare 'models/{id}' canonicalizes to the publisher path.
+    if model.startswith("models/"):
+        _require_project_location()
+        return f"{prefix}publishers/google/{model}"
+
+    # Bare short Gemini name, e.g. 'gemini-2.5-flash'.
+    if "/" not in model and _is_gemini_model(model):
+        _require_project_location()
+        return f"{prefix}publishers/google/models/{model}"
+
+    raise ValueError(
+        f"Unrecognized model name '{model}'. Provide a Gemini model name (e.g."
+        " 'gemini-2.5-flash'), or a fully-qualified publisher-model or endpoint"
+        " resource name (e.g."
+        " 'projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash'"
+        " or"
+        " 'projects/{project}/locations/{location}/endpoints/{endpoint}')."
+    )
+
+
 def _run_inference_internal(
     api_client: BaseApiClient,
     model: Union[Callable[[Any], Any], str],

agentplatform/_genai/evals.py

@@ -2659,6 +2659,11 @@ def create_evaluation_run(
               The key is the candidate name, and the value is the inference config.
               If provided, `agent_info` must be None. If omitted and `agent_info` is provided,
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
+              The `model` field of an inference config accepts a short Gemini model
+              name (e.g. `gemini-2.5-flash`), which is automatically expanded to a
+              fully-qualified resource name using the client's project and location,
+              or an already fully-qualified publisher-model or endpoint resource
+              name.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
           labels: The labels to apply to the evaluation run.
@@ -4448,6 +4453,11 @@ async def create_evaluation_run(
               The key is the candidate name, and the value is the inference config.
               If provided, `agent_info` must be None. If omitted and `agent_info` is provided,
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
+              The `model` field of an inference config accepts a short Gemini model
+              name (e.g. `gemini-2.5-flash`), which is automatically expanded to a
+              fully-qualified resource name using the client's project and location,
+              or an already fully-qualified publisher-model or endpoint resource
+              name.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
           red_teaming_config: This field is experimental and may change in future

agentplatform/_genai/types/common.py

@@ -2586,7 +2586,7 @@ class EvaluationRunInferenceConfig(_common.BaseModel):
     )
     model: Optional[str] = Field(
         default=None,
-        description="""The fully qualified name of the publisher model or endpoint to use for inference.""",
+        description="""The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`).""",
     )
     prompt_template: Optional[EvaluationRunPromptTemplate] = Field(
         default=None, description="""The prompt template used for inference."""
@@ -2611,7 +2611,7 @@ class EvaluationRunInferenceConfigDict(TypedDict, total=False):
     """The agent config."""
 
     model: Optional[str]
-    """The fully qualified name of the publisher model or endpoint to use for inference."""
+    """The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`)."""
 
     prompt_template: Optional[EvaluationRunPromptTemplateDict]
     """The prompt template used for inference."""

tests/unit/agentplatform/genai/test_evals.py

@@ -228,6 +228,81 @@ def test_get_api_client_with_none_location(
         mock_agentplatform_client.assert_not_called()
 
 
+class TestNormalizeInferenceModelName:
+    _FQ_PREFIX = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/"
+
+    def test_short_gemini_name_expanded(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "gemini-2.5-flash", mock_api_client_fixture
+        )
+        assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash")
+
+    def test_location_less_publisher_path_prepended(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "publishers/google/models/gemini-2.5-flash", mock_api_client_fixture
+        )
+        assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash")
+
+    def test_third_party_publisher_path_prepended(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "publishers/anthropic/models/claude-sonnet", mock_api_client_fixture
+        )
+        assert result == (f"{self._FQ_PREFIX}publishers/anthropic/models/claude-sonnet")
+
+    def test_endpoint_path_prepended(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "endpoints/123", mock_api_client_fixture
+        )
+        assert result == f"{self._FQ_PREFIX}endpoints/123"
+
+    def test_models_path_canonicalized(self, mock_api_client_fixture):
+        result = _evals_common._normalize_inference_model_name(
+            "models/gemini-2.5-flash", mock_api_client_fixture
+        )
+        assert result == (f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash")
+
+    def test_fully_qualified_name_unchanged(self, mock_api_client_fixture):
+        fq = f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash"
+        assert (
+            _evals_common._normalize_inference_model_name(fq, mock_api_client_fixture)
+            == fq
+        )
+
+    def test_empty_model_unchanged(self, mock_api_client_fixture):
+        assert (
+            _evals_common._normalize_inference_model_name("", mock_api_client_fixture)
+            == ""
+        )
+
+    def test_missing_project_or_location_raises(self, mock_api_client_fixture):
+        mock_api_client_fixture.location = None
+        with pytest.raises(ValueError, match="missing a project or location"):
+            _evals_common._normalize_inference_model_name(
+                "gemini-2.5-flash", mock_api_client_fixture
+            )
+
+    def test_unrecognized_model_raises(self, mock_api_client_fixture):
+        with pytest.raises(ValueError, match="Unrecognized model name"):
+            _evals_common._normalize_inference_model_name(
+                "not/a/valid/model", mock_api_client_fixture
+            )
+
+    def test_resolve_inference_configs_normalizes_model(self, mock_api_client_fixture):
+        inference_configs = {
+            "candidate-1": agentplatform_genai_types.EvaluationRunInferenceConfig(
+                model="gemini-2.5-flash"
+            )
+        }
+        result = _evals_common._resolve_inference_configs(
+            mock_api_client_fixture,
+            common_types.EvaluationRunDataSource(),
+            inference_configs,
+        )
+        assert result["candidate-1"].model == (
+            f"{self._FQ_PREFIX}publishers/google/models/gemini-2.5-flash"
+        )
+
+
 class TestTransformers:
     """Unit tests for transformers."""
 

vertexai/_genai/_evals_common.py

@@ -513,9 +513,22 @@ def _resolve_inference_configs(
             else:
                 config.agent_configs = parsed_agent_info.agents
 
-    # Resolve prompt template data
+    # Normalize model names and resolve prompt template data.
     if inference_configs:
         for inference_config in inference_configs.values():
+            model_val = (
+                inference_config.get("model")
+                if isinstance(inference_config, dict)
+                else inference_config.model
+            )
+            if model_val:
+                normalized_model = _normalize_inference_model_name(
+                    model_val, api_client
+                )
+                if isinstance(inference_config, dict):
+                    inference_config["model"] = normalized_model
+                else:
+                    inference_config.model = normalized_model
             prompt_template_val = (
                 inference_config.get("prompt_template")
                 if isinstance(inference_config, dict)
@@ -912,6 +925,92 @@ def _is_gemini_model(model: str) -> bool:
     )
 
 
+def _normalize_inference_model_name(model: str, api_client: BaseApiClient) -> str:
+    """Returns a fully-qualified model resource name for server-side inference.
+
+    The Evaluation Service routes inference by parsing the model string into a
+    publisher-model or endpoint resource name, from which it extracts the
+    serving location. Names that are not fully qualified fail server-side in two
+    ways:
+
+    - A short name (e.g. ``gemini-2.5-flash``) is rejected by API validation
+      with a clear 400 ``INVALID_ARGUMENT`` error.
+    - A location-less path (e.g. ``publishers/google/models/gemini-2.5-flash``)
+      passes validation but has no location for the server to route on, so it
+      fails later with an opaque ``INTERNAL`` error that is impossible to debug
+      from the client.
+
+    This helper expands both forms to the fully-qualified resource name using
+    the client's project and location so the value sent to the server is always
+    routable.
+
+    Args:
+        model: The model name from an ``EvaluationRunInferenceConfig``. May be a
+          short Gemini name, a location-less publisher/endpoint/tuned-model
+          path, or an already fully-qualified ``projects/.../locations/...``
+          resource name.
+        api_client: The API client, used to source the project and location for
+          expansion.
+
+    Returns:
+        A fully-qualified model resource name.
+
+    Raises:
+        ValueError: If ``model`` needs expansion but the client is missing a
+          project or location, or if ``model`` is not a recognized Vertex model
+          form.
+    """
+    if not model:
+        return model
+
+    # Already fully-qualified; nothing to do.
+    if model.startswith("projects/"):
+        return model
+
+    project = getattr(api_client, "project", None)
+    location = getattr(api_client, "location", None)
+    prefix = f"projects/{project}/locations/{location}/"
+
+    def _require_project_location() -> None:
+        if not project or not location:
+            raise ValueError(
+                f"Cannot expand model name '{model}' to a fully-qualified"
+                " resource name because the client is missing a project or"
+                " location. Set project and location on the client, or pass a"
+                " fully-qualified"
+                " 'projects/{project}/locations/{location}/publishers/google/models/{model}'"
+                " resource name."
+            )
+
+    # Location-less Vertex resource paths: prepend project/location.
+    if (
+        model.startswith("publishers/")
+        or model.startswith("endpoints/")
+        or model.startswith("tunedModels/")
+    ):
+        _require_project_location()
+        return f"{prefix}{model}"
+
+    # Bare 'models/{id}' canonicalizes to the publisher path.
+    if model.startswith("models/"):
+        _require_project_location()
+        return f"{prefix}publishers/google/{model}"
+
+    # Bare short Gemini name, e.g. 'gemini-2.5-flash'.
+    if "/" not in model and _is_gemini_model(model):
+        _require_project_location()
+        return f"{prefix}publishers/google/models/{model}"
+
+    raise ValueError(
+        f"Unrecognized model name '{model}'. Provide a Gemini model name (e.g."
+        " 'gemini-2.5-flash'), or a fully-qualified publisher-model or endpoint"
+        " resource name (e.g."
+        " 'projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash'"
+        " or"
+        " 'projects/{project}/locations/{location}/endpoints/{endpoint}')."
+    )
+
+
 def _run_inference_internal(
     api_client: BaseApiClient,
     model: Union[Callable[[Any], Any], str],

vertexai/_genai/evals.py

@@ -2659,6 +2659,11 @@ def create_evaluation_run(
               The key is the candidate name, and the value is the inference config.
               If provided, `agent_info` must be None. If omitted and `agent_info` is provided,
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
+              The `model` field of an inference config accepts a short Gemini model
+              name (e.g. `gemini-2.5-flash`), which is automatically expanded to a
+              fully-qualified resource name using the client's project and location,
+              or an already fully-qualified publisher-model or endpoint resource
+              name.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
           labels: The labels to apply to the evaluation run.
@@ -4448,6 +4453,11 @@ async def create_evaluation_run(
               The key is the candidate name, and the value is the inference config.
               If provided, `agent_info` must be None. If omitted and `agent_info` is provided,
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
+              The `model` field of an inference config accepts a short Gemini model
+              name (e.g. `gemini-2.5-flash`), which is automatically expanded to a
+              fully-qualified resource name using the client's project and location,
+              or an already fully-qualified publisher-model or endpoint resource
+              name.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
           red_teaming_config: This field is experimental and may change in future

vertexai/_genai/types/common.py

@@ -2549,7 +2549,7 @@ class EvaluationRunInferenceConfig(_common.BaseModel):
     )
     model: Optional[str] = Field(
         default=None,
-        description="""The fully qualified name of the publisher model or endpoint to use for inference.""",
+        description="""The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`).""",
     )
     prompt_template: Optional[EvaluationRunPromptTemplate] = Field(
         default=None, description="""The prompt template used for inference."""
@@ -2574,7 +2574,7 @@ class EvaluationRunInferenceConfigDict(TypedDict, total=False):
     """The agent config."""
 
     model: Optional[str]
-    """The fully qualified name of the publisher model or endpoint to use for inference."""
+    """The model to use for inference. Accepts a short Gemini model name (e.g. `gemini-2.5-flash`), which is automatically expanded to a fully-qualified resource name using the client's project and location, or an already fully-qualified publisher-model or endpoint resource name (e.g. `projects/{project}/locations/{location}/publishers/google/models/gemini-2.5-flash`)."""
 
     prompt_template: Optional[EvaluationRunPromptTemplateDict]
     """The prompt template used for inference."""