chore: GenAI Client(evals) - Pin all RubricMetric properties to v1

PiperOrigin-RevId: 919325887

Author: jsondai

agentplatform/_genai/_evals_metric_loaders.py

@@ -92,9 +92,9 @@ def _get_latest_version_uri(self, api_client: Any, metric_gcs_dir: str) -> str:
 
         blobs = gcs_utils.storage_client.list_blobs(bucket_name, prefix=prefix)
 
-        version_files: list[dict[str, Union[list[int], str]]] = (
-            []
-        )  # {'version_parts': [1,0,0], 'filename': 'v1.0.0.yaml'}
+        version_files: list[
+            dict[str, Union[list[int], str]]
+        ] = []  # {'version_parts': [1,0,0], 'filename': 'v1.0.0.yaml'}
 
         version_pattern = re.compile(
             r"v(\d+)(?:\.(\d+))?(?:\.(\d+))?\.(yaml|yml|json)$", re.IGNORECASE
@@ -277,27 +277,27 @@ def __getattr__(
 
     @property
     def GENERAL_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("GENERAL_QUALITY")
+        return self.__getattr__("GENERAL_QUALITY", version="v1")
 
     @property
     def TEXT_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("TEXT_QUALITY")
+        return self.__getattr__("TEXT_QUALITY", version="v1")
 
     @property
     def INSTRUCTION_FOLLOWING(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("INSTRUCTION_FOLLOWING")
+        return self.__getattr__("INSTRUCTION_FOLLOWING", version="v1")
 
     @property
     def SAFETY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("SAFETY")
+        return self.__getattr__("SAFETY", version="v1")
 
     @property
     def MULTI_TURN_GENERAL_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("MULTI_TURN_GENERAL_QUALITY")
+        return self.__getattr__("MULTI_TURN_GENERAL_QUALITY", version="v1")
 
     @property
     def MULTI_TURN_TEXT_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("MULTI_TURN_TEXT_QUALITY")
+        return self.__getattr__("MULTI_TURN_TEXT_QUALITY", version="v1")
 
     @property
     def MULTI_TURN_TOOL_USE_QUALITY(self) -> LazyLoadedPrebuiltMetric:
@@ -317,55 +317,72 @@ def FINAL_RESPONSE_MATCH(self) -> LazyLoadedPrebuiltMetric:
 
     @property
     def FINAL_RESPONSE_REFERENCE_FREE(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("FINAL_RESPONSE_REFERENCE_FREE")
+        return self.__getattr__("FINAL_RESPONSE_REFERENCE_FREE", version="v1")
 
     @property
     def COHERENCE(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("COHERENCE")
+        return self.__getattr__("COHERENCE", version="v1")
 
     @property
     def FLUENCY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("FLUENCY")
+        return self.__getattr__("FLUENCY", version="v1")
 
     @property
     def VERBOSITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("VERBOSITY")
+        return self.__getattr__("VERBOSITY", version="v1")
 
     @property
     def SUMMARIZATION_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("SUMMARIZATION_QUALITY")
+        return self.__getattr__("SUMMARIZATION_QUALITY", version="v1")
 
     @property
     def QUESTION_ANSWERING_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("QUESTION_ANSWERING_QUALITY")
+        return self.__getattr__("QUESTION_ANSWERING_QUALITY", version="v1")
 
     @property
     def MULTI_TURN_CHAT_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("MULTI_TURN_CHAT_QUALITY")
+        return self.__getattr__("MULTI_TURN_CHAT_QUALITY", version="v1")
 
     @property
     def MULTI_TURN_SAFETY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("MULTI_TURN_SAFETY")
+        return self.__getattr__("MULTI_TURN_SAFETY", version="v1")
 
     @property
     def FINAL_RESPONSE_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("FINAL_RESPONSE_QUALITY")
+        return self.__getattr__("FINAL_RESPONSE_QUALITY", version="v1")
 
     @property
     def HALLUCINATION(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("HALLUCINATION")
+        return self.__getattr__("HALLUCINATION", version="v1")
+
+    @property
+    def GROUNDING(self) -> LazyLoadedPrebuiltMetric:  # pylint: disable=invalid-name
+        return self.__getattr__("GROUNDING", version="v1")
+
+    @property
+    def GROUNDEDNESS(self) -> LazyLoadedPrebuiltMetric:  # pylint: disable=invalid-name
+        logger.warning(
+            "RubricMetric.GROUNDEDNESS is a deprecated alias and now maps to"
+            " RubricMetric.GROUNDING (grounding_v1). Note that the input"
+            " contract changed: legacy GROUNDEDNESS scored 'response' against"
+            " 'prompt'; grounding_v1 scores 'response' sentence-by-sentence"
+            " against an additional 'context' field. Add a 'context' field to"
+            " your dataset, otherwise scores will silently collapse to 0."
+            " Update your code to use RubricMetric.GROUNDING directly."
+        )
+        return self.__getattr__("GROUNDING", version="v1")
 
     @property
     def TOOL_USE_QUALITY(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("TOOL_USE_QUALITY")
+        return self.__getattr__("TOOL_USE_QUALITY", version="v1")
 
     @property
     def GECKO_TEXT2IMAGE(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("GECKO_TEXT2IMAGE")
+        return self.__getattr__("GECKO_TEXT2IMAGE", version="v1")
 
     @property
     def GECKO_TEXT2VIDEO(self) -> LazyLoadedPrebuiltMetric:
-        return self.__getattr__("GECKO_TEXT2VIDEO")
+        return self.__getattr__("GECKO_TEXT2VIDEO", version="v1")
 
 
 PrebuiltMetric = PrebuiltMetricLoader()

tests/unit/agentplatform/genai/test_evals.py

@@ -123,6 +123,7 @@ def mock_api_client_fixture():
 
 @pytest.fixture
 def mock_eval_dependencies(mock_api_client_fixture):
+    _evals_metric_loaders.LazyLoadedPrebuiltMetric._cache.clear()
     # fmt: off
     with (
         mock.patch("google.cloud.storage.Client") as mock_storage_client,
@@ -1239,9 +1240,9 @@ def test_enrich_scenario_e2e_simulation(self):
             api_response, eval_result
         )
         example = api_response.results[0].clusters[0].examples[0]
-        assert (
-            "scenario_preview" in example.evaluation_result
-        ), f"scenario_preview not found. evaluation_result keys: {list(example.evaluation_result.keys())}"
+        assert "scenario_preview" in example.evaluation_result, (
+            f"scenario_preview not found. evaluation_result keys: {list(example.evaluation_result.keys())}"
+        )
         assert (
             "I need to book a flight" in example.evaluation_result["scenario_preview"]
         )
@@ -1253,9 +1254,9 @@ def test_enrich_scenario_e2e_simulation(self):
         json_str = json.dumps(result_dump)
         parsed = json.loads(json_str)
         ex_parsed = parsed["results"][0]["clusters"][0]["examples"][0]
-        assert "scenario_preview" in ex_parsed.get(
-            "evaluation_result", {}
-        ), f"scenario_preview missing after serialization. Keys: {list(ex_parsed.get('evaluation_result', {}).keys())}"
+        assert "scenario_preview" in ex_parsed.get("evaluation_result", {}), (
+            f"scenario_preview missing after serialization. Keys: {list(ex_parsed.get('evaluation_result', {}).keys())}"
+        )
         assert (
             "I need to book a flight"
             in ex_parsed["evaluation_result"]["scenario_preview"]
@@ -6355,6 +6356,80 @@ def test_metric_to_yaml_file_raises_importerror_if_yaml_is_none(self):
             metric_obj.to_yaml_file("/fake/path/error.yaml")
 
 
+class TestPrebuiltMetricLoaderGroundedness:
+    """Unit tests for legacy RubricMetric.GROUNDEDNESS alias to grounding_v1."""
+
+    def test_grounding_resolves_to_grounding_v1(self):
+        lazy_metric = agentplatform_genai_types.RubricMetric.GROUNDING
+        assert lazy_metric.name == "GROUNDING"
+        assert lazy_metric._get_api_metric_spec_name() == "grounding_v1"
+
+    def test_groundedness_aliases_grounding_v1(self):
+        lazy_metric = agentplatform_genai_types.RubricMetric.GROUNDEDNESS
+        assert lazy_metric.name == "GROUNDING"
+        assert lazy_metric._get_api_metric_spec_name() == "grounding_v1"
+
+    def test_groundedness_logs_field_difference_warning(self, caplog):
+        loader_logger = (
+            "agentplatform._genai._evals_metric_loaders"
+        )
+        with caplog.at_level("WARNING", logger=loader_logger):
+            _ = agentplatform_genai_types.RubricMetric.GROUNDEDNESS
+        messages = [r.getMessage() for r in caplog.records if r.name == loader_logger]
+        assert any("GROUNDEDNESS" in m for m in messages)
+        assert any("grounding_v1" in m for m in messages)
+        assert any("context" in m for m in messages)
+
+    def test_groundedness_resolve_returns_grounding_v1_metric(self):
+        lazy_metric = agentplatform_genai_types.RubricMetric.GROUNDEDNESS
+        resolved = lazy_metric.resolve(api_client=mock.MagicMock())
+        assert isinstance(resolved, agentplatform_genai_types.Metric)
+        assert resolved.name == "grounding_v1"
+
+
+class TestPrebuiltMetricLoaderVersionPinning:
+    """Verifies explicit version pinning for all RubricMetric properties."""
+
+    @pytest.mark.parametrize(
+        "prop_name,expected_spec",
+        [
+            ("GENERAL_QUALITY", "general_quality_v1"),
+            ("TEXT_QUALITY", "text_quality_v1"),
+            ("INSTRUCTION_FOLLOWING", "instruction_following_v1"),
+            ("SAFETY", "safety_v1"),
+            ("MULTI_TURN_GENERAL_QUALITY", "multi_turn_general_quality_v1"),
+            ("MULTI_TURN_TEXT_QUALITY", "multi_turn_text_quality_v1"),
+            ("FINAL_RESPONSE_REFERENCE_FREE", "final_response_reference_free_v1"),
+            ("FINAL_RESPONSE_QUALITY", "final_response_quality_v1"),
+            ("HALLUCINATION", "hallucination_v1"),
+            ("TOOL_USE_QUALITY", "tool_use_quality_v1"),
+            ("GECKO_TEXT2IMAGE", "gecko_text2image_v1"),
+            ("GECKO_TEXT2VIDEO", "gecko_text2video_v1"),
+        ],
+    )
+    def test_predefined_property_pins_to_v1(self, prop_name, expected_spec):
+        lazy_metric = getattr(agentplatform_genai_types.RubricMetric, prop_name)
+        assert lazy_metric.version == "v1"
+        assert lazy_metric._get_api_metric_spec_name() == expected_spec
+
+    @pytest.mark.parametrize(
+        "prop_name",
+        [
+            "COHERENCE",
+            "FLUENCY",
+            "VERBOSITY",
+            "SUMMARIZATION_QUALITY",
+            "QUESTION_ANSWERING_QUALITY",
+            "MULTI_TURN_CHAT_QUALITY",
+            "MULTI_TURN_SAFETY",
+        ],
+    )
+    def test_gcs_backed_property_pins_to_v1(self, prop_name):
+        lazy_metric = getattr(agentplatform_genai_types.RubricMetric, prop_name)
+        assert lazy_metric.version == "v1"
+        assert lazy_metric._get_api_metric_spec_name() is None
+
+
 class TestMergeResponseDatasets:
     """Unit tests for the merge_response_datasets_into_canonical_format function."""
 
@@ -8310,13 +8385,11 @@ def test_execute_evaluation_deduplicates_candidate_names(
             candidate_name="gemini-pro",
         )
 
-        mock_eval_dependencies["mock_evaluate_instances"].return_value = (
-            agentplatform_genai_types.EvaluateInstancesResponse(
-                exact_match_results=agentplatform_genai_types.ExactMatchResults(
-                    exact_match_metric_values=[
-                        genai_types.ExactMatchMetricValue(score=1.0)
-                    ]
-                )
+        mock_eval_dependencies[
+            "mock_evaluate_instances"
+        ].return_value = agentplatform_genai_types.EvaluateInstancesResponse(
+            exact_match_results=agentplatform_genai_types.ExactMatchResults(
+                exact_match_metric_values=[genai_types.ExactMatchMetricValue(score=1.0)]
             )
         )