chore: GenAI SDK client - add more replay tests for eval

PiperOrigin-RevId: 782058943

Author: sararob

tests/unit/vertexai/genai/replays/test_evaluate.py

@@ -0,0 +1,61 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pylint: disable=protected-access,bad-continuation,missing-function-docstring
+
+from tests.unit.vertexai.genai.replays import pytest_helper
+from vertexai._genai import types
+import pandas as pd
+
+
+def test_evaluation_result(client):
+    """Tests that evaluate() produces a correctly structured EvaluationResult."""
+    prompts_df = pd.DataFrame({"prompt": ["What is Taylor Swift's most recent album?"]})
+
+    eval_dataset = client.evals.run_inference(
+        model="gemini-2.5-flash",
+        src=prompts_df,
+    )
+
+    metrics_to_run = [
+        types.PrebuiltMetric.TEXT_QUALITY,
+    ]
+
+    evaluation_result = client.evals.evaluate(
+        dataset=eval_dataset,
+        metrics=metrics_to_run,
+    )
+
+    assert isinstance(evaluation_result, types.EvaluationResult)
+
+    assert evaluation_result.summary_metrics is not None
+    assert len(evaluation_result.summary_metrics) > 0
+    for summary in evaluation_result.summary_metrics:
+        assert isinstance(summary, types.AggregatedMetricResult)
+        assert summary.metric_name is not None
+        assert summary.mean_score is not None
+
+    assert evaluation_result.eval_case_results is not None
+    assert len(evaluation_result.eval_case_results) > 0
+    for case_result in evaluation_result.eval_case_results:
+        assert isinstance(case_result, types.EvalCaseResult)
+        assert case_result.eval_case_index is not None
+        assert case_result.response_candidate_results is not None
+
+
+pytestmark = pytest_helper.setup(
+    file=__file__,
+    globals_for_file=globals(),
+    test_method="evals.evaluate",
+)

tests/unit/vertexai/genai/replays/test_evaluate_instances.py

@@ -18,6 +18,7 @@
 from tests.unit.vertexai.genai.replays import pytest_helper
 from vertexai._genai import types
 import pandas as pd
+import json
 
 
 def test_bleu_metric(client):
@@ -34,6 +35,78 @@ def test_bleu_metric(client):
     assert len(response.bleu_results.bleu_metric_values) == 1
 
 
+def test_exact_match_metric(client):
+    """Tests the _evaluate_instances method with ExactMatchInput."""
+    test_exact_match_input = types.ExactMatchInput(
+        instances=[
+            types.ExactMatchInstance(
+                prediction="The quick brown fox jumps over the lazy dog.",
+                reference="The quick brown fox jumps over the lazy dog.",
+            )
+        ],
+        metric_spec=types.ExactMatchSpec(),
+    )
+    response = client.evals._evaluate_instances(
+        exact_match_input=test_exact_match_input
+    )
+    assert len(response.exact_match_results.exact_match_metric_values) == 1
+
+
+def test_rouge_metric(client):
+    """Tests the _evaluate_instances method with RougeInput."""
+    test_rouge_input = types.RougeInput(
+        instances=[
+            types.RougeInstance(
+                prediction="A fast brown fox leaps over a lazy dog.",
+                reference="The quick brown fox jumps over the lazy dog.",
+            )
+        ],
+        metric_spec=types.RougeSpec(rouge_type="rougeL"),
+    )
+    response = client.evals._evaluate_instances(rouge_input=test_rouge_input)
+    assert len(response.rouge_results.rouge_metric_values) == 1
+
+
+def test_pointwise_metric(client):
+    """Tests the _evaluate_instances method with PointwiseMetricInput."""
+    instance_dict = {"prompt": "What is the capital of France?", "response": "Paris"}
+    json_instance = json.dumps(instance_dict)
+
+    test_input = types.PointwiseMetricInput(
+        instance=types.PointwiseMetricInstance(json_instance=json_instance),
+        metric_spec=types.PointwiseMetricSpec(
+            metric_prompt_template="Evaluate if the response '{response}' correctly answers the prompt '{prompt}'."
+        ),
+    )
+    response = client.evals._evaluate_instances(pointwise_metric_input=test_input)
+    assert response.pointwise_metric_result is not None
+    assert response.pointwise_metric_result.score is not None
+
+
+def test_pairwise_metric_with_autorater(client):
+    """Tests the _evaluate_instances method with PairwiseMetricInput and AutoraterConfig."""
+
+    instance_dict = {
+        "baseline_response": "Short summary.",
+        "candidate_response": "A longer, more detailed summary.",
+    }
+    json_instance = json.dumps(instance_dict)
+
+    test_input = types.PairwiseMetricInput(
+        instance=types.PairwiseMetricInstance(json_instance=json_instance),
+        metric_spec=types.PairwiseMetricSpec(
+            metric_prompt_template="Which response is a better summary? Baseline: '{baseline_response}' or Candidate: '{candidate_response}'"
+        ),
+    )
+    autorater_config = types.AutoraterConfig(sampling_count=2)
+
+    response = client.evals._evaluate_instances(
+        pairwise_metric_input=test_input, autorater_config=autorater_config
+    )
+    assert response.pairwise_metric_result is not None
+    assert response.pairwise_metric_result.pairwise_choice is not None
+
+
 def test_run_inference_with_string_model(client):
     test_df = pd.DataFrame({"prompt": ["test prompt"]})