feat: GenAI Client(evals) - Add red_teaming_config support for create_evaluation_run

PiperOrigin-RevId: 914075746

Author: vertex-sdk-bot

tests/unit/vertexai/genai/replays/test_create_evaluation_run.py

@@ -614,6 +614,7 @@ def test_create_eval_run_with_metric_resource_name(mock_uuid4, client):
 #         assert eval_item.evaluation_request.candidate_responses == []
 #     assert evaluation_run.error is None
 
+
 # def test_create_eval_run_data_source_evaluation_dataset_with_agent_info_and_prompt_template_data(
 #     client,
 # ):
@@ -708,6 +709,35 @@ def test_create_eval_run_with_metric_resource_name(mock_uuid4, client):
 #             == INPUT_DF_WITH_CONTEXT_AND_HISTORY.iloc[i]["response"]
 #         )
 #     assert evaluation_run.error is None
+def test_create_eval_run_with_red_teaming_config(client):
+    """Tests that create_evaluation_run() with red_teaming_config sends analysisConfigs."""
+    evaluation_run = client.evals.create_evaluation_run(
+        name="test_red_teaming",
+        display_name="test_red_teaming",
+        dataset=types.EvaluationRunDataSource(evaluation_set=EVAL_SET_NAME),
+        dest=GCS_DEST,
+        metrics=[],
+        red_teaming_config=types.RedTeamingAnalysisConfig(
+            attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"],
+            vulnerable_tools=[
+                types.VulnerableTool(
+                    tool_name="search_flights",
+                    json_paths=["$.flights[0].description"],
+                ),
+            ],
+        ),
+    )
+    assert isinstance(evaluation_run, types.EvaluationRun)
+    assert evaluation_run.display_name == "test_red_teaming"
+    assert evaluation_run.state == types.EvaluationRunState.PENDING
+    assert evaluation_run.analysis_configs is not None
+    assert len(evaluation_run.analysis_configs) == 1
+    rt_config = evaluation_run.analysis_configs[0].red_teaming_analysis_config
+    assert rt_config.attack_categories == ["FINANCIAL_OR_CREDENTIAL_PHISHING"]
+    assert rt_config.vulnerable_tools[0].tool_name == "search_flights"
+    assert evaluation_run.error is None
+
+
 pytest_plugins = ("pytest_asyncio",)
 
 

tests/unit/vertexai/genai/test_evals.py

@@ -1835,6 +1835,89 @@ def test_loss_analysis_metrics_accepts_metric_object(self):
         assert result[0].candidate == "agent-1"
 
 
+class TestRedTeamingTypes:
+    """Unit tests for red teaming type definitions."""
+
+    def test_red_teaming_analysis_config_construction(self):
+        config = common_types.RedTeamingAnalysisConfig(
+            attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"],
+            vulnerable_tools=[
+                common_types.VulnerableTool(
+                    tool_name="search_flights",
+                    json_paths=["$.flights[0].description"],
+                ),
+            ],
+        )
+        assert len(config.attack_categories) == 1
+        assert config.vulnerable_tools[0].tool_name == "search_flights"
+
+    def test_red_teaming_analysis_config_optional_fields(self):
+        config = common_types.RedTeamingAnalysisConfig()
+        assert config.attack_categories is None
+        assert config.vulnerable_tools is None
+
+    def test_evaluation_run_results_has_red_teaming_results(self):
+        results = common_types.EvaluationRunResults(
+            red_teaming_analysis_results=[
+                common_types.RedTeamingAnalysisResult(
+                    category_results=[
+                        common_types.AttackCategoryResult(
+                            attack_category="FINANCIAL_OR_CREDENTIAL_PHISHING",
+                            attack_success_rate=0.9,
+                        ),
+                    ],
+                )
+            ],
+        )
+        assert len(results.red_teaming_analysis_results) == 1
+        assert (
+            results.red_teaming_analysis_results[0]
+            .category_results[0]
+            .attack_success_rate
+            == 0.9
+        )
+
+    def test_create_params_accepts_analysis_configs(self):
+        params = common_types._CreateEvaluationRunParameters(
+            name="test-run",
+            analysis_configs=[
+                common_types.AnalysisConfig(
+                    red_teaming_analysis_config=common_types.RedTeamingAnalysisConfig(
+                        attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"],
+                    ),
+                ),
+            ],
+        )
+        assert len(params.analysis_configs) == 1
+
+
+class TestResolveRedTeamingConfig:
+    """Unit tests for _resolve_red_teaming_config."""
+
+    def test_none_when_no_config(self):
+        result = _evals_utils._resolve_red_teaming_config()
+        assert result is None
+
+    def test_wraps_config_in_analysis_configs(self):
+        config = common_types.RedTeamingAnalysisConfig(
+            attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"],
+        )
+        result = _evals_utils._resolve_red_teaming_config(config)
+        assert len(result) == 1
+        assert isinstance(result[0], common_types.AnalysisConfig)
+        assert (
+            result[0].red_teaming_analysis_config.attack_categories[0]
+            == "FINANCIAL_OR_CREDENTIAL_PHISHING"
+        )
+
+    def test_accepts_dict_input(self):
+        result = _evals_utils._resolve_red_teaming_config(
+            {"attack_categories": ["INJECTED_HOSTILITY_AND_HARASSMENT"]}
+        )
+        assert len(result) == 1
+        assert isinstance(result[0], common_types.AnalysisConfig)
+
+
 class TestResolveMetricName:
     """Unit tests for _resolve_metric_name."""
 

vertexai/_genai/_evals_utils.py

@@ -541,6 +541,20 @@ def _resolve_eval_run_loss_configs(
     return configs
 
 
+def _resolve_red_teaming_config(
+    red_teaming_config: Optional[types.RedTeamingAnalysisConfigOrDict] = None,
+) -> Optional[list[types.AnalysisConfig]]:
+    """Wraps a RedTeamingAnalysisConfig into analysis_configs for the API."""
+    if not red_teaming_config:
+        return None
+    config = (
+        types.RedTeamingAnalysisConfig.model_validate(red_teaming_config)
+        if isinstance(red_teaming_config, dict)
+        else red_teaming_config
+    )
+    return [types.AnalysisConfig(red_teaming_analysis_config=config)]
+
+
 def _resolve_loss_analysis_config(
     eval_result: types.EvaluationResult,
     config: Optional[types.LossAnalysisConfig] = None,

vertexai/_genai/evals.py

@@ -130,6 +130,13 @@ def _CreateEvaluationRunParameters_to_vertex(
     if getv(from_object, ["config"]) is not None:
         setv(to_object, ["config"], getv(from_object, ["config"]))
 
+    if getv(from_object, ["analysis_configs"]) is not None:
+        setv(
+            to_object,
+            ["analysisConfigs"],
+            [item for item in getv(from_object, ["analysis_configs"])],
+        )
+
     return to_object
 
 
@@ -603,6 +610,13 @@ def _EvaluationRun_from_vertex(
     if getv(from_object, ["labels"]) is not None:
         setv(to_object, ["labels"], getv(from_object, ["labels"]))
 
+    if getv(from_object, ["analysisConfigs"]) is not None:
+        setv(
+            to_object,
+            ["analysis_configs"],
+            [item for item in getv(from_object, ["analysisConfigs"])],
+        )
+
     return to_object
 
 
@@ -1159,6 +1173,7 @@ def _create_evaluation_run(
             dict[str, types.EvaluationRunInferenceConfigOrDict]
         ] = None,
         config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
+        analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None,
     ) -> types.EvaluationRun:
         """
         Creates an EvaluationRun.
@@ -1172,6 +1187,7 @@ def _create_evaluation_run(
             labels=labels,
             inference_configs=inference_configs,
             config=config,
+            analysis_configs=analysis_configs,
         )
 
         request_url_dict: Optional[dict[str, str]]
@@ -2616,6 +2632,7 @@ def create_evaluation_run(
         labels: Optional[dict[str, str]] = None,
         loss_analysis_metrics: Optional[list[Union[str, types.MetricOrDict]]] = None,
         loss_analysis_configs: Optional[list[types.LossAnalysisConfigOrDict]] = None,
+        red_teaming_config: Optional[types.RedTeamingAnalysisConfigOrDict] = None,
         config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
     ) -> types.EvaluationRun:
         """Creates an EvaluationRun.
@@ -2734,6 +2751,9 @@ def create_evaluation_run(
             loss_analysis_configs=loss_analysis_configs,
             inference_configs=inference_configs,
         )
+        resolved_analysis_configs = _evals_utils._resolve_red_teaming_config(
+            red_teaming_config
+        )
         evaluation_config = types.EvaluationRunConfig(
             output_config=output_config,
             metrics=resolved_metrics,
@@ -2751,6 +2771,7 @@ def create_evaluation_run(
             data_source=resolved_dataset,
             evaluation_config=evaluation_config,
             inference_configs=resolved_inference_configs,
+            analysis_configs=resolved_analysis_configs,
             labels=resolved_labels,
             config=config,
         )
@@ -3299,6 +3320,7 @@ async def _create_evaluation_run(
             dict[str, types.EvaluationRunInferenceConfigOrDict]
         ] = None,
         config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
+        analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None,
     ) -> types.EvaluationRun:
         """
         Creates an EvaluationRun.
@@ -3312,6 +3334,7 @@ async def _create_evaluation_run(
             labels=labels,
             inference_configs=inference_configs,
             config=config,
+            analysis_configs=analysis_configs,
         )
 
         request_url_dict: Optional[dict[str, str]]
@@ -4395,6 +4418,7 @@ async def create_evaluation_run(
         inference_configs: Optional[
             dict[str, types.EvaluationRunInferenceConfigOrDict]
         ] = None,
+        red_teaming_config: Optional[types.RedTeamingAnalysisConfigOrDict] = None,
         labels: Optional[dict[str, str]] = None,
         loss_analysis_metrics: Optional[list[Union[str, types.MetricOrDict]]] = None,
         loss_analysis_configs: Optional[list[types.LossAnalysisConfigOrDict]] = None,
@@ -4426,6 +4450,11 @@ async def create_evaluation_run(
               this will be automatically constructed using `agent_info` and `user_simulator_config`.
               Example:
               {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
+          red_teaming_config: This field is experimental and may change in future
+              versions. Optional configuration for automated Agent Red Teaming
+              analysis. Specifies attack categories and vulnerable tools to
+              test. When provided, the server runs a red teaming pipeline
+              instead of standard evaluation metrics.
           labels: The labels to apply to the evaluation run.
           loss_analysis_metrics: This field is experimental and may change in future
               versions. Optional list of metrics to run loss analysis on. The
@@ -4511,6 +4540,9 @@ async def create_evaluation_run(
             loss_analysis_configs=loss_analysis_configs,
             inference_configs=inference_configs,
         )
+        resolved_analysis_configs = _evals_utils._resolve_red_teaming_config(
+            red_teaming_config
+        )
         evaluation_config = types.EvaluationRunConfig(
             output_config=output_config,
             metrics=resolved_metrics,
@@ -4529,6 +4561,7 @@ async def create_evaluation_run(
             data_source=resolved_dataset,
             evaluation_config=evaluation_config,
             inference_configs=resolved_inference_configs,
+            analysis_configs=resolved_analysis_configs,
             labels=resolved_labels,
             config=config,
         )

vertexai/_genai/types/__init__.py

@@ -181,6 +181,9 @@
 from .common import AggregatedMetricResult
 from .common import AggregatedMetricResultDict
 from .common import AggregatedMetricResultOrDict
+from .common import AnalysisConfig
+from .common import AnalysisConfigDict
+from .common import AnalysisConfigOrDict
 from .common import AppendAgentEngineSessionEventConfig
 from .common import AppendAgentEngineSessionEventConfigDict
 from .common import AppendAgentEngineSessionEventConfigOrDict
@@ -202,6 +205,9 @@
 from .common import AssessDatasetConfig
 from .common import AssessDatasetConfigDict
 from .common import AssessDatasetConfigOrDict
+from .common import AttackCategoryResult
+from .common import AttackCategoryResultDict
+from .common import AttackCategoryResultOrDict
 from .common import BatchPredictionResourceUsageAssessmentConfig
 from .common import BatchPredictionResourceUsageAssessmentConfigDict
 from .common import BatchPredictionResourceUsageAssessmentConfigOrDict
@@ -1063,6 +1069,12 @@
 from .common import ReasoningEngineTrafficConfigTrafficSplitManualTarget
 from .common import ReasoningEngineTrafficConfigTrafficSplitManualTargetDict
 from .common import ReasoningEngineTrafficConfigTrafficSplitManualTargetOrDict
+from .common import RedTeamingAnalysisConfig
+from .common import RedTeamingAnalysisConfigDict
+from .common import RedTeamingAnalysisConfigOrDict
+from .common import RedTeamingAnalysisResult
+from .common import RedTeamingAnalysisResultDict
+from .common import RedTeamingAnalysisResultOrDict
 from .common import ReservationAffinity
 from .common import ReservationAffinityDict
 from .common import ReservationAffinityOrDict
@@ -1440,6 +1452,9 @@
 from .common import VertexBaseConfig
 from .common import VertexBaseConfigDict
 from .common import VertexBaseConfigOrDict
+from .common import VulnerableTool
+from .common import VulnerableToolDict
+from .common import VulnerableToolOrDict
 from .common import WinRateStats
 from .common import WinRateStatsDict
 from .common import WinRateStatsOrDict
@@ -1577,12 +1592,27 @@
     "EvaluationRunInferenceConfig",
     "EvaluationRunInferenceConfigDict",
     "EvaluationRunInferenceConfigOrDict",
+    "VulnerableTool",
+    "VulnerableToolDict",
+    "VulnerableToolOrDict",
+    "RedTeamingAnalysisConfig",
+    "RedTeamingAnalysisConfigDict",
+    "RedTeamingAnalysisConfigOrDict",
+    "AnalysisConfig",
+    "AnalysisConfigDict",
+    "AnalysisConfigOrDict",
     "CreateEvaluationRunConfig",
     "CreateEvaluationRunConfigDict",
     "CreateEvaluationRunConfigOrDict",
     "SummaryMetric",
     "SummaryMetricDict",
     "SummaryMetricOrDict",
+    "AttackCategoryResult",
+    "AttackCategoryResultDict",
+    "AttackCategoryResultOrDict",
+    "RedTeamingAnalysisResult",
+    "RedTeamingAnalysisResultDict",
+    "RedTeamingAnalysisResultOrDict",
     "LossTaxonomyEntry",
     "LossTaxonomyEntryDict",
     "LossTaxonomyEntryOrDict",