feat: GenAI Client(evals) - generate_conversation_scenarios support for Gemini Agents API agents

PiperOrigin-RevId: 947873256

Author: jsondai

agentplatform/_genai/_evals_common.py

@@ -988,6 +988,33 @@ def _agent_data_response_text(agent_data: types.evals.AgentData) -> Optional[str
     return "".join(text_parts) or None
 
 
+def _agent_resource_to_agent_info(
+    agent: str, api_client: BaseApiClient
+) -> "types.evals.AgentInfo":
+    """Builds an `AgentInfo` from a Gemini Agents API agent resource name.
+
+    Fetches the agent through the SDK's `api_client` (so replay recording is
+    preserved) via `_fetch_agent_config_dict` and derives a single-agent
+    `AgentInfo`: the agent's short name is the agents-map key and
+    `root_agent_id`.
+
+    Args:
+        agent: The Gemini Agents API agent resource name
+          (`projects/{p}/locations/{l}/agents/{name}`).
+        api_client: The API client used to fetch the agent.
+
+    Returns:
+        An `AgentInfo` describing the fetched agent.
+    """
+    agent_config = _fetch_agent_config_dict(api_client, agent)
+    short_name = agent_config.agent_id
+    return types.evals.AgentInfo(  # pytype: disable=missing-parameter
+        name=short_name,
+        agents={short_name: agent_config},
+        root_agent_id=short_name,
+    )
+
+
 _INTERACTION_TERMINAL_STATES = frozenset(
     ["completed", "failed", "cancelled", "incomplete", "budget_exceeded"]
 )

agentplatform/_genai/evals.py

@@ -2956,28 +2956,54 @@ def create_evaluation_set(
     def generate_conversation_scenarios(
         self,
         *,
-        agent_info: evals_types.AgentInfoOrDict,
+        agent_info: Optional[evals_types.AgentInfoOrDict] = None,
+        agent: Optional[str] = None,
         config: evals_types.UserScenarioGenerationConfigOrDict,
         allow_cross_region_model: Optional[bool] = None,
     ) -> types.EvaluationDataset:
         """Generates an evaluation dataset with user scenarios,
            which helps to generate conversations between a simulated user
            and the agent under test.
 
+        Exactly one of `agent_info` or `agent` must be provided. When `agent` is
+        a Gemini Agents API agent resource name, the agent is fetched and an
+        `AgentInfo` is derived from it.
+
         Args:
-            agent_info: The agent info to generate user scenarios for.
+            agent_info: The agent info to generate user scenarios for. Mutually
+                exclusive with `agent`.
+            agent: A Gemini Agents API agent resource name
+                (`projects/{p}/locations/{l}/agents/{name}`). When provided, the
+                agent is fetched and its configuration is used to build the agent
+                info. Mutually exclusive with `agent_info`.
             config: Configuration for generating user scenarios.
             allow_cross_region_model: Opt-in flag to authorize cross-region
                 routing for model inference.
 
         Returns:
             An EvaluationDataset containing the generated user scenarios.
         """
-        parsed_agent_info = (
-            evals_types.AgentInfo.model_validate(agent_info)
-            if isinstance(agent_info, dict)
-            else agent_info
-        )
+        if agent is not None and agent_info is not None:
+            raise ValueError(
+                "Only one of `agent` or `agent_info` may be provided, not both."
+            )
+        if agent is None and agent_info is None:
+            raise ValueError("One of `agent` or `agent_info` must be provided.")
+        if agent is not None:
+            if not _evals_common._is_gemini_agent_resource(agent):
+                raise ValueError(
+                    "`agent` must be a Gemini Agents API agent resource name of the"
+                    " form projects/{project}/locations/{location}/agents/{agent}."
+                )
+            parsed_agent_info = _evals_common._agent_resource_to_agent_info(
+                agent, self._api_client
+            )
+        else:
+            parsed_agent_info = (
+                evals_types.AgentInfo.model_validate(agent_info)
+                if isinstance(agent_info, dict)
+                else agent_info
+            )
         response = self._generate_user_scenarios(
             agents=parsed_agent_info.agents,
             root_agent_id=parsed_agent_info.root_agent_id,
@@ -4769,28 +4795,54 @@ async def create_evaluation_set(
     async def generate_conversation_scenarios(
         self,
         *,
-        agent_info: evals_types.AgentInfoOrDict,
+        agent_info: Optional[evals_types.AgentInfoOrDict] = None,
+        agent: Optional[str] = None,
         config: evals_types.UserScenarioGenerationConfigOrDict,
         allow_cross_region_model: Optional[bool] = None,
     ) -> types.EvaluationDataset:
         """Generates an evaluation dataset with user scenarios,
            which helps to generate conversations between a simulated user
            and the agent under test.
 
+        Exactly one of `agent_info` or `agent` must be provided. When `agent` is
+        a Gemini Agents API agent resource name, the agent is fetched and an
+        `AgentInfo` is derived from it.
+
         Args:
-            agent_info: The agent info to generate user scenarios for.
+            agent_info: The agent info to generate user scenarios for. Mutually
+                exclusive with `agent`.
+            agent: A Gemini Agents API agent resource name
+                (`projects/{p}/locations/{l}/agents/{name}`). When provided, the
+                agent is fetched and its configuration is used to build the agent
+                info. Mutually exclusive with `agent_info`.
             config: Configuration for generating user scenarios.
             allow_cross_region_model: Opt-in flag to authorize cross-region
                 routing for model inference.
 
         Returns:
             An EvaluationDataset containing the generated user scenarios.
         """
-        parsed_agent_info = (
-            evals_types.AgentInfo.model_validate(agent_info)
-            if isinstance(agent_info, dict)
-            else agent_info
-        )
+        if agent is not None and agent_info is not None:
+            raise ValueError(
+                "Only one of `agent` or `agent_info` may be provided, not both."
+            )
+        if agent is None and agent_info is None:
+            raise ValueError("One of `agent` or `agent_info` must be provided.")
+        if agent is not None:
+            if not _evals_common._is_gemini_agent_resource(agent):
+                raise ValueError(
+                    "`agent` must be a Gemini Agents API agent resource name of the"
+                    " form projects/{project}/locations/{location}/agents/{agent}."
+                )
+            parsed_agent_info = _evals_common._agent_resource_to_agent_info(
+                agent, self._api_client
+            )
+        else:
+            parsed_agent_info = (
+                evals_types.AgentInfo.model_validate(agent_info)
+                if isinstance(agent_info, dict)
+                else agent_info
+            )
         response = await self._generate_user_scenarios(
             agents=parsed_agent_info.agents,
             root_agent_id=parsed_agent_info.root_agent_id,

tests/unit/agentplatform/genai/replays/test_generate_conversation_scenarios.py

@@ -103,6 +103,30 @@ async def test_gen_conversation_scenarios_async(client):
     assert eval_dataset.eval_cases[1].user_scenario.conversation_plan
 
 
+def test_scenarios_from_gemini_agent(client):
+    """Tests generate_conversation_scenarios() against a Gemini Agents API agent.
+
+    Fetches the agent via google.genai agents.get, derives an AgentInfo from it,
+    and generates user scenarios from the derived config.
+    """
+    eval_dataset = client.evals.generate_conversation_scenarios(
+        agent=("projects/model-evaluation-dev/locations/global/agents/test-agent-eval"),
+        config=types.evals.UserScenarioGenerationConfig(
+            count=2,
+            generation_instruction=(
+                "Generate scenarios where the user tries to book a flight but"
+                " changes their mind about the destination."
+            ),
+            environment_context="Today is Monday. Flights to Paris are available.",
+            model_name="gemini-2.5-flash",
+        ),
+    )
+    assert isinstance(eval_dataset, types.EvaluationDataset)
+    assert len(eval_dataset.eval_cases) == 2
+    assert eval_dataset.eval_cases[0].user_scenario.starting_prompt
+    assert eval_dataset.eval_cases[0].user_scenario.conversation_plan
+
+
 pytestmark = pytest_helper.setup(
     file=__file__,
     globals_for_file=globals(),

tests/unit/agentplatform/genai/test_evals.py

@@ -9133,6 +9133,68 @@ async def test_async_generate_conversation_scenarios(self):
         request_body = call_args[0][2]  # Third positional arg is the request dict
         assert request_body.get("allowCrossRegionModel") is True
 
+    @mock.patch.object(_evals_common, "_fetch_agent_config_dict")
+    def test_generate_conversation_scenarios_from_gemini_agent(
+        self, mock_fetch_agent_config
+    ):
+        mock_fetch_agent_config.return_value = (
+            agentplatform_genai_types.evals.AgentConfig(
+                agent_id="test-agent",
+                instruction="You are a helpful travel assistant.",
+                description="An agent that books flights.",
+                tools=[genai_types.Tool(google_search=genai_types.GoogleSearch())],
+            )
+        )
+
+        evals_module = evals.Evals(api_client_=self.mock_api_client)
+
+        with mock.patch.object(
+            evals_module, "_generate_user_scenarios"
+        ) as mock_generate_user_scenarios:
+            mock_generate_user_scenarios.return_value = self.mock_response
+            evals_module.generate_conversation_scenarios(
+                agent=_TEST_GEMINI_AGENT,
+                config={"count": 2},
+            )
+
+        mock_fetch_agent_config.assert_called_once_with(
+            self.mock_api_client, _TEST_GEMINI_AGENT
+        )
+        call_kwargs = mock_generate_user_scenarios.call_args.kwargs
+        assert call_kwargs["root_agent_id"] == "test-agent"
+        agents = call_kwargs["agents"]
+        assert "test-agent" in agents
+        derived_config = agents["test-agent"]
+        assert derived_config.instruction == "You are a helpful travel assistant."
+        assert derived_config.description == "An agent that books flights."
+        assert derived_config.tools is not None
+        assert derived_config.tools[0].google_search is not None
+
+    def test_generate_conversation_scenarios_agent_and_agent_info_raises(self):
+        evals_module = evals.Evals(api_client_=self.mock_api_client)
+        with pytest.raises(ValueError, match="not both"):
+            evals_module.generate_conversation_scenarios(
+                agent=_TEST_GEMINI_AGENT,
+                agent_info=agentplatform_genai_types.evals.AgentInfo(
+                    agents={"agent_1": {}},
+                    root_agent_id="agent_1",
+                ),
+                config={"count": 2},
+            )
+
+    def test_generate_conversation_scenarios_no_agent_raises(self):
+        evals_module = evals.Evals(api_client_=self.mock_api_client)
+        with pytest.raises(ValueError, match="must be provided"):
+            evals_module.generate_conversation_scenarios(config={"count": 2})
+
+    def test_generate_conversation_scenarios_non_gemini_agent_raises(self):
+        evals_module = evals.Evals(api_client_=self.mock_api_client)
+        with pytest.raises(ValueError, match="Gemini Agents API"):
+            evals_module.generate_conversation_scenarios(
+                agent=_TEST_AGENT_ENGINE,
+                config={"count": 2},
+            )
+
 
 class TestTransformDataframe:
     """Unit tests for the _transform_dataframe function."""