Skip to content

Commit 89d330e

Browse files
jsondaicopybara-github
authored andcommitted
fix: GenAI Client(evals)-Propagate Vertex AI client config to LlmBackedUserSimulator
FUTURE_COPYBARA_INTEGRATE_REVIEW=#6596 from googleapis:release-please--branches--main b82c8bd PiperOrigin-RevId: 902796340
1 parent 3c55f26 commit 89d330e

1 file changed

Lines changed: 25 additions & 2 deletions

File tree

vertexai/_genai/_evals_common.py

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -559,6 +559,7 @@ def agent_run_wrapper( # type: ignore[no-untyped-def]
559559
contents=contents_arg,
560560
user_simulator_config=user_simulator_config_arg,
561561
agent=agent_arg,
562+
api_client=api_client_arg,
562563
)
563564

564565
future = executor.submit(
@@ -955,6 +956,7 @@ async def _run_adk_user_simulation(
955956
row: pd.Series,
956957
agent: "LlmAgent", # type: ignore # noqa: F821
957958
config: Optional[types.evals.UserSimulatorConfig] = None,
959+
api_client: Optional[BaseApiClient] = None,
958960
) -> list[dict[str, Any]]:
959961
"""Runs a multi-turn user simulation using ADK's EvaluationGenerator."""
960962
# Lazy-import ADK dependencies to avoid top-level import failures when
@@ -996,6 +998,23 @@ async def _run_adk_user_simulation(
996998
if config.max_turn is not None:
997999
user_simulator_kwargs["max_allowed_invocations"] = config.max_turn
9981000

1001+
# When using a Vertex AI client, convert the user simulator model name to
1002+
# a full resource path so that ADK's Gemini class automatically uses the
1003+
# Vertex AI backend. This removes the need for users to manually set the
1004+
# GOOGLE_GENAI_USE_VERTEXAI environment variable.
1005+
if (
1006+
api_client
1007+
and getattr(api_client, "project", None)
1008+
and getattr(api_client, "location", None)
1009+
):
1010+
model_name = user_simulator_kwargs.get("model", "gemini-2.5-flash")
1011+
if not model_name.startswith("projects/"):
1012+
user_simulator_kwargs["model"] = (
1013+
f"projects/{api_client.project}"
1014+
f"/locations/{api_client.location}"
1015+
f"/publishers/google/models/{model_name}"
1016+
)
1017+
9991018
user_simulator_config = LlmBackedUserSimulatorConfig(**user_simulator_kwargs)
10001019
user_simulator = LlmBackedUserSimulator(
10011020
conversation_scenario=scenario, config=user_simulator_config
@@ -2095,11 +2114,12 @@ def _execute_local_agent_run_with_retry(
20952114
agent: "LlmAgent", # type: ignore # noqa: F821
20962115
max_retries: int = 3,
20972116
user_simulator_config: Optional[types.evals.UserSimulatorConfig] = None,
2117+
api_client: Optional[BaseApiClient] = None,
20982118
) -> Union[list[dict[str, Any]], dict[str, Any]]:
20992119
"""Executes agent run locally for a single prompt synchronously."""
21002120
return asyncio.run(
21012121
_execute_local_agent_run_with_retry_async(
2102-
row, contents, agent, max_retries, user_simulator_config
2122+
row, contents, agent, max_retries, user_simulator_config, api_client
21032123
)
21042124
)
21052125

@@ -2110,6 +2130,7 @@ async def _execute_local_agent_run_with_retry_async(
21102130
agent: "LlmAgent", # type: ignore # noqa: F821
21112131
max_retries: int = 3,
21122132
user_simulator_config: Optional[types.evals.UserSimulatorConfig] = None,
2133+
api_client: Optional[BaseApiClient] = None,
21132134
) -> Union[list[dict[str, Any]], dict[str, Any]]:
21142135
"""Executes agent run locally for a single prompt asynchronously."""
21152136
# Lazy-import ADK dependencies to avoid top-level import failures when
@@ -2120,7 +2141,9 @@ async def _execute_local_agent_run_with_retry_async(
21202141
# Multi-turn agent scraping with user simulation.
21212142
if user_simulator_config or "conversation_plan" in row:
21222143
try:
2123-
return await _run_adk_user_simulation(row, agent, user_simulator_config)
2144+
return await _run_adk_user_simulation(
2145+
row, agent, user_simulator_config, api_client
2146+
)
21242147
except Exception as e: # pylint: disable=broad-exception-caught
21252148
logger.error("Multi-turn agent run with user simulation failed: %s", e)
21262149
return {"error": f"Multi-turn agent run with user simulation failed: {e}"}

0 commit comments

Comments
 (0)