Skip to content

Commit ebaef9f

Browse files
ftnextGWeale
authored andcommitted
fix(eval): support get_agent_async in adk eval
Merge google#4413 **Please ensure you have read the [contribution guide](https://github.com/google/adk-python/blob/main/CONTRIBUTING.md) before creating a pull request.** ### Link to Issue or Description of Change **1. Link to an existing issue (if applicable):** - Related: google#4410 **Problem:** `adk eval` only resolved `agent.root_agent`, while `AgentEvaluator` supports both `root_agent` and `get_agent_async`. This inconsistency caused valid agent modules (for `AgentEvaluator`) to fail in CLI evaluation. **Solution:** Aligned `adk eval` agent resolution with `AgentEvaluator` by updating CLI loading logic to support both entry points: - `root_agent` - `get_agent_async` Implementation details: - Made `get_root_agent` in `cli_eval.py` asynchronous and added fallback resolution to `get_agent_async`. - Updated `cli_tools_click.py` to call `get_root_agent` via `asyncio.run(...)`. - Added/updated unit tests for both resolution paths and the error path when neither is present. ### Testing Plan **Unit Tests:** - [x] I have added or updated unit tests for my change. - [x] All unit tests pass locally. ``` % pytest tests/unittests/cli ========================= 271 passed, 140 warnings in 7.69s ========================= ``` **Manual End-to-End (E2E) Tests:** No manual E2E test was run for this PR. ### Checklist - [x] I have read the [CONTRIBUTING.md](https://github.com/google/adk-python/blob/main/CONTRIBUTING.md) document. - [x] I have performed a self-review of my own code. - [x] I have commented my code, particularly in hard-to-understand areas. - [x] I have added tests that prove my fix is effective or that my feature works. - [x] New and existing unit tests pass locally with my changes. - [ ] I have manually tested my changes end-to-end. - [x] Any dependent changes have been merged and published in downstream modules. ### Additional context This PR focuses on one scoped item from google#4410: agent resolution parity between `adk eval` and `AgentEvaluator` (`root_agent` / `get_agent_async`). Co-authored-by: George Weale <gweale@google.com> COPYBARA_INTEGRATE_REVIEW=google#4413 from ftnext:adk-eval-get-agent-async 267c196 PiperOrigin-RevId: 952361014
1 parent bb56aa5 commit ebaef9f

4 files changed

Lines changed: 63 additions & 9 deletions

File tree

src/google/adk/cli/cli_eval.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
import click
2727
from google.genai import types as genai_types
2828

29-
from ..agents.llm_agent import Agent
29+
from ..agents.base_agent import BaseAgent
3030
from ..evaluation.base_eval_service import BaseEvalService
3131
from ..evaluation.base_eval_service import EvaluateConfig
3232
from ..evaluation.base_eval_service import EvaluateRequest
@@ -74,11 +74,18 @@ def _get_agent_module(agent_module_file_path: str) -> ModuleType:
7474
return _import_from_path(module_name, file_path)
7575

7676

77-
def get_root_agent(agent_module_file_path: str) -> Agent:
77+
async def get_root_agent(agent_module_file_path: str) -> BaseAgent:
7878
"""Returns root agent given the agent module."""
7979
agent_module = _get_agent_module(agent_module_file_path)
80-
root_agent = agent_module.agent.root_agent
81-
return cast(Agent, root_agent)
80+
agent_module_with_agent = getattr(agent_module, "agent", agent_module)
81+
if hasattr(agent_module_with_agent, "root_agent"):
82+
return cast(BaseAgent, agent_module_with_agent.root_agent)
83+
elif hasattr(agent_module_with_agent, "get_agent_async"):
84+
root_agent, _ = await agent_module_with_agent.get_agent_async()
85+
return cast(BaseAgent, root_agent)
86+
raise ValueError(
87+
"Agent module should have either `root_agent` or `get_agent_async`."
88+
)
8289

8390

8491
def try_get_reset_func(agent_module_file_path: str) -> Any:

src/google/adk/cli/cli_tools_click.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@
2626
import sys
2727
import tempfile
2828
import textwrap
29+
from typing import cast
2930
from typing import Optional
31+
from typing import TYPE_CHECKING
3032

3133
import click
3234
from click.core import ParameterSource
@@ -42,6 +44,9 @@
4244
from .utils import envs
4345
from .utils import logs
4446

47+
if TYPE_CHECKING:
48+
from ..agents.llm_agent import LlmAgent
49+
4550
LOG_LEVELS = click.Choice(
4651
["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
4752
case_sensitive=False,
@@ -1027,7 +1032,7 @@ def cli_eval(
10271032
print(f"Using evaluation criteria: {eval_config}")
10281033
eval_metrics = get_eval_metrics_from_config(eval_config)
10291034

1030-
root_agent = get_root_agent(agent_module_file_path)
1035+
root_agent = asyncio.run(get_root_agent(agent_module_file_path))
10311036
app_name = os.path.basename(agent_module_file_path)
10321037
agents_dir = os.path.dirname(agent_module_file_path)
10331038
eval_sets_manager = None
@@ -1257,7 +1262,7 @@ def cli_optimize(
12571262
else:
12581263
optimizer_config = GEPARootAgentPromptOptimizerConfig()
12591264

1260-
root_agent = get_root_agent(agent_module_file_path)
1265+
root_agent = asyncio.run(get_root_agent(agent_module_file_path))
12611266
app_name = os.path.basename(agent_module_file_path)
12621267
agents_dir = os.path.dirname(agent_module_file_path)
12631268
if app_name != sampler_config.app_name:
@@ -1270,7 +1275,9 @@ def cli_optimize(
12701275
sampler = LocalEvalSampler(sampler_config, eval_sets_manager)
12711276
optimizer = GEPARootAgentPromptOptimizer(optimizer_config)
12721277

1273-
optimization_result = asyncio.run(optimizer.optimize(root_agent, sampler))
1278+
optimization_result = asyncio.run(
1279+
optimizer.optimize(cast("LlmAgent", root_agent), sampler)
1280+
)
12741281
best_idx = optimization_result.gepa_result["best_idx"]
12751282

12761283
click.echo("=" * 80)
@@ -1483,7 +1490,7 @@ def cli_generate_eval_cases(
14831490

14841491
try:
14851492
eval_sets_manager = get_eval_sets_manager(eval_storage_uri, agents_dir)
1486-
root_agent = get_root_agent(agent_module_file_path)
1493+
root_agent = asyncio.run(get_root_agent(agent_module_file_path))
14871494

14881495
# Try to create if it doesn't already exist.
14891496
if (

tests/unittests/cli/utils/test_cli_eval.py

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from types import SimpleNamespace
2020
from unittest import mock
2121

22+
from google.adk.cli.cli_eval import get_root_agent
2223
import pytest
2324

2425

@@ -110,3 +111,40 @@ def test_get_eval_sets_manager_gcs(monkeypatch):
110111
)
111112
assert manager == mock_gcs_manager
112113
mock_create_gcs.assert_called_once_with("gs://bucket")
114+
115+
116+
@pytest.mark.asyncio
117+
async def test_get_root_agent_supports_root_agent(monkeypatch):
118+
root_agent = mock.MagicMock()
119+
agent_module = SimpleNamespace(agent=SimpleNamespace(root_agent=root_agent))
120+
monkeypatch.setattr(
121+
"google.adk.cli.cli_eval._get_agent_module",
122+
lambda _agent_module_file_path: agent_module,
123+
)
124+
assert await get_root_agent("some/dir") == root_agent
125+
126+
127+
@pytest.mark.asyncio
128+
async def test_get_root_agent_supports_get_agent_async(monkeypatch):
129+
root_agent = mock.MagicMock()
130+
get_agent_async = mock.AsyncMock(return_value=(root_agent, object()))
131+
agent_module = SimpleNamespace(
132+
agent=SimpleNamespace(get_agent_async=get_agent_async)
133+
)
134+
monkeypatch.setattr(
135+
"google.adk.cli.cli_eval._get_agent_module",
136+
lambda _agent_module_file_path: agent_module,
137+
)
138+
assert await get_root_agent("some/dir") == root_agent
139+
get_agent_async.assert_awaited_once()
140+
141+
142+
@pytest.mark.asyncio
143+
async def test_get_root_agent_raises_without_supported_entrypoint(monkeypatch):
144+
agent_module = SimpleNamespace(agent=SimpleNamespace())
145+
monkeypatch.setattr(
146+
"google.adk.cli.cli_eval._get_agent_module",
147+
lambda _agent_module_file_path: agent_module,
148+
)
149+
with pytest.raises(ValueError, match="root_agent|get_agent_async"):
150+
await get_root_agent("some/dir")

tests/unittests/cli/utils/test_cli_tools_click.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,9 @@ def mock_load_eval_set_from_file():
6161

6262
@pytest.fixture
6363
def mock_get_root_agent():
64-
with mock.patch("google.adk.cli.cli_eval.get_root_agent") as mock_func:
64+
with mock.patch(
65+
"google.adk.cli.cli_eval.get_root_agent", new_callable=mock.AsyncMock
66+
) as mock_func:
6567
mock_func.return_value = root_agent
6668
yield mock_func
6769

0 commit comments

Comments
 (0)