From fa2dc3b899f25e0b628ede04ba59892391a902e5 Mon Sep 17 00:00:00 2001 From: doug <110487462+doughayden@users.noreply.github.com> Date: Sat, 25 Jul 2026 13:00:05 -0400 Subject: [PATCH] fix(eval): wire App plugins through eval paths - Thread app= from cli, dev_server, and AgentEvaluator - Build eval Runner from app.model_copy with merged plugins - Share Runner construction across live and non-live paths - Add unit tests covering each eval surface Closes #5503 --- src/google/adk/cli/cli_eval.py | 32 +++- src/google/adk/cli/cli_tools_click.py | 5 +- src/google/adk/cli/dev_server.py | 3 + src/google/adk/evaluation/agent_evaluator.py | 23 ++- .../adk/evaluation/evaluation_generator.py | 98 +++++++++-- .../adk/evaluation/local_eval_service.py | 15 ++ tests/unittests/cli/utils/test_cli_eval.py | 86 ++++++++++ .../cli/utils/test_cli_tools_click.py | 11 +- .../evaluation/test_agent_evaluator.py | 161 ++++++++++++++++++ .../evaluation/test_evaluation_generator.py | 133 +++++++++++++++ .../evaluation/test_local_eval_service.py | 115 +++++++++++++ 11 files changed, 660 insertions(+), 22 deletions(-) create mode 100644 tests/unittests/evaluation/test_agent_evaluator.py diff --git a/src/google/adk/cli/cli_eval.py b/src/google/adk/cli/cli_eval.py index 471bfb5257e..f191f5efbad 100644 --- a/src/google/adk/cli/cli_eval.py +++ b/src/google/adk/cli/cli_eval.py @@ -27,6 +27,7 @@ from google.genai import types as genai_types from ..agents.base_agent import BaseAgent +from ..apps.app import App from ..evaluation.base_eval_service import BaseEvalService from ..evaluation.base_eval_service import EvaluateConfig from ..evaluation.base_eval_service import EvaluateRequest @@ -75,20 +76,43 @@ def _get_agent_module(agent_module_file_path: str) -> ModuleType: return _import_from_path(module_name, file_path) -async def get_root_agent(agent_module_file_path: str) -> BaseAgent: - """Returns root agent given the agent module.""" +async def get_app_or_root_agent( + agent_module_file_path: str, +) -> tuple[Optional[App], BaseAgent]: + """Returns the (app, root_agent) pair for the given agent module. + + If the module exposes an `App` instance via `app`, that App and its + `root_agent` are returned. Otherwise `app` is None and the root agent is + resolved the same way as `get_root_agent`. This lets eval flows participate + in the App's plugin / cache / resumability lifecycle when one is defined, + while preserving the bare-`root_agent` path for projects that don't use App. + """ agent_module = _get_agent_module(agent_module_file_path) agent_module_with_agent = getattr(agent_module, "agent", agent_module) + app = getattr(agent_module_with_agent, "app", None) + if isinstance(app, App): + return app, cast(BaseAgent, app.root_agent) if hasattr(agent_module_with_agent, "root_agent"): - return cast(BaseAgent, agent_module_with_agent.root_agent) + return None, cast(BaseAgent, agent_module_with_agent.root_agent) elif hasattr(agent_module_with_agent, "get_agent_async"): root_agent, _ = await agent_module_with_agent.get_agent_async() - return cast(BaseAgent, root_agent) + return None, cast(BaseAgent, root_agent) raise ValueError( "Agent module should have either `root_agent` or `get_agent_async`." ) +async def get_root_agent(agent_module_file_path: str) -> BaseAgent: + """Returns root agent given the agent module. + + Kept for backward compatibility. New callers should prefer + `get_app_or_root_agent`, which also surfaces the wrapping `App` (if any) + so plugins, context-cache, and resumability configs are honored. + """ + _, root_agent = await get_app_or_root_agent(agent_module_file_path) + return root_agent + + def try_get_reset_func(agent_module_file_path: str) -> Any: """Returns reset function for the agent, if present, given the agent module.""" agent_module = _get_agent_module(agent_module_file_path) diff --git a/src/google/adk/cli/cli_tools_click.py b/src/google/adk/cli/cli_tools_click.py index 379c314a5a4..f03dd1f5790 100644 --- a/src/google/adk/cli/cli_tools_click.py +++ b/src/google/adk/cli/cli_tools_click.py @@ -1022,7 +1022,7 @@ def cli_eval( from ..evaluation.simulation.user_simulator_provider import UserSimulatorProvider from .cli_eval import _collect_eval_results from .cli_eval import _collect_inferences - from .cli_eval import get_root_agent + from .cli_eval import get_app_or_root_agent from .cli_eval import parse_and_get_evals_to_run from .cli_eval import pretty_print_eval_result except ModuleNotFoundError as mnf: @@ -1032,7 +1032,7 @@ def cli_eval( print(f"Using evaluation criteria: {eval_config}") eval_metrics = get_eval_metrics_from_config(eval_config) - root_agent = asyncio.run(get_root_agent(agent_module_file_path)) + app, root_agent = asyncio.run(get_app_or_root_agent(agent_module_file_path)) app_name = os.path.basename(agent_module_file_path) agents_dir = os.path.dirname(agent_module_file_path) eval_sets_manager = None @@ -1124,6 +1124,7 @@ def cli_eval( eval_set_results_manager=eval_set_results_manager, user_simulator_provider=user_simulator_provider, metric_evaluator_registry=metric_evaluator_registry, + app=app, ) inference_results = asyncio.run( diff --git a/src/google/adk/cli/dev_server.py b/src/google/adk/cli/dev_server.py index 264cd52abe7..3a7f6382075 100644 --- a/src/google/adk/cli/dev_server.py +++ b/src/google/adk/cli/dev_server.py @@ -48,6 +48,7 @@ import yaml from . import agent_graph +from ..apps.app import App from ..errors.not_found_error import NotFoundError from ..evaluation.base_eval_service import InferenceConfig from ..evaluation.base_eval_service import InferenceRequest @@ -1068,6 +1069,7 @@ async def run_eval( agent_or_app = self.agent_loader.load_agent(app_name) root_agent = self._get_root_agent(agent_or_app) + app = agent_or_app if isinstance(agent_or_app, App) else None eval_case_results = [] @@ -1077,6 +1079,7 @@ async def run_eval( eval_set_results_manager=self.eval_set_results_manager, session_service=self.session_service, artifact_service=self.artifact_service, + app=app, ) inference_request = InferenceRequest( app_name=app_name, diff --git a/src/google/adk/evaluation/agent_evaluator.py b/src/google/adk/evaluation/agent_evaluator.py index 65c9c9b1d9f..5e80ec9d97c 100644 --- a/src/google/adk/evaluation/agent_evaluator.py +++ b/src/google/adk/evaluation/agent_evaluator.py @@ -34,6 +34,7 @@ from pydantic import ValidationError from ..agents.base_agent import BaseAgent +from ..apps.app import App from ..utils.context_utils import Aclosing from .constants import MISSING_EVAL_DEPENDENCIES_MESSAGE from .eval_case import get_all_tool_calls @@ -147,7 +148,7 @@ async def evaluate_eval_set( if eval_config is None: raise ValueError("`eval_config` is required.") - agent_for_eval = await AgentEvaluator._get_agent_for_eval( + agent_for_eval, app = await AgentEvaluator._get_agent_for_eval( module_name=agent_module, agent_name=agent_name ) eval_metrics = get_eval_metrics_from_config(eval_config) @@ -163,6 +164,7 @@ async def evaluate_eval_set( eval_metrics=eval_metrics, num_runs=num_runs, user_simulator_provider=user_simulator_provider, + app=app, ) # Step 2: Post-process the results! @@ -496,7 +498,16 @@ def _convert_tool_calls_to_text( @staticmethod async def _get_agent_for_eval( module_name: str, agent_name: Optional[str] = None - ) -> BaseAgent: + ) -> tuple[BaseAgent, Optional[App]]: + """Returns the (agent_for_eval, app) pair for the given module. + + If the module exposes an `App` instance via `agent.app`, that App is + returned alongside the agent to evaluate, so `app.plugins`, context-cache, + and resumability configs participate in the eval run. Otherwise `app` is + None and only the bare agent is returned. When `agent_name` is provided, + the returned agent is the corresponding sub-agent, but the App (if any) is + still surfaced so its application-wide configuration is honored. + """ module_path = f"{module_name}" agent_module = importlib.import_module(module_path) @@ -522,12 +533,16 @@ async def _get_agent_for_eval( " get_agent_async method." ) + app = getattr(agent_module_with_agent, "app", None) + if not isinstance(app, App): + app = None + agent_for_eval = root_agent if agent_name: agent_for_eval = root_agent.find_agent(agent_name) assert agent_for_eval, f"Sub-Agent `{agent_name}` not found." - return agent_for_eval + return agent_for_eval, app @staticmethod def _get_eval_sets_manager( @@ -554,6 +569,7 @@ async def _get_eval_results_by_eval_id( eval_metrics: list[EvalMetric], num_runs: int, user_simulator_provider: UserSimulatorProvider, + app: Optional[App] = None, ) -> dict[str, list[EvalCaseResult]]: """Returns EvalCaseResults grouped by eval case id. @@ -578,6 +594,7 @@ async def _get_eval_results_by_eval_id( app_name=app_name, eval_set=eval_set ), user_simulator_provider=user_simulator_provider, + app=app, ) inference_requests = [ diff --git a/src/google/adk/evaluation/evaluation_generator.py b/src/google/adk/evaluation/evaluation_generator.py index 910c192cd36..5ccf691f98b 100644 --- a/src/google/adk/evaluation/evaluation_generator.py +++ b/src/google/adk/evaluation/evaluation_generator.py @@ -36,6 +36,7 @@ from ..agents.llm_agent import Agent from ..agents.run_config import RunConfig from ..agents.run_config import StreamingMode +from ..apps.app import App from ..artifacts.base_artifact_service import BaseArtifactService from ..artifacts.in_memory_artifact_service import InMemoryArtifactService from ..events.event import Event @@ -43,6 +44,7 @@ from ..memory.base_memory_service import BaseMemoryService from ..memory.in_memory_memory_service import InMemoryMemoryService from ..models.llm_request import LlmRequest +from ..plugins.base_plugin import BasePlugin from ..runners import Runner from ..sessions.base_session_service import BaseSessionService from ..sessions.in_memory_session_service import InMemorySessionService @@ -73,6 +75,39 @@ _DEFAULT_AUTHOR = "agent" +def _build_eval_runner_kwargs( + root_agent: Agent, + app_name: str, + app: Optional[App], + internal_eval_plugins: list[BasePlugin], +) -> dict[str, Any]: + """Returns the Runner kwargs used to evaluate `root_agent`. + + When `app` is provided, the Runner is built from a copy of the App with the + internal eval plugins merged into `app.plugins`, so the App's + `context_cache_config`, `resumability_config`, and any other + application-wide configuration participate in the eval run. The copy leaves + the caller's App instance untouched, and `root_agent` is overridden so the + Runner targets the agent the caller asked to evaluate, which may be a + sub-agent. When `app` is None, the Runner is built from the bare + `root_agent` with only the internal eval plugins. + """ + if app is None: + return { + "app_name": app_name, + "agent": root_agent, + "plugins": internal_eval_plugins, + } + + runner_app = app.model_copy( + update={ + "plugins": list(app.plugins) + internal_eval_plugins, + "root_agent": root_agent, + } + ) + return {"app": runner_app, "app_name": app_name} + + class EvalCaseResponses(BaseModel): """Contains multiple responses associated with an EvalCase. @@ -349,20 +384,30 @@ async def _process_query( """Process a query using the agent and evaluation dataset.""" module_path = f"{module_name}" agent_module = importlib.import_module(module_path) - root_agent = agent_module.agent.root_agent + # Prefer the wrapping `App` when the module exposes one, so that + # `app.plugins`, context-cache, and resumability configs participate + # in eval runs the same way they do for `adk web` / `adk run`. + app_obj = getattr(agent_module.agent, "app", None) + if isinstance(app_obj, App): + root_agent = app_obj.root_agent + else: + app_obj = None + root_agent = agent_module.agent.root_agent reset_func = getattr(agent_module.agent, "reset_data", None) agent_to_evaluate = root_agent if agent_name: - agent_to_evaluate = root_agent.find_agent(agent_name) - assert agent_to_evaluate, f"Sub-Agent `{agent_name}` not found." + found_agent = root_agent.find_agent(agent_name) + assert found_agent, f"Sub-Agent `{agent_name}` not found." + agent_to_evaluate = found_agent return await EvaluationGenerator._generate_inferences_from_root_agent( agent_to_evaluate, user_simulator=user_simulator, reset_func=reset_func, initial_session=initial_session, + app=app_obj, ) @staticmethod @@ -467,8 +512,14 @@ async def _generate_inferences_from_root_agent_live( artifact_service: Optional[BaseArtifactService] = None, memory_service: Optional[BaseMemoryService] = None, live_timeout_seconds: int = DEFAULT_LIVE_TIMEOUT_SECONDS, + app: Optional[App] = None, ) -> list[Invocation]: - """Scrapes the root agent in coordination with the user simulator in live mode.""" + """Scrapes the root agent in coordination with the user simulator in live mode. + + Mirrors `_generate_inferences_from_root_agent`: when `app` is provided the + Runner carries the App's plugins and configuration, otherwise the bare + `root_agent` is used. + """ if not session_service: session_service = InMemorySessionService() @@ -504,13 +555,21 @@ async def _generate_inferences_from_root_agent_live( request_intercepter_plugin = _RequestIntercepterPlugin( name="request_intercepter_plugin" ) - async with Runner( + runner_kwargs = _build_eval_runner_kwargs( + root_agent=root_agent, app_name=app_name, - agent=root_agent, + app=app, + internal_eval_plugins=[ + request_intercepter_plugin, + ensure_retry_options_plugin, + ], + ) + + async with Runner( + **runner_kwargs, artifact_service=artifact_service, session_service=session_service, memory_service=memory_service, - plugins=[request_intercepter_plugin, ensure_retry_options_plugin], ) as runner: events: list[Event] = [] @@ -573,8 +632,17 @@ async def _generate_inferences_from_root_agent( session_service: Optional[BaseSessionService] = None, artifact_service: Optional[BaseArtifactService] = None, memory_service: Optional[BaseMemoryService] = None, + app: Optional[App] = None, ) -> list[Invocation]: - """Scrapes the root agent in coordination with the user simulator.""" + """Scrapes the root agent in coordination with the user simulator. + + If `app` is provided, the eval Runner is built from a copy of the App + with internal eval plugins merged into `app.plugins`, preserving the + App's `context_cache_config`, `resumability_config`, and any other + application-wide configuration. Otherwise the Runner is built from + the bare `root_agent` with only the internal eval plugins, matching + the legacy behavior. + """ if not session_service: session_service = InMemorySessionService() @@ -611,13 +679,21 @@ async def _generate_inferences_from_root_agent( ensure_retry_options_plugin = EnsureRetryOptionsPlugin( name="ensure_retry_options" ) - async with Runner( + runner_kwargs = _build_eval_runner_kwargs( + root_agent=root_agent, app_name=app_name, - agent=root_agent, + app=app, + internal_eval_plugins=[ + request_intercepter_plugin, + ensure_retry_options_plugin, + ], + ) + + async with Runner( + **runner_kwargs, artifact_service=artifact_service, session_service=session_service, memory_service=memory_service, - plugins=[request_intercepter_plugin, ensure_retry_options_plugin], ) as runner: events: list[Event] = [] while True: diff --git a/src/google/adk/evaluation/local_eval_service.py b/src/google/adk/evaluation/local_eval_service.py index 3246f8de07a..0eb08fd2a9c 100644 --- a/src/google/adk/evaluation/local_eval_service.py +++ b/src/google/adk/evaluation/local_eval_service.py @@ -25,6 +25,7 @@ from typing_extensions import override from ..agents.base_agent import BaseAgent +from ..apps.app import App from ..artifacts.base_artifact_service import BaseArtifactService from ..artifacts.in_memory_artifact_service import InMemoryArtifactService from ..errors.not_found_error import NotFoundError @@ -123,8 +124,20 @@ def __init__( session_id_supplier: Callable[[], str] = _get_session_id, user_simulator_provider: UserSimulatorProvider = UserSimulatorProvider(), memory_service: Optional[BaseMemoryService] = None, + *, + app: Optional[App] = None, ): + """Initializes a LocalEvalService. + + Args: + app: Optional `App` that wraps `root_agent`. When provided, eval runs + are executed through a Runner built from the App, so `app.plugins`, + `app.context_cache_config`, and `app.resumability_config` are + honored during inference. When None, the legacy bare-agent path is + used. + """ self._root_agent = root_agent + self._app = app self._eval_sets_manager = eval_sets_manager metric_evaluator_registry = ( metric_evaluator_registry or DEFAULT_METRIC_EVALUATOR_REGISTRY @@ -528,6 +541,7 @@ async def _perform_inference_single_eval_item( artifact_service=self._artifact_service, memory_service=self._memory_service, live_timeout_seconds=live_timeout_seconds, + app=self._app, ) else: inferences = ( @@ -541,6 +555,7 @@ async def _perform_inference_single_eval_item( session_service=self._session_service, artifact_service=self._artifact_service, memory_service=self._memory_service, + app=self._app, ) ) diff --git a/tests/unittests/cli/utils/test_cli_eval.py b/tests/unittests/cli/utils/test_cli_eval.py index c8cb82e41f2..04f44699dd8 100644 --- a/tests/unittests/cli/utils/test_cli_eval.py +++ b/tests/unittests/cli/utils/test_cli_eval.py @@ -19,6 +19,8 @@ from types import SimpleNamespace from unittest import mock +from google.adk.agents.base_agent import BaseAgent +from google.adk.apps.app import App from google.adk.cli.cli_eval import get_root_agent import pytest @@ -175,3 +177,87 @@ def test_parse_evals_splits_case_selector_from_right(): assert parse_and_get_evals_to_run([r"C:\evals\set.json:case1,case2"]) == { r"C:\evals\set.json": ["case1", "case2"] } + + +def _patch_agent_module(monkeypatch, agent_namespace): + """Patches `_get_agent_module` to return a stub whose `.agent` matches.""" + monkeypatch.setattr( + "google.adk.cli.cli_eval._get_agent_module", + lambda _path: SimpleNamespace(agent=agent_namespace), + ) + + +@pytest.mark.asyncio +async def test_get_app_or_root_agent_with_app(monkeypatch): + """When the module exposes an App, both app and its root_agent are returned.""" + root_agent = BaseAgent(name="root_agent") + app = App(name="my_app", root_agent=root_agent) + _patch_agent_module( + monkeypatch, SimpleNamespace(root_agent=root_agent, app=app) + ) + + from google.adk.cli.cli_eval import get_app_or_root_agent + + resolved_app, resolved_root = await get_app_or_root_agent("some/path") + assert resolved_app is app + assert resolved_root is root_agent + + +@pytest.mark.asyncio +async def test_get_app_or_root_agent_without_app(monkeypatch): + """When only `root_agent` is exposed, app is None.""" + root_agent = BaseAgent(name="root_agent") + _patch_agent_module(monkeypatch, SimpleNamespace(root_agent=root_agent)) + + from google.adk.cli.cli_eval import get_app_or_root_agent + + resolved_app, resolved_root = await get_app_or_root_agent("some/path") + assert resolved_app is None + assert resolved_root is root_agent + + +@pytest.mark.asyncio +async def test_get_app_or_root_agent_supports_get_agent_async(monkeypatch): + """Modules exposing only `get_agent_async` still resolve, with app None.""" + root_agent = BaseAgent(name="root_agent") + get_agent_async = mock.AsyncMock(return_value=(root_agent, object())) + _patch_agent_module( + monkeypatch, SimpleNamespace(get_agent_async=get_agent_async) + ) + + from google.adk.cli.cli_eval import get_app_or_root_agent + + resolved_app, resolved_root = await get_app_or_root_agent("some/path") + assert resolved_app is None + assert resolved_root is root_agent + get_agent_async.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_get_app_or_root_agent_app_attribute_not_an_app_instance( + monkeypatch, +): + """If `app` exists but is not an App, it is ignored and we fall back.""" + root_agent = BaseAgent(name="root_agent") + _patch_agent_module( + monkeypatch, + SimpleNamespace(root_agent=root_agent, app="not-an-app"), + ) + + from google.adk.cli.cli_eval import get_app_or_root_agent + + resolved_app, resolved_root = await get_app_or_root_agent("some/path") + assert resolved_app is None + assert resolved_root is root_agent + + +@pytest.mark.asyncio +async def test_get_root_agent_back_compat(monkeypatch): + """Existing `get_root_agent` callers keep getting the bare agent back.""" + root_agent = BaseAgent(name="root_agent") + app = App(name="my_app", root_agent=root_agent) + _patch_agent_module( + monkeypatch, SimpleNamespace(root_agent=root_agent, app=app) + ) + + assert await get_root_agent("some/path") is root_agent diff --git a/tests/unittests/cli/utils/test_cli_tools_click.py b/tests/unittests/cli/utils/test_cli_tools_click.py index c4f4a8544f3..9cc5d003770 100644 --- a/tests/unittests/cli/utils/test_cli_tools_click.py +++ b/tests/unittests/cli/utils/test_cli_tools_click.py @@ -61,10 +61,17 @@ def mock_load_eval_set_from_file(): @pytest.fixture def mock_get_root_agent(): + """Patches the agent resolver used by the eval CLI. + + `cli_eval` resolves agents via `get_app_or_root_agent` (which returns + `(app, root_agent)`); the eval-set tests don't exercise the App path, + so we yield `(None, root_agent)`. + """ with mock.patch( - "google.adk.cli.cli_eval.get_root_agent", new_callable=mock.AsyncMock + "google.adk.cli.cli_eval.get_app_or_root_agent", + new_callable=mock.AsyncMock, ) as mock_func: - mock_func.return_value = root_agent + mock_func.return_value = (None, root_agent) yield mock_func diff --git a/tests/unittests/evaluation/test_agent_evaluator.py b/tests/unittests/evaluation/test_agent_evaluator.py new file mode 100644 index 00000000000..5538161ca38 --- /dev/null +++ b/tests/unittests/evaluation/test_agent_evaluator.py @@ -0,0 +1,161 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the App-aware threading in AgentEvaluator.""" + +from __future__ import annotations + +from types import SimpleNamespace + +from google.adk.agents.base_agent import BaseAgent +from google.adk.apps.app import App +from google.adk.evaluation.agent_evaluator import AgentEvaluator +from google.adk.evaluation.eval_set import EvalSet +from google.adk.evaluation.simulation.user_simulator_provider import UserSimulatorProvider +import pytest + + +class TestGetAgentForEval: + """Resolution of the wrapping App alongside the agent to evaluate.""" + + @pytest.mark.asyncio + async def test_resolves_app_when_module_exposes_one(self, mocker): + """When the module's `agent` exposes an `app`, it is returned too.""" + root_agent = BaseAgent(name="root_agent") + app = App(name="my_app", root_agent=root_agent) + fake_module = SimpleNamespace( + agent=SimpleNamespace(root_agent=root_agent, app=app) + ) + mocker.patch("importlib.import_module", return_value=fake_module) + + resolved_agent, resolved_app = await AgentEvaluator._get_agent_for_eval( + module_name="some.module" + ) + + assert resolved_agent is root_agent + assert resolved_app is app + + @pytest.mark.asyncio + async def test_returns_none_app_when_module_has_no_app(self, mocker): + """When only `root_agent` is exposed, app is None.""" + root_agent = BaseAgent(name="root_agent") + fake_module = SimpleNamespace(agent=SimpleNamespace(root_agent=root_agent)) + mocker.patch("importlib.import_module", return_value=fake_module) + + resolved_agent, resolved_app = await AgentEvaluator._get_agent_for_eval( + module_name="some.module" + ) + + assert resolved_agent is root_agent + assert resolved_app is None + + @pytest.mark.asyncio + async def test_ignores_app_attribute_that_is_not_an_app(self, mocker): + """A non-App `app` attribute is ignored and app resolves to None.""" + root_agent = BaseAgent(name="root_agent") + fake_module = SimpleNamespace( + agent=SimpleNamespace(root_agent=root_agent, app="not-an-app") + ) + mocker.patch("importlib.import_module", return_value=fake_module) + + resolved_agent, resolved_app = await AgentEvaluator._get_agent_for_eval( + module_name="some.module" + ) + + assert resolved_agent is root_agent + assert resolved_app is None + + @pytest.mark.asyncio + async def test_surfaces_app_even_when_selecting_sub_agent(self, mocker): + """A sub-agent is returned for eval, but the wrapping App is still surfaced.""" + sub_agent = BaseAgent(name="sub_agent") + root_agent = BaseAgent(name="root_agent", sub_agents=[sub_agent]) + app = App(name="my_app", root_agent=root_agent) + fake_module = SimpleNamespace( + agent=SimpleNamespace(root_agent=root_agent, app=app) + ) + mocker.patch("importlib.import_module", return_value=fake_module) + + resolved_agent, resolved_app = await AgentEvaluator._get_agent_for_eval( + module_name="some.module", agent_name="sub_agent" + ) + + assert resolved_agent is sub_agent + assert resolved_app is app + + +class TestGetEvalResultsByEvalId: + """The pytest-gate path forwards the App into LocalEvalService.""" + + @staticmethod + def _empty_async_gen_factory(): + async def _agen(*args, **kwargs): + return + yield # pragma: no cover - marks this as an async generator + + return _agen + + @pytest.mark.asyncio + async def test_app_is_forwarded_to_local_eval_service(self, mocker): + """`_get_eval_results_by_eval_id` passes `app=` into LocalEvalService.""" + root_agent = BaseAgent(name="root_agent") + app = App(name="my_app", root_agent=root_agent) + + mock_service_cls = mocker.patch( + "google.adk.evaluation.local_eval_service.LocalEvalService" + ) + mock_service = mock_service_cls.return_value + mock_service.perform_inference = mocker.MagicMock( + side_effect=self._empty_async_gen_factory() + ) + mock_service.evaluate = mocker.MagicMock( + side_effect=self._empty_async_gen_factory() + ) + + await AgentEvaluator._get_eval_results_by_eval_id( + agent_for_eval=root_agent, + eval_set=EvalSet(eval_set_id="set-1", eval_cases=[]), + eval_metrics=[], + num_runs=1, + user_simulator_provider=UserSimulatorProvider(), + app=app, + ) + + assert mock_service_cls.call_args.kwargs["app"] is app + + @pytest.mark.asyncio + async def test_none_app_is_forwarded_by_default(self, mocker): + """When no App is provided, LocalEvalService receives app=None.""" + root_agent = BaseAgent(name="root_agent") + + mock_service_cls = mocker.patch( + "google.adk.evaluation.local_eval_service.LocalEvalService" + ) + mock_service = mock_service_cls.return_value + mock_service.perform_inference = mocker.MagicMock( + side_effect=self._empty_async_gen_factory() + ) + mock_service.evaluate = mocker.MagicMock( + side_effect=self._empty_async_gen_factory() + ) + + await AgentEvaluator._get_eval_results_by_eval_id( + agent_for_eval=root_agent, + eval_set=EvalSet(eval_set_id="set-1", eval_cases=[]), + eval_metrics=[], + num_runs=1, + user_simulator_provider=UserSimulatorProvider(), + ) + + assert mock_service_cls.call_args.kwargs["app"] is None diff --git a/tests/unittests/evaluation/test_evaluation_generator.py b/tests/unittests/evaluation/test_evaluation_generator.py index e916cb19225..806eb9922a9 100644 --- a/tests/unittests/evaluation/test_evaluation_generator.py +++ b/tests/unittests/evaluation/test_evaluation_generator.py @@ -16,6 +16,8 @@ import asyncio +from google.adk.agents.base_agent import BaseAgent +from google.adk.apps.app import App from google.adk.evaluation.app_details import AgentDetails from google.adk.evaluation.app_details import AppDetails from google.adk.evaluation.conversation_scenarios import ConversationScenario @@ -33,6 +35,7 @@ from google.adk.events.event import Event from google.adk.events.event_actions import EventActions from google.adk.models.llm_request import LlmRequest +from google.adk.plugins.base_plugin import BasePlugin from google.genai import types import pytest @@ -1142,3 +1145,133 @@ def test_convert_events_preserves_tool_calls_when_skip_summarization(): assert len(tool_calls) == 1 assert tool_calls[0].name == "execute_sql" assert tool_calls[0].args == {"project_id": "my-proj", "query": "SELECT 1"} + + +class _SpyPlugin(BasePlugin): + """A user-defined plugin used to assert merge behavior.""" + + pass + + +class TestGenerateInferencesFromRootAgentWithApp: + """Tests that App.plugins / configs are honored when an App is provided.""" + + @pytest.fixture + def runner_cls(self, mocker): + """Patches Runner and returns the patched class for kwargs inspection.""" + mock_runner_cls = mocker.patch( + "google.adk.evaluation.evaluation_generator.Runner" + ) + mock_runner_instance = mocker.AsyncMock() + mock_runner_instance.__aenter__.return_value = mock_runner_instance + mock_runner_cls.return_value = mock_runner_instance + yield mock_runner_cls + + @pytest.fixture + def stop_immediately_simulator(self, mocker): + """Returns a UserSimulator that stops on first call (no inference work).""" + sim = mocker.MagicMock(spec=UserSimulator) + sim.get_next_user_message = mocker.AsyncMock( + return_value=NextUserMessage( + status=UserSimulatorStatus.STOP_SIGNAL_DETECTED + ) + ) + return sim + + @pytest.mark.asyncio + async def test_runner_built_from_app_when_provided( + self, runner_cls, mock_session_service, stop_immediately_simulator + ): + """When `app` is passed, Runner is built with `app=` (merged) instead of `agent=`.""" + root_agent = BaseAgent(name="root_agent") + user_plugin = _SpyPlugin(name="user_plugin") + app = App(name="my_app", root_agent=root_agent, plugins=[user_plugin]) + + await EvaluationGenerator._generate_inferences_from_root_agent( + root_agent=root_agent, + user_simulator=stop_immediately_simulator, + app=app, + ) + + runner_cls.assert_called_once() + kwargs = runner_cls.call_args.kwargs + assert "agent" not in kwargs, ( + "Runner must not receive `agent=` when `app=` is provided " + "(would raise ValueError)." + ) + assert "plugins" not in kwargs, ( + "Runner must not receive `plugins=` when `app=` is provided " + "(would raise ValueError)." + ) + runner_app = kwargs["app"] + assert isinstance(runner_app, App) + plugin_names = [p.name for p in runner_app.plugins] + assert ( + "user_plugin" in plugin_names + ), "User plugin must be preserved in the merged App passed to Runner." + assert "request_intercepter_plugin" in plugin_names + assert "ensure_retry_options" in plugin_names + + @pytest.mark.asyncio + async def test_user_app_is_not_mutated( + self, runner_cls, mock_session_service, stop_immediately_simulator + ): + """The user's App instance must not be mutated across eval runs.""" + root_agent = BaseAgent(name="root_agent") + user_plugin = _SpyPlugin(name="user_plugin") + app = App(name="my_app", root_agent=root_agent, plugins=[user_plugin]) + original_plugins_id = id(app.plugins) + + for _ in range(3): + await EvaluationGenerator._generate_inferences_from_root_agent( + root_agent=root_agent, + user_simulator=stop_immediately_simulator, + app=app, + ) + + # The user's App instance must still hold exactly its original plugin set, + # regardless of how many eval runs reused it. + assert app.plugins == [user_plugin] + assert id(app.plugins) == original_plugins_id + + @pytest.mark.asyncio + async def test_runner_falls_back_to_bare_agent_when_no_app( + self, runner_cls, mock_session_service, stop_immediately_simulator + ): + """When `app` is None, Runner is built with the legacy `agent=`/`plugins=` shape.""" + root_agent = BaseAgent(name="root_agent") + + await EvaluationGenerator._generate_inferences_from_root_agent( + root_agent=root_agent, + user_simulator=stop_immediately_simulator, + ) + + runner_cls.assert_called_once() + kwargs = runner_cls.call_args.kwargs + assert "app" not in kwargs + assert kwargs["agent"] is root_agent + plugin_names = [p.name for p in kwargs["plugins"]] + assert plugin_names == [ + "request_intercepter_plugin", + "ensure_retry_options", + ] + + @pytest.mark.asyncio + async def test_root_agent_override_propagates_to_merged_app( + self, runner_cls, mock_session_service, stop_immediately_simulator + ): + """If a sub-agent is passed as root_agent, the merged App reflects that.""" + full_root = BaseAgent(name="full_root") + sub_agent = BaseAgent(name="sub_agent") + app = App(name="my_app", root_agent=full_root) + + await EvaluationGenerator._generate_inferences_from_root_agent( + root_agent=sub_agent, + user_simulator=stop_immediately_simulator, + app=app, + ) + + runner_app = runner_cls.call_args.kwargs["app"] + assert runner_app.root_agent is sub_agent + # User's App must be untouched. + assert app.root_agent is full_root diff --git a/tests/unittests/evaluation/test_local_eval_service.py b/tests/unittests/evaluation/test_local_eval_service.py index 770ea3a9a2f..fdd098550db 100644 --- a/tests/unittests/evaluation/test_local_eval_service.py +++ b/tests/unittests/evaluation/test_local_eval_service.py @@ -18,6 +18,7 @@ from typing import Optional from google.adk.agents.llm_agent import LlmAgent +from google.adk.apps.app import App from google.adk.errors.not_found_error import NotFoundError from google.adk.evaluation.base_eval_service import EvaluateConfig from google.adk.evaluation.base_eval_service import EvaluateRequest @@ -908,6 +909,7 @@ async def test_perform_inference_single_eval_item_live( artifact_service=eval_service._artifact_service, memory_service=eval_service._memory_service, live_timeout_seconds=600, + app=None, ) @@ -938,6 +940,10 @@ async def test_perform_inference_single_eval_item_non_live( live_timeout_seconds=300, ) + # The non-live branch forwards `app=self._app` to the underlying + # `_generate_inferences_from_root_agent` (see fix in + # `local_eval_service.py`). The `eval_service` fixture builds the service + # without an `app`, so we expect `app=None`. mock_generate.assert_called_once_with( root_agent=dummy_agent, user_simulator=mock_user_sim, @@ -946,4 +952,113 @@ async def test_perform_inference_single_eval_item_non_live( session_service=eval_service._session_service, artifact_service=eval_service._artifact_service, memory_service=eval_service._memory_service, + app=None, ) + + +@pytest.mark.asyncio +async def test_perform_inference_forwards_app_to_evaluation_generator( + dummy_agent, mock_eval_sets_manager, mocker +): + """LocalEvalService passes its `app` through to _generate_inferences_from_root_agent.""" + app = App(name="test_app", root_agent=dummy_agent) + + eval_case = EvalCase(eval_id="case-1", conversation=[]) + mock_eval_sets_manager.get_eval_set.return_value = EvalSet( + eval_set_id="set-1", + eval_cases=[eval_case], + ) + + mock_generate = mocker.patch( + "google.adk.evaluation.local_eval_service.EvaluationGenerator._generate_inferences_from_root_agent", + new=mocker.AsyncMock(return_value=[]), + ) + + service = LocalEvalService( + root_agent=dummy_agent, + eval_sets_manager=mock_eval_sets_manager, + app=app, + ) + + request = InferenceRequest( + app_name="test_app", + eval_set_id="set-1", + eval_case_ids=["case-1"], + inference_config=InferenceConfig(), + ) + async for _ in service.perform_inference(inference_request=request): + pass + + mock_generate.assert_awaited_once() + assert mock_generate.await_args.kwargs["app"] is app + + +@pytest.mark.asyncio +async def test_perform_inference_passes_none_when_no_app( + dummy_agent, mock_eval_sets_manager, mocker +): + """When LocalEvalService has no `app`, it forwards None (legacy behavior).""" + eval_case = EvalCase(eval_id="case-1", conversation=[]) + mock_eval_sets_manager.get_eval_set.return_value = EvalSet( + eval_set_id="set-1", + eval_cases=[eval_case], + ) + + mock_generate = mocker.patch( + "google.adk.evaluation.local_eval_service.EvaluationGenerator._generate_inferences_from_root_agent", + new=mocker.AsyncMock(return_value=[]), + ) + + service = LocalEvalService( + root_agent=dummy_agent, + eval_sets_manager=mock_eval_sets_manager, + ) + + request = InferenceRequest( + app_name="test_app", + eval_set_id="set-1", + eval_case_ids=["case-1"], + inference_config=InferenceConfig(), + ) + async for _ in service.perform_inference(inference_request=request): + pass + + mock_generate.assert_awaited_once() + assert mock_generate.await_args.kwargs["app"] is None + + +@pytest.mark.asyncio +async def test_perform_inference_live_forwards_app( + dummy_agent, mock_eval_sets_manager, mocker +): + """The live branch forwards `app` the same way the non-live branch does.""" + app = App(name="test_app", root_agent=dummy_agent) + + eval_case = EvalCase(eval_id="case-1", conversation=[]) + mock_eval_sets_manager.get_eval_set.return_value = EvalSet( + eval_set_id="set-1", + eval_cases=[eval_case], + ) + + mock_generate_live = mocker.patch( + "google.adk.evaluation.local_eval_service.EvaluationGenerator._generate_inferences_from_root_agent_live", + new=mocker.AsyncMock(return_value=[]), + ) + + service = LocalEvalService( + root_agent=dummy_agent, + eval_sets_manager=mock_eval_sets_manager, + app=app, + ) + + request = InferenceRequest( + app_name="test_app", + eval_set_id="set-1", + eval_case_ids=["case-1"], + inference_config=InferenceConfig(use_live=True), + ) + async for _ in service.perform_inference(inference_request=request): + pass + + mock_generate_live.assert_awaited_once() + assert mock_generate_live.await_args.kwargs["app"] is app