fix(eval): wire App plugins through eval paths

- Thread app= from cli, dev_server, and AgentEvaluator
- Build eval Runner from app.model_copy with merged plugins
- Share Runner construction across live and non-live paths
- Add unit tests covering each eval surface

Closes #5503

Author: doughayden

src/google/adk/cli/cli_eval.py

@@ -27,6 +27,7 @@
 from google.genai import types as genai_types
 
 from ..agents.base_agent import BaseAgent
+from ..apps.app import App
 from ..evaluation.base_eval_service import BaseEvalService
 from ..evaluation.base_eval_service import EvaluateConfig
 from ..evaluation.base_eval_service import EvaluateRequest
@@ -75,20 +76,43 @@ def _get_agent_module(agent_module_file_path: str) -> ModuleType:
   return _import_from_path(module_name, file_path)
 
 
-async def get_root_agent(agent_module_file_path: str) -> BaseAgent:
-  """Returns root agent given the agent module."""
+async def get_app_or_root_agent(
+    agent_module_file_path: str,
+) -> tuple[Optional[App], BaseAgent]:
+  """Returns the (app, root_agent) pair for the given agent module.
+
+  If the module exposes an `App` instance via `app`, that App and its
+  `root_agent` are returned. Otherwise `app` is None and the root agent is
+  resolved the same way as `get_root_agent`. This lets eval flows participate
+  in the App's plugin / cache / resumability lifecycle when one is defined,
+  while preserving the bare-`root_agent` path for projects that don't use App.
+  """
   agent_module = _get_agent_module(agent_module_file_path)
   agent_module_with_agent = getattr(agent_module, "agent", agent_module)
+  app = getattr(agent_module_with_agent, "app", None)
+  if isinstance(app, App):
+    return app, cast(BaseAgent, app.root_agent)
   if hasattr(agent_module_with_agent, "root_agent"):
-    return cast(BaseAgent, agent_module_with_agent.root_agent)
+    return None, cast(BaseAgent, agent_module_with_agent.root_agent)
   elif hasattr(agent_module_with_agent, "get_agent_async"):
     root_agent, _ = await agent_module_with_agent.get_agent_async()
-    return cast(BaseAgent, root_agent)
+    return None, cast(BaseAgent, root_agent)
   raise ValueError(
       "Agent module should have either `root_agent` or `get_agent_async`."
   )
 
 
+async def get_root_agent(agent_module_file_path: str) -> BaseAgent:
+  """Returns root agent given the agent module.
+
+  Kept for backward compatibility. New callers should prefer
+  `get_app_or_root_agent`, which also surfaces the wrapping `App` (if any)
+  so plugins, context-cache, and resumability configs are honored.
+  """
+  _, root_agent = await get_app_or_root_agent(agent_module_file_path)
+  return root_agent
+
+
 def try_get_reset_func(agent_module_file_path: str) -> Any:
   """Returns reset function for the agent, if present, given the agent module."""
   agent_module = _get_agent_module(agent_module_file_path)

src/google/adk/cli/cli_tools_click.py

@@ -1022,7 +1022,7 @@ def cli_eval(
     from ..evaluation.simulation.user_simulator_provider import UserSimulatorProvider
     from .cli_eval import _collect_eval_results
     from .cli_eval import _collect_inferences
-    from .cli_eval import get_root_agent
+    from .cli_eval import get_app_or_root_agent
     from .cli_eval import parse_and_get_evals_to_run
     from .cli_eval import pretty_print_eval_result
   except ModuleNotFoundError as mnf:
@@ -1032,7 +1032,7 @@ def cli_eval(
   print(f"Using evaluation criteria: {eval_config}")
   eval_metrics = get_eval_metrics_from_config(eval_config)
 
-  root_agent = asyncio.run(get_root_agent(agent_module_file_path))
+  app, root_agent = asyncio.run(get_app_or_root_agent(agent_module_file_path))
   app_name = os.path.basename(agent_module_file_path)
   agents_dir = os.path.dirname(agent_module_file_path)
   eval_sets_manager = None
@@ -1124,6 +1124,7 @@ def cli_eval(
         eval_set_results_manager=eval_set_results_manager,
         user_simulator_provider=user_simulator_provider,
         metric_evaluator_registry=metric_evaluator_registry,
+        app=app,
     )
 
     inference_results = asyncio.run(

src/google/adk/cli/dev_server.py

@@ -48,6 +48,7 @@
 import yaml
 
 from . import agent_graph
+from ..apps.app import App
 from ..errors.not_found_error import NotFoundError
 from ..evaluation.base_eval_service import InferenceConfig
 from ..evaluation.base_eval_service import InferenceRequest
@@ -1068,6 +1069,7 @@ async def run_eval(
 
         agent_or_app = self.agent_loader.load_agent(app_name)
         root_agent = self._get_root_agent(agent_or_app)
+        app = agent_or_app if isinstance(agent_or_app, App) else None
 
         eval_case_results = []
 
@@ -1077,6 +1079,7 @@ async def run_eval(
             eval_set_results_manager=self.eval_set_results_manager,
             session_service=self.session_service,
             artifact_service=self.artifact_service,
+            app=app,
         )
         inference_request = InferenceRequest(
             app_name=app_name,

src/google/adk/evaluation/agent_evaluator.py

@@ -34,6 +34,7 @@
 from pydantic import ValidationError
 
 from ..agents.base_agent import BaseAgent
+from ..apps.app import App
 from ..utils.context_utils import Aclosing
 from .constants import MISSING_EVAL_DEPENDENCIES_MESSAGE
 from .eval_case import get_all_tool_calls
@@ -147,7 +148,7 @@ async def evaluate_eval_set(
     if eval_config is None:
       raise ValueError("`eval_config` is required.")
 
-    agent_for_eval = await AgentEvaluator._get_agent_for_eval(
+    agent_for_eval, app = await AgentEvaluator._get_agent_for_eval(
         module_name=agent_module, agent_name=agent_name
     )
     eval_metrics = get_eval_metrics_from_config(eval_config)
@@ -163,6 +164,7 @@ async def evaluate_eval_set(
         eval_metrics=eval_metrics,
         num_runs=num_runs,
         user_simulator_provider=user_simulator_provider,
+        app=app,
     )
 
     # Step 2: Post-process the results!
@@ -496,7 +498,16 @@ def _convert_tool_calls_to_text(
   @staticmethod
   async def _get_agent_for_eval(
       module_name: str, agent_name: Optional[str] = None
-  ) -> BaseAgent:
+  ) -> tuple[BaseAgent, Optional[App]]:
+    """Returns the (agent_for_eval, app) pair for the given module.
+
+    If the module exposes an `App` instance via `agent.app`, that App is
+    returned alongside the agent to evaluate, so `app.plugins`, context-cache,
+    and resumability configs participate in the eval run. Otherwise `app` is
+    None and only the bare agent is returned. When `agent_name` is provided,
+    the returned agent is the corresponding sub-agent, but the App (if any) is
+    still surfaced so its application-wide configuration is honored.
+    """
     module_path = f"{module_name}"
     agent_module = importlib.import_module(module_path)
 
@@ -522,12 +533,16 @@ async def _get_agent_for_eval(
           " get_agent_async method."
       )
 
+    app = getattr(agent_module_with_agent, "app", None)
+    if not isinstance(app, App):
+      app = None
+
     agent_for_eval = root_agent
     if agent_name:
       agent_for_eval = root_agent.find_agent(agent_name)
       assert agent_for_eval, f"Sub-Agent `{agent_name}` not found."
 
-    return agent_for_eval
+    return agent_for_eval, app
 
   @staticmethod
   def _get_eval_sets_manager(
@@ -554,6 +569,7 @@ async def _get_eval_results_by_eval_id(
       eval_metrics: list[EvalMetric],
       num_runs: int,
       user_simulator_provider: UserSimulatorProvider,
+      app: Optional[App] = None,
   ) -> dict[str, list[EvalCaseResult]]:
     """Returns EvalCaseResults grouped by eval case id.
 
@@ -578,6 +594,7 @@ async def _get_eval_results_by_eval_id(
             app_name=app_name, eval_set=eval_set
         ),
         user_simulator_provider=user_simulator_provider,
+        app=app,
     )
 
     inference_requests = [

src/google/adk/evaluation/evaluation_generator.py

@@ -36,13 +36,15 @@
 from ..agents.llm_agent import Agent
 from ..agents.run_config import RunConfig
 from ..agents.run_config import StreamingMode
+from ..apps.app import App
 from ..artifacts.base_artifact_service import BaseArtifactService
 from ..artifacts.in_memory_artifact_service import InMemoryArtifactService
 from ..events.event import Event
 from ..flows.llm_flows.functions import handle_function_calls_live
 from ..memory.base_memory_service import BaseMemoryService
 from ..memory.in_memory_memory_service import InMemoryMemoryService
 from ..models.llm_request import LlmRequest
+from ..plugins.base_plugin import BasePlugin
 from ..runners import Runner
 from ..sessions.base_session_service import BaseSessionService
 from ..sessions.in_memory_session_service import InMemorySessionService
@@ -73,6 +75,39 @@
 _DEFAULT_AUTHOR = "agent"
 
 
+def _build_eval_runner_kwargs(
+    root_agent: Agent,
+    app_name: str,
+    app: Optional[App],
+    internal_eval_plugins: list[BasePlugin],
+) -> dict[str, Any]:
+  """Returns the Runner kwargs used to evaluate `root_agent`.
+
+  When `app` is provided, the Runner is built from a copy of the App with the
+  internal eval plugins merged into `app.plugins`, so the App's
+  `context_cache_config`, `resumability_config`, and any other
+  application-wide configuration participate in the eval run. The copy leaves
+  the caller's App instance untouched, and `root_agent` is overridden so the
+  Runner targets the agent the caller asked to evaluate, which may be a
+  sub-agent. When `app` is None, the Runner is built from the bare
+  `root_agent` with only the internal eval plugins.
+  """
+  if app is None:
+    return {
+        "app_name": app_name,
+        "agent": root_agent,
+        "plugins": internal_eval_plugins,
+    }
+
+  runner_app = app.model_copy(
+      update={
+          "plugins": list(app.plugins) + internal_eval_plugins,
+          "root_agent": root_agent,
+      }
+  )
+  return {"app": runner_app, "app_name": app_name}
+
+
 class EvalCaseResponses(BaseModel):
   """Contains multiple responses associated with an EvalCase.
 
@@ -349,20 +384,30 @@ async def _process_query(
     """Process a query using the agent and evaluation dataset."""
     module_path = f"{module_name}"
     agent_module = importlib.import_module(module_path)
-    root_agent = agent_module.agent.root_agent
+    # Prefer the wrapping `App` when the module exposes one, so that
+    # `app.plugins`, context-cache, and resumability configs participate
+    # in eval runs the same way they do for `adk web` / `adk run`.
+    app_obj = getattr(agent_module.agent, "app", None)
+    if isinstance(app_obj, App):
+      root_agent = app_obj.root_agent
+    else:
+      app_obj = None
+      root_agent = agent_module.agent.root_agent
 
     reset_func = getattr(agent_module.agent, "reset_data", None)
 
     agent_to_evaluate = root_agent
     if agent_name:
-      agent_to_evaluate = root_agent.find_agent(agent_name)
-      assert agent_to_evaluate, f"Sub-Agent `{agent_name}` not found."
+      found_agent = root_agent.find_agent(agent_name)
+      assert found_agent, f"Sub-Agent `{agent_name}` not found."
+      agent_to_evaluate = found_agent
 
     return await EvaluationGenerator._generate_inferences_from_root_agent(
         agent_to_evaluate,
         user_simulator=user_simulator,
         reset_func=reset_func,
         initial_session=initial_session,
+        app=app_obj,
     )
 
   @staticmethod
@@ -467,8 +512,14 @@ async def _generate_inferences_from_root_agent_live(
       artifact_service: Optional[BaseArtifactService] = None,
       memory_service: Optional[BaseMemoryService] = None,
       live_timeout_seconds: int = DEFAULT_LIVE_TIMEOUT_SECONDS,
+      app: Optional[App] = None,
   ) -> list[Invocation]:
-    """Scrapes the root agent in coordination with the user simulator in live mode."""
+    """Scrapes the root agent in coordination with the user simulator in live mode.
+
+    Mirrors `_generate_inferences_from_root_agent`: when `app` is provided the
+    Runner carries the App's plugins and configuration, otherwise the bare
+    `root_agent` is used.
+    """
     if not session_service:
       session_service = InMemorySessionService()
 
@@ -504,13 +555,21 @@ async def _generate_inferences_from_root_agent_live(
     request_intercepter_plugin = _RequestIntercepterPlugin(
         name="request_intercepter_plugin"
     )
-    async with Runner(
+    runner_kwargs = _build_eval_runner_kwargs(
+        root_agent=root_agent,
         app_name=app_name,
-        agent=root_agent,
+        app=app,
+        internal_eval_plugins=[
+            request_intercepter_plugin,
+            ensure_retry_options_plugin,
+        ],
+    )
+
+    async with Runner(
+        **runner_kwargs,
         artifact_service=artifact_service,
         session_service=session_service,
         memory_service=memory_service,
-        plugins=[request_intercepter_plugin, ensure_retry_options_plugin],
     ) as runner:
       events: list[Event] = []
 
@@ -573,8 +632,17 @@ async def _generate_inferences_from_root_agent(
       session_service: Optional[BaseSessionService] = None,
       artifact_service: Optional[BaseArtifactService] = None,
       memory_service: Optional[BaseMemoryService] = None,
+      app: Optional[App] = None,
   ) -> list[Invocation]:
-    """Scrapes the root agent in coordination with the user simulator."""
+    """Scrapes the root agent in coordination with the user simulator.
+
+    If `app` is provided, the eval Runner is built from a copy of the App
+    with internal eval plugins merged into `app.plugins`, preserving the
+    App's `context_cache_config`, `resumability_config`, and any other
+    application-wide configuration. Otherwise the Runner is built from
+    the bare `root_agent` with only the internal eval plugins, matching
+    the legacy behavior.
+    """
 
     if not session_service:
       session_service = InMemorySessionService()
@@ -611,13 +679,21 @@ async def _generate_inferences_from_root_agent(
     ensure_retry_options_plugin = EnsureRetryOptionsPlugin(
         name="ensure_retry_options"
     )
-    async with Runner(
+    runner_kwargs = _build_eval_runner_kwargs(
+        root_agent=root_agent,
         app_name=app_name,
-        agent=root_agent,
+        app=app,
+        internal_eval_plugins=[
+            request_intercepter_plugin,
+            ensure_retry_options_plugin,
+        ],
+    )
+
+    async with Runner(
+        **runner_kwargs,
         artifact_service=artifact_service,
         session_service=session_service,
         memory_service=memory_service,
-        plugins=[request_intercepter_plugin, ensure_retry_options_plugin],
     ) as runner:
       events: list[Event] = []
       while True: