feat: GenAI Client(evals) - add Gemini Agents API related eval types

PiperOrigin-RevId: 942246076

Author: jsondai

agentplatform/_genai/_evals_common.py

@@ -556,12 +556,31 @@ def _resolve_inference_configs(
     return inference_configs
 
 
+def _is_gemini_agent_resource(agent: str) -> bool:
+    """Returns True if `agent` is a Gemini Agent resource name.
+
+    A Gemini Agent resource name has the format
+    `projects/{project}/locations/{location}/agents/{agent}`, as opposed to an
+    Agent Engine resource name which uses `.../reasoningEngines/{id}`.
+    """
+    parts = agent.split("/")
+    return (
+        len(parts) == 6
+        and parts[0] == "projects"
+        and parts[2] == "locations"
+        and parts[4] == "agents"
+        and bool(parts[1])
+        and bool(parts[3])
+        and bool(parts[5])
+    )
+
+
 def _add_evaluation_run_labels(
     labels: Optional[dict[str, str]] = None,
     agent: Optional[str] = None,
 ) -> Optional[dict[str, str]]:
     """Adds labels to the evaluation run."""
-    if agent:
+    if agent and "reasoningEngines/" in agent and not _is_gemini_agent_resource(agent):
         labels = labels or {}
         labels["vertex-ai-evaluation-agent-engine-id"] = agent.split(
             "reasoningEngines/"

agentplatform/_genai/evals.py

@@ -340,6 +340,13 @@ def _EvaluationInstance_to_vertex(
     if getv(from_object, ["rubric_groups"]) is not None:
         setv(to_object, ["rubricGroups"], getv(from_object, ["rubric_groups"]))
 
+    if getv(from_object, ["interactions_data_source"]) is not None:
+        setv(
+            to_object,
+            ["interactionsDataSource"],
+            getv(from_object, ["interactions_data_source"]),
+        )
+
     return to_object
 
 
@@ -2645,10 +2652,15 @@ def create_evaluation_run(
           display_name: The display name of the evaluation run.
           agent_info: The agent info to evaluate. Mutually exclusive with
               `inference_configs`.
-          agent: The agent engine resource name in str type, with format
-              `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`.
-              If provided, runs inference with the deployed agent to get agent responses
-              for evaluation. This is required if `agent_info` is provided.
+          agent: The agent resource name in str type. Accepts either an Agent
+              Engine resource name
+              `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`
+              or a Gemini Agent (Vertex AI Agent) resource name
+              `projects/{project}/locations/{location}/agents/{agent}`. When a Gemini
+              Agent resource is provided, the backend scrapes the agent to produce
+              agent responses. If an Agent Engine resource name is provided, runs
+              inference with the deployed agent to get agent responses for evaluation.
+              The `agent` parameter is required if `agent_info` is provided.
           user_simulator_config: The user simulator configuration for agent evaluation.
               If `agent_info` is provided without `inference_configs`, this config is used
               to automatically construct the inference configuration. If not specified,
@@ -2730,13 +2742,20 @@ def create_evaluation_run(
                 parsed_user_simulator_config.max_turn = 5
 
             candidate_name = parsed_agent_info.name or "candidate-1"
+            if agent and _evals_common._is_gemini_agent_resource(agent):
+                agent_run_config = types.AgentRunConfig(
+                    gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent),
+                    user_simulator_config=parsed_user_simulator_config,
+                )
+            else:
+                agent_run_config = types.AgentRunConfig(
+                    agent_engine=agent,
+                    user_simulator_config=parsed_user_simulator_config,
+                )
             inference_configs = {
                 candidate_name: types.EvaluationRunInferenceConfig(
                     agent_configs=parsed_agent_info.agents,
-                    agent_run_config=types.AgentRunConfig(
-                        agent_engine=agent,
-                        user_simulator_config=parsed_user_simulator_config,
-                    ),
+                    agent_run_config=agent_run_config,
                 )
             }
 
@@ -4439,10 +4458,15 @@ async def create_evaluation_run(
           display_name: The display name of the evaluation run.
           agent_info: The agent info to evaluate. Mutually exclusive with
               `inference_configs`.
-          agent: The agent engine resource name in str type, with format
-              `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`.
-              If provided, runs inference with the deployed agent to get agent responses
-              for evaluation. This is required if `agent_info` is provided.
+          agent: The agent resource name in str type. Accepts either an Agent
+              Engine resource name
+              `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`
+              or a Gemini Agent (Vertex AI Agent) resource name
+              `projects/{project}/locations/{location}/agents/{agent}`. When a Gemini
+              Agent resource is provided, the backend scrapes the agent to produce
+              agent responses. If an Agent Engine resource name is provided, runs
+              inference with the deployed agent to get agent responses for evaluation.
+              The `agent` parameter is required if `agent_info` is provided.
           user_simulator_config: The user simulator configuration for agent evaluation.
               If `agent_info` is provided without `inference_configs`, this config is used
               to automatically construct the inference configuration. If not specified,
@@ -4524,13 +4548,20 @@ async def create_evaluation_run(
                 parsed_user_simulator_config.max_turn = 5
 
             candidate_name = parsed_agent_info.name or "candidate-1"
+            if agent and _evals_common._is_gemini_agent_resource(agent):
+                agent_run_config = types.AgentRunConfig(
+                    gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent),
+                    user_simulator_config=parsed_user_simulator_config,
+                )
+            else:
+                agent_run_config = types.AgentRunConfig(
+                    agent_engine=agent,
+                    user_simulator_config=parsed_user_simulator_config,
+                )
             inference_configs = {
                 candidate_name: types.EvaluationRunInferenceConfig(
                     agent_configs=parsed_agent_info.agents,
-                    agent_run_config=types.AgentRunConfig(
-                        agent_engine=agent,
-                        user_simulator_config=parsed_user_simulator_config,
-                    ),
+                    agent_run_config=agent_run_config,
                 )
             }
 

agentplatform/_genai/types/__init__.py

@@ -603,6 +603,9 @@
 from .common import FlexStartDict
 from .common import FlexStartOrDict
 from .common import Framework
+from .common import GeminiAgentConfig
+from .common import GeminiAgentConfigDict
+from .common import GeminiAgentConfigOrDict
 from .common import GeminiExample
 from .common import GeminiExampleDict
 from .common import GeminiExampleOrDict
@@ -771,6 +774,9 @@
 from .common import IngestionDirectContentsSourceEventDict
 from .common import IngestionDirectContentsSourceEventOrDict
 from .common import IngestionDirectContentsSourceOrDict
+from .common import InteractionsDataSource
+from .common import InteractionsDataSourceDict
+from .common import InteractionsDataSourceOrDict
 from .common import IntermediateExtractedMemory
 from .common import IntermediateExtractedMemoryDict
 from .common import IntermediateExtractedMemoryOrDict
@@ -1980,6 +1986,9 @@
     "EvaluationRunAgentConfig",
     "EvaluationRunAgentConfigDict",
     "EvaluationRunAgentConfigOrDict",
+    "GeminiAgentConfig",
+    "GeminiAgentConfigDict",
+    "GeminiAgentConfigOrDict",
     "AgentRunConfig",
     "AgentRunConfigDict",
     "AgentRunConfigOrDict",
@@ -2043,6 +2052,9 @@
     "ResponseCandidate",
     "ResponseCandidateDict",
     "ResponseCandidateOrDict",
+    "InteractionsDataSource",
+    "InteractionsDataSourceDict",
+    "InteractionsDataSourceOrDict",
     "EvalCase",
     "EvalCaseDict",
     "EvalCaseOrDict",

agentplatform/_genai/types/common.py

@@ -2607,6 +2607,35 @@ class EvaluationRunAgentConfigDict(TypedDict, total=False):
 ]
 
 
+class GeminiAgentConfig(_common.BaseModel):
+    """Config for scraping a Gemini Agent.
+
+    A Gemini Agent is a Vertex AI Agent resource scraped via the Vertex
+    Interactions API.
+    """
+
+    gemini_agent: Optional[str] = Field(
+        default=None,
+        description="""The resource name of the Gemini Agent.
+      Format: `projects/{project}/locations/{location}/agents/{agent}`.""",
+    )
+
+
+class GeminiAgentConfigDict(TypedDict, total=False):
+    """Config for scraping a Gemini Agent.
+
+    A Gemini Agent is a Vertex AI Agent resource scraped via the Vertex
+    Interactions API.
+    """
+
+    gemini_agent: Optional[str]
+    """The resource name of the Gemini Agent.
+      Format: `projects/{project}/locations/{location}/agents/{agent}`."""
+
+
+GeminiAgentConfigOrDict = Union[GeminiAgentConfig, GeminiAgentConfigDict]
+
+
 class AgentRunConfig(_common.BaseModel):
     """Configuration for an Agent Run."""
 
@@ -2622,6 +2651,11 @@ class AgentRunConfig(_common.BaseModel):
         Contains configuration for a user simulator that
         uses an LLM to generate messages on behalf of the user.""",
     )
+    gemini_agent_config: Optional[GeminiAgentConfig] = Field(
+        default=None,
+        description="""Config for scraping a Gemini Agent (Vertex AI Agent resource).
+      Used to target a Gemini agent for an evaluation run.""",
+    )
 
 
 class AgentRunConfigDict(TypedDict, total=False):
@@ -2638,6 +2672,10 @@ class AgentRunConfigDict(TypedDict, total=False):
         Contains configuration for a user simulator that
         uses an LLM to generate messages on behalf of the user."""
 
+    gemini_agent_config: Optional[GeminiAgentConfigDict]
+    """Config for scraping a Gemini Agent (Vertex AI Agent resource).
+      Used to target a Gemini agent for an evaluation run."""
+
 
 AgentRunConfigOrDict = Union[AgentRunConfig, AgentRunConfigDict]
 
@@ -3370,6 +3408,38 @@ class ResponseCandidateDict(TypedDict, total=False):
 ResponseCandidateOrDict = Union[ResponseCandidate, ResponseCandidateDict]
 
 
+class InteractionsDataSource(_common.BaseModel):
+    """Source for populating agent data from an Interactions API interaction."""
+
+    gemini_agent_config: Optional[GeminiAgentConfig] = Field(
+        default=None,
+        description="""The Gemini Agent (Vertex AI Agent resource) that produced the
+      interaction.""",
+    )
+    interaction: Optional[str] = Field(
+        default=None,
+        description="""The interaction to evaluate. Required by the backend.
+      Format:
+      `projects/{project}/locations/{location}/interactions/{interaction}`.""",
+    )
+
+
+class InteractionsDataSourceDict(TypedDict, total=False):
+    """Source for populating agent data from an Interactions API interaction."""
+
+    gemini_agent_config: Optional[GeminiAgentConfigDict]
+    """The Gemini Agent (Vertex AI Agent resource) that produced the
+      interaction."""
+
+    interaction: Optional[str]
+    """The interaction to evaluate. Required by the backend.
+      Format:
+      `projects/{project}/locations/{location}/interactions/{interaction}`."""
+
+
+InteractionsDataSourceOrDict = Union[InteractionsDataSource, InteractionsDataSourceDict]
+
+
 class EvalCase(_common.BaseModel):
     """A comprehensive representation of a GenAI interaction for evaluation."""
 
@@ -3414,6 +3484,10 @@ class EvalCase(_common.BaseModel):
         default=None,
         description="""This field is experimental and may change in future versions. The user scenario for the evaluation case.""",
     )
+    interactions_data_source: Optional[InteractionsDataSource] = Field(
+        default=None,
+        description="""This field is experimental and may change in future versions. Source for populating agent data from an Interactions API interaction. When set, the backend fetches the interaction (and the Gemini Agent config) and parses it into agent data for evaluation; agent_data must not also be set.""",
+    )
     # Allow extra fields to support custom metric prompts and stay backward compatible.
     model_config = ConfigDict(frozen=True, extra="allow")
 
@@ -3454,6 +3528,9 @@ class EvalCaseDict(TypedDict, total=False):
     user_scenario: Optional[evals_types.UserScenario]
     """This field is experimental and may change in future versions. The user scenario for the evaluation case."""
 
+    interactions_data_source: Optional[InteractionsDataSourceDict]
+    """This field is experimental and may change in future versions. Source for populating agent data from an Interactions API interaction. When set, the backend fetches the interaction (and the Gemini Agent config) and parses it into agent data for evaluation; agent_data must not also be set."""
+
 
 EvalCaseOrDict = Union[EvalCase, EvalCaseDict]
 
@@ -4593,6 +4670,13 @@ class EvaluationInstance(_common.BaseModel):
         default=None,
         description="""Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group.""",
     )
+    interactions_data_source: Optional[InteractionsDataSource] = Field(
+        default=None,
+        description="""Source for populating agent data from an Interactions API
+      interaction. If set, no other agent data source may be set. The backend
+      fetches the interaction (and the agent that produced it) and parses it
+      into agent data for grading.""",
+    )
 
 
 class EvaluationInstanceDict(TypedDict, total=False):
@@ -4616,6 +4700,12 @@ class EvaluationInstanceDict(TypedDict, total=False):
     rubric_groups: Optional[dict[str, "RubricGroupDict"]]
     """Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group."""
 
+    interactions_data_source: Optional[InteractionsDataSourceDict]
+    """Source for populating agent data from an Interactions API
+      interaction. If set, no other agent data source may be set. The backend
+      fetches the interaction (and the agent that produced it) and parses it
+      into agent data for grading."""
+
 
 EvaluationInstanceOrDict = Union[EvaluationInstance, EvaluationInstanceDict]