|
| 1 | +# Copyright 2025 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | +# |
| 15 | +# pylint: disable=protected-access,bad-continuation,missing-function-docstring |
| 16 | + |
| 17 | +from tests.unit.vertexai.genai.replays import pytest_helper |
| 18 | +from vertexai._genai import types |
| 19 | +from google.genai import types as genai_types |
| 20 | + |
| 21 | + |
| 22 | +def test_inference_with_eval_cases_multi_turn_agent_data(client): |
| 23 | + """Tests run_inference with multi-turn agent_data in eval_cases. |
| 24 | +
|
| 25 | + Verifies that run_inference() accepts an EvaluationDataset with |
| 26 | + eval_cases containing agent_data (no eval_dataset_df). The agent_data |
| 27 | + has 2 turns: turn 0 is a completed user+agent exchange (history), |
| 28 | + turn 1 is a new user query. The agent should see the history and |
| 29 | + respond to the final query in context. |
| 30 | + """ |
| 31 | + from google.adk.agents import LlmAgent |
| 32 | + |
| 33 | + agent = LlmAgent( |
| 34 | + name="test_agent", |
| 35 | + model="gemini-2.5-flash", |
| 36 | + instruction="You are a helpful assistant. Answer questions concisely.", |
| 37 | + ) |
| 38 | + |
| 39 | + eval_case = types.EvalCase( |
| 40 | + agent_data=types.evals.AgentData( |
| 41 | + turns=[ |
| 42 | + types.evals.ConversationTurn( |
| 43 | + turn_index=0, |
| 44 | + events=[ |
| 45 | + types.evals.AgentEvent( |
| 46 | + author="user", |
| 47 | + content=genai_types.Content( |
| 48 | + role="user", |
| 49 | + parts=[genai_types.Part(text="My name is Alice.")], |
| 50 | + ), |
| 51 | + ), |
| 52 | + types.evals.AgentEvent( |
| 53 | + author="test_agent", |
| 54 | + content=genai_types.Content( |
| 55 | + role="model", |
| 56 | + parts=[ |
| 57 | + genai_types.Part( |
| 58 | + text="Hello Alice! How can I help you?" |
| 59 | + ) |
| 60 | + ], |
| 61 | + ), |
| 62 | + ), |
| 63 | + ], |
| 64 | + ), |
| 65 | + types.evals.ConversationTurn( |
| 66 | + turn_index=1, |
| 67 | + events=[ |
| 68 | + types.evals.AgentEvent( |
| 69 | + author="user", |
| 70 | + content=genai_types.Content( |
| 71 | + role="user", |
| 72 | + parts=[genai_types.Part(text="What is my name?")], |
| 73 | + ), |
| 74 | + ), |
| 75 | + ], |
| 76 | + ), |
| 77 | + ], |
| 78 | + ), |
| 79 | + ) |
| 80 | + eval_dataset = types.EvaluationDataset(eval_cases=[eval_case]) |
| 81 | + |
| 82 | + inference_result = client.evals.run_inference( |
| 83 | + agent=agent, |
| 84 | + src=eval_dataset, |
| 85 | + ) |
| 86 | + assert isinstance(inference_result, types.EvaluationDataset) |
| 87 | + assert inference_result.eval_dataset_df is not None |
| 88 | + assert "agent_data" in inference_result.eval_dataset_df.columns |
| 89 | + |
| 90 | + |
| 91 | +def test_inference_with_eval_cases_agent_engine_agent_data(client): |
| 92 | + """Tests N+1 inference with agent_data via remote Agent Engine.""" |
| 93 | + agent_engine = client.agent_engines.get( |
| 94 | + name="projects/977012026409/locations/us-central1" |
| 95 | + "/reasoningEngines/7188347537655332864" |
| 96 | + ) |
| 97 | + |
| 98 | + eval_case = types.EvalCase( |
| 99 | + agent_data=types.evals.AgentData( |
| 100 | + turns=[ |
| 101 | + types.evals.ConversationTurn( |
| 102 | + turn_index=0, |
| 103 | + events=[ |
| 104 | + types.evals.AgentEvent( |
| 105 | + author="user", |
| 106 | + content=genai_types.Content( |
| 107 | + role="user", |
| 108 | + parts=[genai_types.Part(text="My name is Bob.")], |
| 109 | + ), |
| 110 | + ), |
| 111 | + types.evals.AgentEvent( |
| 112 | + author="model", |
| 113 | + content=genai_types.Content( |
| 114 | + role="model", |
| 115 | + parts=[ |
| 116 | + genai_types.Part(text="Hi Bob! Nice to meet you.") |
| 117 | + ], |
| 118 | + ), |
| 119 | + ), |
| 120 | + ], |
| 121 | + ), |
| 122 | + types.evals.ConversationTurn( |
| 123 | + turn_index=1, |
| 124 | + events=[ |
| 125 | + types.evals.AgentEvent( |
| 126 | + author="user", |
| 127 | + content=genai_types.Content( |
| 128 | + role="user", |
| 129 | + parts=[genai_types.Part(text="What is my name?")], |
| 130 | + ), |
| 131 | + ), |
| 132 | + ], |
| 133 | + ), |
| 134 | + ], |
| 135 | + ), |
| 136 | + ) |
| 137 | + eval_dataset = types.EvaluationDataset(eval_cases=[eval_case]) |
| 138 | + |
| 139 | + inference_result = client.evals.run_inference( |
| 140 | + agent=agent_engine, |
| 141 | + src=eval_dataset, |
| 142 | + ) |
| 143 | + assert isinstance(inference_result, types.EvaluationDataset) |
| 144 | + assert inference_result.eval_dataset_df is not None |
| 145 | + assert "agent_data" in inference_result.eval_dataset_df.columns |
| 146 | + |
| 147 | + |
| 148 | +pytestmark = pytest_helper.setup( |
| 149 | + file=__file__, |
| 150 | + globals_for_file=globals(), |
| 151 | + test_method="evals.run_inference", |
| 152 | +) |
0 commit comments