Skip to content

Commit 9d0a5a8

Browse files
haranrkcopybara-github
haranrk
authored and
copybara-github
committed
test(samples): add replay tests for the interactions_api sample
Add replay test fixtures covering the documented flows of the contributing/samples/models/interactions_api sample (basic text, Google Search grounding, multi-turn stateful recall, and a custom function tool), so the sample is exercised by tests/unittests/test_samples.py. Also fix the shared sample test harness (cli/agent_test_runner.py) to support the Interactions API: - Exclude the volatile interaction_id (a server-issued token) and turn_complete fields from fixture comparison and from rebuilt fixtures, matching how other non-reproducible fields (timestamps, usage metadata, etc.) are already handled. The Interactions API stamps these onto every model response and the replay MockModel cannot reproduce them. - Drive all turns of a fixture rebuild on a single persistent event loop. The sync Runner.run() uses asyncio.run() per call, which closes the loop the model's cached async api_client is bound to, so subsequent turns fail with "Event loop is closed" and corrupt multi-turn fixtures. Reusing one loop keeps the cached client valid across the conversation. Co-authored-by: Haran Rajkumar <haranrk@google.com> PiperOrigin-RevId: 940691339
1 parent 7329f7d commit 9d0a5a8

6 files changed

Lines changed: 402 additions & 3 deletions

File tree

‎contributing/samples/models/interactions_api/tests/basic_text.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"events": [
3+
{
4+
"author": "user",
5+
"content": {
6+
"parts": [
7+
{
8+
"text": "Hello! What can you help me with?"
9+
}
10+
],
11+
"role": "user"
12+
},
13+
"id": "e-1",
14+
"invocationId": "i-1",
15+
"nodeInfo": {
16+
"path": ""
17+
}
18+
},
19+
{
20+
"author": "interactions_test_agent",
21+
"content": {
22+
"parts": [
23+
{
24+
"text": "Hello! I am \"interactions_test_agent,\" an agent designed for testing the Interactions API integration.\n\nI can help you with two main tasks:\n1. **Searching the web:** If you have questions that require up-to-date information, I can search the web for you.\n2. **Checking the weather:** I can provide the current weather for a specific city.\n\nHow can I assist you today?"
25+
}
26+
],
27+
"role": "model"
28+
},
29+
"finishReason": "STOP",
30+
"id": "e-2",
31+
"invocationId": "i-1",
32+
"nodeInfo": {
33+
"path": "interactions_test_agent@1"
34+
}
35+
}
36+
]
37+
}

‎contributing/samples/models/interactions_api/tests/custom_function_weather.json‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
{
2+
"events": [
3+
{
4+
"author": "user",
5+
"content": {
6+
"parts": [
7+
{
8+
"text": "What's the weather like in Tokyo?"
9+
}
10+
],
11+
"role": "user"
12+
},
13+
"id": "e-1",
14+
"invocationId": "i-1",
15+
"nodeInfo": {
16+
"path": ""
17+
}
18+
},
19+
{
20+
"author": "interactions_test_agent",
21+
"content": {
22+
"parts": [
23+
{
24+
"functionCall": {
25+
"args": {
26+
"city": "Tokyo"
27+
},
28+
"id": "fc-1",
29+
"name": "get_current_weather"
30+
}
31+
}
32+
],
33+
"role": "model"
34+
},
35+
"finishReason": "STOP",
36+
"id": "e-2",
37+
"invocationId": "i-1",
38+
"longRunningToolIds": [],
39+
"nodeInfo": {
40+
"path": "interactions_test_agent@1"
41+
}
42+
},
43+
{
44+
"author": "interactions_test_agent",
45+
"content": {
46+
"parts": [
47+
{
48+
"functionResponse": {
49+
"id": "fc-1",
50+
"name": "get_current_weather",
51+
"response": {
52+
"city": "Tokyo",
53+
"condition": "Partly Cloudy",
54+
"humidity": 60,
55+
"temperature_f": 68
56+
}
57+
}
58+
}
59+
],
60+
"role": "user"
61+
},
62+
"id": "e-3",
63+
"invocationId": "i-1",
64+
"nodeInfo": {
65+
"path": "interactions_test_agent@1"
66+
}
67+
},
68+
{
69+
"author": "interactions_test_agent",
70+
"content": {
71+
"parts": [
72+
{
73+
"text": "I have retrieved the weather information for Tokyo. It is currently 68\u00b0F and partly cloudy with 60% humidity."
74+
}
75+
],
76+
"role": "model"
77+
},
78+
"finishReason": "STOP",
79+
"id": "e-4",
80+
"invocationId": "i-1",
81+
"nodeInfo": {
82+
"path": "interactions_test_agent@1"
83+
}
84+
}
85+
]
86+
}

‎contributing/samples/models/interactions_api/tests/google_search_1984.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"events": [
3+
{
4+
"author": "user",
5+
"content": {
6+
"parts": [
7+
{
8+
"text": "Use google search to find out who wrote the novel '1984'."
9+
}
10+
],
11+
"role": "user"
12+
},
13+
"id": "e-1",
14+
"invocationId": "i-1",
15+
"nodeInfo": {
16+
"path": ""
17+
}
18+
},
19+
{
20+
"author": "interactions_test_agent",
21+
"content": {
22+
"parts": [
23+
{
24+
"text": "search_suggestions='<style>\\n.container {\\n align-items: center;\\n border-radius: 8px;\\n display: flex;\\n font-family: Google Sans, Roboto, sans-serif;\\n font-size: 14px;\\n line-height: 20px;\\n padding: 8px 12px;\\n}\\n.chip {\\n display: inline-block;\\n border: solid 1px;\\n border-radius: 16px;\\n min-width: 14px;\\n padding: 5px 16px;\\n text-align: center;\\n user-select: none;\\n margin: 0 8px;\\n -webkit-tap-highlight-color: transparent;\\n}\\n.carousel {\\n overflow: auto;\\n scrollbar-width: none;\\n white-space: nowrap;\\n margin-right: -12px;\\n}\\n.headline {\\n display: flex;\\n margin-right: 4px;\\n}\\n.gradient-container {\\n position: relative;\\n}\\n.gradient {\\n position: absolute;\\n transform: translate(3px, -9px);\\n height: 36px;\\n width: 9px;\\n}\\n@media (prefers-color-scheme: light) {\\n .container {\\n background-color: #fafafa;\\n box-shadow: 0 0 0 1px #0000000f;\\n }\\n .headline-label {\\n color: #1f1f1f;\\n }\\n .chip {\\n background-color: #ffffff;\\n border-color: #d2d2d2;\\n color: #5e5e5e;\\n text-decoration: none;\\n }\\n .chip:hover {\\n background-color: #f2f2f2;\\n }\\n .chip:focus {\\n background-color: #f2f2f2;\\n }\\n .chip:active {\\n background-color: #d8d8d8;\\n border-color: #b6b6b6;\\n }\\n .logo-dark {\\n display: none;\\n }\\n .gradient {\\n background: linear-gradient(90deg, #fafafa 15%, #fafafa00 100%);\\n }\\n}\\n@media (prefers-color-scheme: dark) {\\n .container {\\n background-color: #1f1f1f;\\n box-shadow: 0 0 0 1px #ffffff26;\\n }\\n .headline-label {\\n color: #fff;\\n }\\n .chip {\\n background-color: #2c2c2c;\\n border-color: #3c4043;\\n color: #fff;\\n text-decoration: none;\\n }\\n .chip:hover {\\n background-color: #353536;\\n }\\n .chip:focus {\\n background-color: #353536;\\n }\\n .chip:active {\\n background-color: #464849;\\n border-color: #53575b;\\n }\\n .logo-light {\\n display: none;\\n }\\n .gradient {\\n background: linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);\\n }\\n}\\n</style>\\n<div class=\"container\">\\n <div class=\"headline\">\\n <svg class=\"logo-light\" width=\"18\" height=\"18\" viewBox=\"9 9 35 35\" fill=\"none\" xmlns=\"http://www.w3.org/2000/svg\">\\n <path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M42.8622 27.0064C42.8622 25.7839 42.7525 24.6084 42.5487 23.4799H26.3109V30.1568H35.5897C35.1821 32.3041 33.9596 34.1222 32.1258 35.3448V39.6864H37.7213C40.9814 36.677 42.8622 32.2571 42.8622 27.0064V27.0064Z\" fill=\"#4285F4\"/>\\n <path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M26.3109 43.8555C30.9659 43.8555 34.8687 42.3195 37.7213 39.6863L32.1258 35.3447C30.5898 36.3792 28.6306 37.0061 26.3109 37.0061C21.8282 37.0061 18.0195 33.9811 16.6559 29.906H10.9194V34.3573C13.7563 39.9841 19.5712 43.8555 26.3109 43.8555V43.8555Z\" fill=\"#34A853\"/>\\n <path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M16.6559 29.8904C16.3111 28.8559 16.1074 27.7588 16.1074 26.6146C16.1074 25.4704 16.3111 24.3733 16.6559 23.3388V18.8875H10.9194C9.74388 21.2072 9.06992 23.8247 9.06992 26.6146C9.06992 29.4045 9.74388 32.022 10.9194 34.3417L15.3864 30.8621L16.6559 29.8904V29.8904Z\" fill=\"#FBBC05\"/>\\n <path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M26.3109 16.2386C28.85 16.2386 31.107 17.1164 32.9095 18.8091L37.8466 13.8719C34.853 11.082 30.9659 9.3736 26.3109 9.3736C19.5712 9.3736 13.7563 13.245 10.9194 18.8875L16.6559 23.3388C18.0195 19.2636 21.8282 16.2386 26.3109 16.2386V16.2386Z\" fill=\"#EA4335\"/>\\n </svg>\\n <svg class=\"logo-dark\" width=\"18\" height=\"18\" viewBox=\"0 0 48 48\" xmlns=\"http://www.w3.org/2000/svg\">\\n <circle cx=\"24\" cy=\"23\" fill=\"#FFF\" r=\"22\"/>\\n <path d=\"M33.76 34.26c2.75-2.56 4.49-6.37 4.49-11.26 0-.89-.08-1.84-.29-3H24.01v5.99h8.03c-.4 2.02-1.5 3.56-3.07 4.56v.75l3.91 2.97h.88z\" fill=\"#4285F4\"/>\\n <path d=\"M15.58 25.77A8.845 8.845 0 0 0 24 31.86c1.92 0 3.62-.46 4.97-1.31l4.79 3.71C31.14 36.7 27.65 38 24 38c-5.93 0-11.01-3.4-13.45-8.36l.17-1.01 4.06-2.85h.8z\" fill=\"#34A853\"/>\\n <path d=\"M15.59 20.21a8.864 8.864 0 0 0 0 5.58l-5.03 3.86c-.98-2-1.53-4.25-1.53-6.64 0-2.39.55-4.64 1.53-6.64l1-.22 3.81 2.98.22 1.08z\" fill=\"#FBBC05\"/>\\n <path d=\"M24 14.14c2.11 0 4.02.75 5.52 1.98l4.36-4.36C31.22 9.43 27.81 8 24 8c-5.93 0-11.01 3.4-13.45 8.36l5.03 3.85A8.86 8.86 0 0 1 24 14.14z\" fill=\"#EA4335\"/>\\n </svg>\\n <div class=\"gradient-container\"><div class=\"gradient\"></div></div>\\n </div>\\n <div class=\"carousel\">\\n<a class=\"chip\" href=\"https://www.google.com/search?q=who+wrote+the+novel+1984&client=app-vertex-grounding&safesearch=active\">who wrote the novel 1984</a>\\n </div>\\n</div>'"
25+
},
26+
{
27+
"text": "The novel *1984* was written by the English author **George Orwell** (whose real name was Eric Arthur Blair). It was published in 1949."
28+
}
29+
],
30+
"role": "model"
31+
},
32+
"finishReason": "STOP",
33+
"id": "e-2",
34+
"invocationId": "i-1",
35+
"nodeInfo": {
36+
"path": "interactions_test_agent@1"
37+
}
38+
}
39+
]
40+
}

‎contributing/samples/models/interactions_api/tests/google_search_france.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"events": [
3+
{
4+
"author": "user",
5+
"content": {
6+
"parts": [
7+
{
8+
"text": "Search for the capital of France."
9+
}
10+
],
11+
"role": "user"
12+
},
13+
"id": "e-1",
14+
"invocationId": "i-1",
15+
"nodeInfo": {
16+
"path": ""
17+
}
18+
},
19+
{
20+
"author": "interactions_test_agent",
21+
"content": {
22+
"parts": [
23+
{
24+
"text": "search_suggestions='<style>\\n.container {\\n align-items: center;\\n border-radius: 8px;\\n display: flex;\\n font-family: Google Sans, Roboto, sans-serif;\\n font-size: 14px;\\n line-height: 20px;\\n padding: 8px 12px;\\n}\\n.chip {\\n display: inline-block;\\n border: solid 1px;\\n border-radius: 16px;\\n min-width: 14px;\\n padding: 5px 16px;\\n text-align: center;\\n user-select: none;\\n margin: 0 8px;\\n -webkit-tap-highlight-color: transparent;\\n}\\n.carousel {\\n overflow: auto;\\n scrollbar-width: none;\\n white-space: nowrap;\\n margin-right: -12px;\\n}\\n.headline {\\n display: flex;\\n margin-right: 4px;\\n}\\n.gradient-container {\\n position: relative;\\n}\\n.gradient {\\n position: absolute;\\n transform: translate(3px, -9px);\\n height: 36px;\\n width: 9px;\\n}\\n@media (prefers-color-scheme: light) {\\n .container {\\n background-color: #fafafa;\\n box-shadow: 0 0 0 1px #0000000f;\\n }\\n .headline-label {\\n color: #1f1f1f;\\n }\\n .chip {\\n background-color: #ffffff;\\n border-color: #d2d2d2;\\n color: #5e5e5e;\\n text-decoration: none;\\n }\\n .chip:hover {\\n background-color: #f2f2f2;\\n }\\n .chip:focus {\\n background-color: #f2f2f2;\\n }\\n .chip:active {\\n background-color: #d8d8d8;\\n border-color: #b6b6b6;\\n }\\n .logo-dark {\\n display: none;\\n }\\n .gradient {\\n background: linear-gradient(90deg, #fafafa 15%, #fafafa00 100%);\\n }\\n}\\n@media (prefers-color-scheme: dark) {\\n .container {\\n background-color: #1f1f1f;\\n box-shadow: 0 0 0 1px #ffffff26;\\n }\\n .headline-label {\\n color: #fff;\\n }\\n .chip {\\n background-color: #2c2c2c;\\n border-color: #3c4043;\\n color: #fff;\\n text-decoration: none;\\n }\\n .chip:hover {\\n background-color: #353536;\\n }\\n .chip:focus {\\n background-color: #353536;\\n }\\n .chip:active {\\n background-color: #464849;\\n border-color: #53575b;\\n }\\n .logo-light {\\n display: none;\\n }\\n .gradient {\\n background: linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);\\n }\\n}\\n</style>\\n<div class=\"container\">\\n <div class=\"headline\">\\n <svg class=\"logo-light\" width=\"18\" height=\"18\" viewBox=\"9 9 35 35\" fill=\"none\" xmlns=\"http://www.w3.org/2000/svg\">\\n <path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M42.8622 27.0064C42.8622 25.7839 42.7525 24.6084 42.5487 23.4799H26.3109V30.1568H35.5897C35.1821 32.3041 33.9596 34.1222 32.1258 35.3448V39.6864H37.7213C40.9814 36.677 42.8622 32.2571 42.8622 27.0064V27.0064Z\" fill=\"#4285F4\"/>\\n <path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M26.3109 43.8555C30.9659 43.8555 34.8687 42.3195 37.7213 39.6863L32.1258 35.3447C30.5898 36.3792 28.6306 37.0061 26.3109 37.0061C21.8282 37.0061 18.0195 33.9811 16.6559 29.906H10.9194V34.3573C13.7563 39.9841 19.5712 43.8555 26.3109 43.8555V43.8555Z\" fill=\"#34A853\"/>\\n <path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M16.6559 29.8904C16.3111 28.8559 16.1074 27.7588 16.1074 26.6146C16.1074 25.4704 16.3111 24.3733 16.6559 23.3388V18.8875H10.9194C9.74388 21.2072 9.06992 23.8247 9.06992 26.6146C9.06992 29.4045 9.74388 32.022 10.9194 34.3417L15.3864 30.8621L16.6559 29.8904V29.8904Z\" fill=\"#FBBC05\"/>\\n <path fill-rule=\"evenodd\" clip-rule=\"evenodd\" d=\"M26.3109 16.2386C28.85 16.2386 31.107 17.1164 32.9095 18.8091L37.8466 13.8719C34.853 11.082 30.9659 9.3736 26.3109 9.3736C19.5712 9.3736 13.7563 13.245 10.9194 18.8875L16.6559 23.3388C18.0195 19.2636 21.8282 16.2386 26.3109 16.2386V16.2386Z\" fill=\"#EA4335\"/>\\n </svg>\\n <svg class=\"logo-dark\" width=\"18\" height=\"18\" viewBox=\"0 0 48 48\" xmlns=\"http://www.w3.org/2000/svg\">\\n <circle cx=\"24\" cy=\"23\" fill=\"#FFF\" r=\"22\"/>\\n <path d=\"M33.76 34.26c2.75-2.56 4.49-6.37 4.49-11.26 0-.89-.08-1.84-.29-3H24.01v5.99h8.03c-.4 2.02-1.5 3.56-3.07 4.56v.75l3.91 2.97h.88z\" fill=\"#4285F4\"/>\\n <path d=\"M15.58 25.77A8.845 8.845 0 0 0 24 31.86c1.92 0 3.62-.46 4.97-1.31l4.79 3.71C31.14 36.7 27.65 38 24 38c-5.93 0-11.01-3.4-13.45-8.36l.17-1.01 4.06-2.85h.8z\" fill=\"#34A853\"/>\\n <path d=\"M15.59 20.21a8.864 8.864 0 0 0 0 5.58l-5.03 3.86c-.98-2-1.53-4.25-1.53-6.64 0-2.39.55-4.64 1.53-6.64l1-.22 3.81 2.98.22 1.08z\" fill=\"#FBBC05\"/>\\n <path d=\"M24 14.14c2.11 0 4.02.75 5.52 1.98l4.36-4.36C31.22 9.43 27.81 8 24 8c-5.93 0-11.01 3.4-13.45 8.36l5.03 3.85A8.86 8.86 0 0 1 24 14.14z\" fill=\"#EA4335\"/>\\n </svg>\\n <div class=\"gradient-container\"><div class=\"gradient\"></div></div>\\n </div>\\n <div class=\"carousel\">\\n<a class=\"chip\" href=\"https://www.google.com/search?q=capital+of+France&client=app-vertex-grounding&safesearch=active\">capital of France</a>\\n </div>\\n</div>'"
25+
},
26+
{
27+
"text": "I have searched for the capital of France and confirmed that it is Paris."
28+
}
29+
],
30+
"role": "model"
31+
},
32+
"finishReason": "STOP",
33+
"id": "e-2",
34+
"invocationId": "i-1",
35+
"nodeInfo": {
36+
"path": "interactions_test_agent@1"
37+
}
38+
}
39+
]
40+
}

‎contributing/samples/models/interactions_api/tests/multi_turn.json‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
{
2+
"events": [
3+
{
4+
"author": "user",
5+
"content": {
6+
"parts": [
7+
{
8+
"text": "My favorite color is blue. Just acknowledge this, don't use any tools."
9+
}
10+
],
11+
"role": "user"
12+
},
13+
"id": "e-1",
14+
"invocationId": "i-1",
15+
"nodeInfo": {
16+
"path": ""
17+
}
18+
},
19+
{
20+
"author": "interactions_test_agent",
21+
"content": {
22+
"parts": [
23+
{
24+
"text": "I acknowledge that your favorite color is blue."
25+
}
26+
],
27+
"role": "model"
28+
},
29+
"finishReason": "STOP",
30+
"id": "e-2",
31+
"invocationId": "i-1",
32+
"nodeInfo": {
33+
"path": "interactions_test_agent@1"
34+
}
35+
},
36+
{
37+
"author": "user",
38+
"content": {
39+
"parts": [
40+
{
41+
"text": "What's the weather like in London?"
42+
}
43+
],
44+
"role": "user"
45+
},
46+
"id": "e-3",
47+
"invocationId": "i-2",
48+
"nodeInfo": {
49+
"path": ""
50+
}
51+
},
52+
{
53+
"author": "interactions_test_agent",
54+
"content": {
55+
"parts": [
56+
{
57+
"functionCall": {
58+
"args": {
59+
"city": "London"
60+
},
61+
"id": "fc-1",
62+
"name": "get_current_weather"
63+
}
64+
}
65+
],
66+
"role": "model"
67+
},
68+
"finishReason": "STOP",
69+
"id": "e-4",
70+
"invocationId": "i-2",
71+
"longRunningToolIds": [],
72+
"nodeInfo": {
73+
"path": "interactions_test_agent@1"
74+
}
75+
},
76+
{
77+
"author": "interactions_test_agent",
78+
"content": {
79+
"parts": [
80+
{
81+
"functionResponse": {
82+
"id": "fc-1",
83+
"name": "get_current_weather",
84+
"response": {
85+
"city": "London",
86+
"condition": "Cloudy",
87+
"humidity": 78,
88+
"temperature_f": 59
89+
}
90+
}
91+
}
92+
],
93+
"role": "user"
94+
},
95+
"id": "e-5",
96+
"invocationId": "i-2",
97+
"nodeInfo": {
98+
"path": "interactions_test_agent@1"
99+
}
100+
},
101+
{
102+
"author": "interactions_test_agent",
103+
"content": {
104+
"parts": [
105+
{
106+
"text": "I have checked the weather in London for you. It is currently cloudy with a temperature of 59\u00b0F and 78% humidity."
107+
}
108+
],
109+
"role": "model"
110+
},
111+
"finishReason": "STOP",
112+
"id": "e-6",
113+
"invocationId": "i-2",
114+
"nodeInfo": {
115+
"path": "interactions_test_agent@1"
116+
}
117+
},
118+
{
119+
"author": "user",
120+
"content": {
121+
"parts": [
122+
{
123+
"text": "What is my favorite color that I mentioned earlier in our conversation?"
124+
}
125+
],
126+
"role": "user"
127+
},
128+
"id": "e-7",
129+
"invocationId": "i-3",
130+
"nodeInfo": {
131+
"path": ""
132+
}
133+
},
134+
{
135+
"author": "interactions_test_agent",
136+
"content": {
137+
"parts": [
138+
{
139+
"text": "Your favorite color, which you mentioned earlier, is blue."
140+
}
141+
],
142+
"role": "model"
143+
},
144+
"finishReason": "STOP",
145+
"id": "e-8",
146+
"invocationId": "i-3",
147+
"nodeInfo": {
148+
"path": "interactions_test_agent@1"
149+
}
150+
}
151+
]
152+
}

0 commit comments

Comments
 (0)