Skip to content

Commit 9943901

Browse files
andrewheardandrewheard
authored
[AI] Reduce number of permutations in integration tests (#16170)
1 parent ddf0433 commit 9943901

9 files changed

Lines changed: 70 additions & 104 deletions

‎FirebaseAI/Tests/TestApp/Sources/Constants.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public enum ModelNames {
2727
public static let gemini2_5_FlashLive = "gemini-live-2.5-flash-native-audio"
2828
public static let gemini2_5_FlashLivePreview = "gemini-2.5-flash-native-audio-preview-12-2025"
2929
public static let gemini2_5_Pro = "gemini-2.5-pro"
30-
public static let gemini3_1_FlashLitePreview = "gemini-3.1-flash-lite-preview"
30+
public static let gemini3_1_FlashLite = "gemini-3.1-flash-lite"
3131
public static let gemini3_1_FlashImagePreview = "gemini-3.1-flash-image-preview"
3232
public static let gemma4_31B = "gemma-4-31b-it"
3333
}

‎FirebaseAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ struct CountTokensIntegrationTests {
4545
parts: "You are a friendly and helpful assistant."
4646
)
4747

48-
@Test(arguments: InstanceConfig.allConfigs)
48+
@Test(arguments: InstanceConfig.defaultConfigs)
4949
func countTokens_text(_ config: InstanceConfig) async throws {
5050
let prompt = "Why is the sky blue?"
5151
let model = FirebaseAI.componentInstance(config).generativeModel(
52-
modelName: ModelNames.gemini2_5_Flash,
52+
modelName: ModelNames.gemini3_1_FlashLite,
5353
generationConfig: generationConfig,
5454
safetySettings: safetySettings
5555
)
@@ -63,10 +63,10 @@ struct CountTokensIntegrationTests {
6363
#expect(promptTokensDetails.tokenCount == response.totalTokens)
6464
}
6565

66-
@Test(arguments: InstanceConfig.allConfigs)
66+
@Test(arguments: InstanceConfig.defaultConfigs)
6767
func countTokens_text_systemInstruction(_ config: InstanceConfig) async throws {
6868
let model = FirebaseAI.componentInstance(config).generativeModel(
69-
modelName: ModelNames.gemini2_5_Flash,
69+
modelName: ModelNames.gemini3_1_FlashLite,
7070
generationConfig: generationConfig,
7171
safetySettings: safetySettings,
7272
systemInstruction: systemInstruction
@@ -81,10 +81,10 @@ struct CountTokensIntegrationTests {
8181
#expect(promptTokensDetails.tokenCount == response.totalTokens)
8282
}
8383

84-
@Test(arguments: InstanceConfig.allConfigs)
84+
@Test(arguments: InstanceConfig.defaultConfigs)
8585
func countTokens_jsonSchema(_ config: InstanceConfig) async throws {
8686
let model = FirebaseAI.componentInstance(config).generativeModel(
87-
modelName: ModelNames.gemini2_5_Flash,
87+
modelName: ModelNames.gemini3_1_FlashLite,
8888
generationConfig: GenerationConfig(
8989
responseMIMEType: "application/json",
9090
responseSchema: Schema.object(properties: [

‎FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift‎

Lines changed: 26 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ struct GenerateContentIntegrationTests {
5151
(InstanceConfig.vertexAI_v1beta, ModelNames.gemini2_5_FlashLite),
5252
(InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2_5_FlashLite),
5353
(InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini2_5_FlashLite),
54-
(InstanceConfig.googleAI_v1beta, ModelNames.gemini3_1_FlashLitePreview),
55-
(InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini3_1_FlashLitePreview),
54+
(InstanceConfig.googleAI_v1beta, ModelNames.gemini3_1_FlashLite),
55+
(InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini3_1_FlashLite),
5656
(InstanceConfig.googleAI_v1beta, ModelNames.gemma4_31B),
5757
(InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma4_31B),
5858
// Note: The following configs are commented out for easy one-off manual testing.
@@ -105,7 +105,7 @@ struct GenerateContentIntegrationTests {
105105

106106
@Test(
107107
"Generate an enum and provide a system instruction",
108-
arguments: InstanceConfig.allConfigs
108+
arguments: InstanceConfig.defaultConfigs
109109
)
110110
func generateContentEnum(_ config: InstanceConfig) async throws {
111111
let model = FirebaseAI.componentInstance(config).generativeModel(
@@ -159,47 +159,18 @@ struct GenerateContentIntegrationTests {
159159
(.vertexAI_v1beta_global, ModelNames.gemini2_5_Pro, ThinkingConfig(
160160
thinkingBudget: 32768, includeThoughts: true
161161
)),
162-
(.googleAI_v1beta, ModelNames.gemini2_5_Flash, ThinkingConfig(thinkingBudget: 0)),
163-
(.googleAI_v1beta, ModelNames.gemini2_5_Flash, ThinkingConfig(thinkingBudget: 24576)),
164-
(.googleAI_v1beta, ModelNames.gemini2_5_Flash, ThinkingConfig(
162+
(.googleAI_v1beta, ModelNames.gemini2_5_FlashLite, ThinkingConfig(thinkingBudget: 0)),
163+
(.googleAI_v1beta, ModelNames.gemini2_5_FlashLite, ThinkingConfig(thinkingBudget: 24576)),
164+
(.googleAI_v1beta, ModelNames.gemini2_5_FlashLite, ThinkingConfig(
165165
thinkingBudget: 24576, includeThoughts: true
166166
)),
167-
(.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig(thinkingBudget: 128)),
168-
(.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig(thinkingBudget: 32768)),
169-
(.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig(
170-
thinkingBudget: 32768, includeThoughts: true
171-
)),
172-
(
173-
.googleAI_v1beta,
174-
ModelNames.gemini3_1_FlashLitePreview,
175-
ThinkingConfig(thinkingLevel: .minimal)
176-
),
177-
(
178-
.googleAI_v1beta,
179-
ModelNames.gemini3_1_FlashLitePreview,
180-
ThinkingConfig(thinkingLevel: .low)
181-
),
182-
(
183-
.googleAI_v1beta,
184-
ModelNames.gemini3_1_FlashLitePreview,
185-
ThinkingConfig(thinkingLevel: .medium)
186-
),
187-
(
188-
.googleAI_v1beta,
189-
ModelNames.gemini3_1_FlashLitePreview,
190-
ThinkingConfig(thinkingLevel: .high)
191-
),
192-
(
193-
.googleAI_v1beta,
194-
ModelNames.gemini3_1_FlashLitePreview,
195-
ThinkingConfig(thinkingBudget: 0)
196-
),
197-
(
198-
.googleAI_v1beta,
199-
ModelNames.gemini3_1_FlashLitePreview,
200-
ThinkingConfig(thinkingBudget: 32768)
201-
),
202-
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLitePreview, ThinkingConfig(
167+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(thinkingLevel: .minimal)),
168+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(thinkingLevel: .low)),
169+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(thinkingLevel: .medium)),
170+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(thinkingLevel: .high)),
171+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(thinkingBudget: 0)),
172+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(thinkingBudget: 32768)),
173+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(
203174
thinkingBudget: 32768, includeThoughts: true
204175
)),
205176
// Note: The following configs are commented out for easy one-off manual testing.
@@ -304,16 +275,12 @@ struct GenerateContentIntegrationTests {
304275
(.vertexAI_v1beta_global, ModelNames.gemini2_5_Pro, ThinkingConfig(
305276
thinkingBudget: -1, includeThoughts: true
306277
)),
307-
(.googleAI_v1beta, ModelNames.gemini2_5_Flash, ThinkingConfig(thinkingBudget: -1)),
308-
(.googleAI_v1beta, ModelNames.gemini2_5_Flash, ThinkingConfig(
278+
(.googleAI_v1beta, ModelNames.gemini2_5_FlashLite, ThinkingConfig(thinkingBudget: -1)),
279+
(.googleAI_v1beta, ModelNames.gemini2_5_FlashLite, ThinkingConfig(
309280
thinkingBudget: -1, includeThoughts: true
310281
)),
311-
(.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig(thinkingBudget: -1)),
312-
(.googleAI_v1beta, ModelNames.gemini2_5_Pro, ThinkingConfig(
313-
thinkingBudget: -1, includeThoughts: true
314-
)),
315-
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLitePreview, ThinkingConfig(thinkingBudget: -1)),
316-
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLitePreview, ThinkingConfig(
282+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(thinkingBudget: -1)),
283+
(.googleAI_v1beta, ModelNames.gemini3_1_FlashLite, ThinkingConfig(
317284
thinkingBudget: -1, includeThoughts: true
318285
)),
319286
] as [(InstanceConfig, String, ThinkingConfig)]
@@ -549,11 +516,11 @@ struct GenerateContentIntegrationTests {
549516

550517
@Test(
551518
"generateContent with Google Search returns grounding metadata",
552-
arguments: InstanceConfig.allConfigs
519+
arguments: InstanceConfig.defaultConfigs
553520
)
554521
func generateContent_withGoogleSearch_succeeds(_ config: InstanceConfig) async throws {
555522
let model = FirebaseAI.componentInstance(config).generativeModel(
556-
modelName: ModelNames.gemini2_5_Flash,
523+
modelName: ModelNames.gemini3_1_FlashLite,
557524
tools: [.googleSearch()]
558525
)
559526
let prompt = "What is the weather in Toronto today?"
@@ -588,11 +555,11 @@ struct GenerateContentIntegrationTests {
588555

589556
@Test(
590557
"generateContent with URL Context",
591-
arguments: InstanceConfig.allConfigs
558+
arguments: InstanceConfig.defaultConfigs
592559
)
593560
func generateContent_withURLContext_succeeds(_ config: InstanceConfig) async throws {
594561
let model = FirebaseAI.componentInstance(config).generativeModel(
595-
modelName: ModelNames.gemini2_5_Flash,
562+
modelName: ModelNames.gemini3_1_FlashLite,
596563
tools: [.urlContext()]
597564
)
598565
let url = "https://developers.googleblog.com/en/introducing-gemma-3-270m/"
@@ -607,10 +574,10 @@ struct GenerateContentIntegrationTests {
607574
#expect(retrievedURL == URL(string: url))
608575
}
609576

610-
@Test(arguments: InstanceConfig.allConfigs)
577+
@Test(arguments: InstanceConfig.defaultConfigs)
611578
func generateContent_codeExecution_succeeds(_ config: InstanceConfig) async throws {
612579
let model = FirebaseAI.componentInstance(config).generativeModel(
613-
modelName: ModelNames.gemini2_5_FlashLite,
580+
modelName: ModelNames.gemini3_1_FlashLite,
614581
generationConfig: generationConfig,
615582
tools: [.codeExecution()]
616583
)
@@ -640,11 +607,8 @@ struct GenerateContentIntegrationTests {
640607

641608
@Test(arguments: [
642609
(InstanceConfig.vertexAI_v1beta, ModelNames.gemini2_5_FlashLite),
643-
(InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini3_1_FlashLitePreview),
644-
(
645-
InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse,
646-
ModelNames.gemini3_1_FlashLitePreview
647-
),
610+
(InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini3_1_FlashLite),
611+
(InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini3_1_FlashLite),
648612
(InstanceConfig.googleAI_v1beta, ModelNames.gemini2_5_FlashLite),
649613
(InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2_5_FlashLite),
650614
(InstanceConfig.googleAI_v1beta, ModelNames.gemma4_31B),
@@ -778,7 +742,7 @@ struct GenerateContentIntegrationTests {
778742
@Test(arguments: InstanceConfig.appCheckNotConfiguredConfigs)
779743
func generateContent_appCheckNotConfigured_shouldFail(_ config: InstanceConfig) async throws {
780744
let model = FirebaseAI.componentInstance(config).generativeModel(
781-
modelName: ModelNames.gemini2_5_Flash
745+
modelName: ModelNames.gemini3_1_FlashLite
782746
)
783747
let prompt = "Where is Google headquarters located? Answer with the city name only."
784748

‎FirebaseAI/Tests/TestApp/Tests/Integration/GenerativeModelSessionTests.swift‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@
278278
func respondTextWithAutomaticFunctionCalling(_ config: InstanceConfig) async throws {
279279
let temperatureTool = GetTemperature()
280280
let session = FirebaseAI.componentInstance(config).generativeModelSession(
281-
model: ModelNames.gemini3_1_FlashLitePreview,
281+
model: ModelNames.gemini3_1_FlashLite,
282282
tools: [temperatureTool],
283283
instructions: """
284284
You are a weather bot that specializes in reporting outdoor temperatures in Celsius.
@@ -312,7 +312,7 @@
312312
func respondGenerableWithAutomaticFunctionCalling(_ config: InstanceConfig) async throws {
313313
let temperatureTool = GetTemperature()
314314
let session = FirebaseAI.componentInstance(config).generativeModelSession(
315-
model: ModelNames.gemini3_1_FlashLitePreview,
315+
model: ModelNames.gemini3_1_FlashLite,
316316
tools: [temperatureTool],
317317
instructions: """
318318
You are a weather bot that specializes in reporting outdoor temperatures in Celsius.
@@ -341,7 +341,7 @@
341341
@Test(arguments: [InstanceConfig.vertexAI_v1beta_global, InstanceConfig.googleAI_v1beta])
342342
func respondTextWithURLContext(_ config: InstanceConfig) async throws {
343343
let session = FirebaseAI.componentInstance(config).generativeModelSession(
344-
model: ModelNames.gemini2_5_Flash,
344+
model: ModelNames.gemini3_1_FlashLite,
345345
tools: [.urlContext()]
346346
)
347347
let url = "https://blog.google/innovation-and-ai/technology/developers-tools/functiongemma/"
@@ -463,7 +463,7 @@
463463
@available(watchOS, unavailable)
464464
func streamResponseGenerable(_ config: InstanceConfig) async throws {
465465
let firebaseAI = FirebaseAI.componentInstance(config)
466-
let session = firebaseAI.generativeModelSession(model: ModelNames.gemini2_5_FlashLite)
466+
let session = firebaseAI.generativeModelSession(model: ModelNames.gemini3_1_FlashLite)
467467
let prompt = "Generate a Ragdoll kitten"
468468
let config = GenerationConfig(
469469
thinkingConfig: ThinkingConfig(thinkingBudget: -1, includeThoughts: true)
@@ -538,7 +538,7 @@
538538
func streamResponseTextWithAutomaticFunctionCalling(_ config: InstanceConfig) async throws {
539539
let temperatureTool = GetTemperature()
540540
let session = FirebaseAI.componentInstance(config).generativeModelSession(
541-
model: ModelNames.gemini3_1_FlashLitePreview,
541+
model: ModelNames.gemini3_1_FlashLite,
542542
tools: [temperatureTool],
543543
instructions: """
544544
You are a weather bot that specializes in reporting outdoor temperatures in Celsius.

‎FirebaseAI/Tests/TestApp/Tests/Integration/ImplicitCacheTests.swift‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ struct ImplicitCacheTests {
2828

2929
@Test(arguments: [
3030
(InstanceConfig.vertexAI_v1beta, ModelNames.gemini2_5_Flash),
31-
(InstanceConfig.googleAI_v1beta, ModelNames.gemini2_5_Flash),
32-
(InstanceConfig.googleAI_v1beta, ModelNames.gemini2_5_Pro),
33-
(InstanceConfig.googleAI_v1beta, ModelNames.gemini3_1_FlashLitePreview),
31+
(InstanceConfig.vertexAI_v1beta, ModelNames.gemini2_5_Pro),
32+
(InstanceConfig.googleAI_v1beta, ModelNames.gemini2_5_FlashLite),
33+
(InstanceConfig.googleAI_v1beta, ModelNames.gemini3_1_FlashLite),
3434
])
3535
func implicitCaching(_ config: InstanceConfig, modelName: String) async throws {
3636
let model = FirebaseAI.componentInstance(config).generativeModel(

‎FirebaseAI/Tests/TestApp/Tests/Integration/IntegrationTests.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@ final class IntegrationTests: XCTestCase {
5151

5252
override func setUp() async throws {
5353
userID1 = try await TestHelpers.getUserID()
54-
vertex = FirebaseAI.firebaseAI(backend: .vertexAI())
54+
vertex = FirebaseAI.firebaseAI(backend: .vertexAI(location: "global"))
5555
model = vertex.generativeModel(
56-
modelName: ModelNames.gemini2_5_Flash,
56+
modelName: ModelNames.gemini3_1_FlashLite,
5757
generationConfig: generationConfig,
5858
safetySettings: safetySettings,
5959
tools: [],
@@ -69,7 +69,7 @@ final class IntegrationTests: XCTestCase {
6969
func testCountTokens_text() async throws {
7070
let prompt = "Why is the sky blue?"
7171
model = vertex.generativeModel(
72-
modelName: ModelNames.gemini2_5_Flash,
72+
modelName: ModelNames.gemini3_1_FlashLite,
7373
generationConfig: generationConfig,
7474
safetySettings: [
7575
SafetySetting(harmCategory: .harassment, threshold: .blockLowAndAbove, method: .severity),
@@ -173,7 +173,7 @@ final class IntegrationTests: XCTestCase {
173173
parameters: ["x": .integer(), "y": .integer()]
174174
)
175175
model = vertex.generativeModel(
176-
modelName: ModelNames.gemini2_5_Flash,
176+
modelName: ModelNames.gemini3_1_FlashLite,
177177
tools: [.functionDeclarations([sumDeclaration])],
178178
toolConfig: .init(functionCallingConfig: .any(allowedFunctionNames: ["sum"]))
179179
)

‎FirebaseAI/Tests/TestApp/Tests/Integration/MapsGroundingIntegrationTests.swift‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@ import Testing
2121
struct MapsGroundingIntegrationTests {
2222
@Test(
2323
"generateContent with Google Maps returns grounding metadata",
24-
arguments: InstanceConfig.allConfigs
24+
arguments: InstanceConfig.defaultConfigs
2525
)
2626
func generateContent_withGoogleMaps_succeeds(_ config: InstanceConfig) async throws {
2727
let model = FirebaseAI.componentInstance(config).generativeModel(
28-
modelName: ModelNames.gemini2_5_Flash,
28+
modelName: ModelNames.gemini3_1_FlashLite,
2929
tools: [.googleMaps()]
3030
)
3131
let prompt = "Where is a good place to grab a coffee near Alameda, CA?"
@@ -37,11 +37,11 @@ struct MapsGroundingIntegrationTests {
3737

3838
@Test(
3939
"respondTo with Google Maps returns grounding metadata",
40-
arguments: InstanceConfig.allConfigs
40+
arguments: InstanceConfig.defaultConfigs
4141
)
4242
func generativeModelSession_respondTo_withGoogleMaps_succeeds(_ config: InstanceConfig) async throws {
4343
let session = FirebaseAI.componentInstance(config).generativeModelSession(
44-
model: ModelNames.gemini2_5_Flash,
44+
model: ModelNames.gemini3_1_FlashLite,
4545
tools: [.googleMaps()]
4646
)
4747
let prompt = "Where is a good place to grab a coffee near Alameda, CA?"
@@ -53,11 +53,12 @@ struct MapsGroundingIntegrationTests {
5353

5454
@Test(
5555
"streamResponse with Google Maps returns grounding metadata",
56-
arguments: InstanceConfig.allConfigs
56+
arguments: InstanceConfig.defaultConfigs
5757
)
5858
func generativeModelSession_streamResponse_withGoogleMaps_succeeds(_ config: InstanceConfig) async throws {
5959
let session = FirebaseAI.componentInstance(config).generativeModelSession(
60-
model: ModelNames.gemini2_5_Flash,
60+
// TODO: Replace with gemini3_1_FlashLite after resolving decoding failure.
61+
model: ModelNames.gemini2_5_FlashLite,
6162
tools: [.googleMaps()]
6263
)
6364
let prompt = "Where is a good place to grab a coffee near Alameda, CA?"
@@ -70,7 +71,7 @@ struct MapsGroundingIntegrationTests {
7071

7172
@Test(
7273
"generateContent with Google Maps and RetrievalConfig returns grounding metadata",
73-
arguments: InstanceConfig.allConfigs
74+
arguments: InstanceConfig.defaultConfigs
7475
)
7576
func generateContent_withGoogleMapsAndRetrievalConfig_succeeds(_ config: InstanceConfig) async throws {
7677
let toolConfig = ToolConfig(
@@ -79,7 +80,7 @@ struct MapsGroundingIntegrationTests {
7980
)
8081
)
8182
let model = FirebaseAI.componentInstance(config).generativeModel(
82-
modelName: ModelNames.gemini2_5_Flash,
83+
modelName: ModelNames.gemini3_1_FlashLite,
8384
tools: [.googleMaps()],
8485
toolConfig: toolConfig
8586
)
@@ -92,7 +93,7 @@ struct MapsGroundingIntegrationTests {
9293

9394
@Test(
9495
"generateContent with Google Maps and languageCode returns grounding metadata",
95-
arguments: InstanceConfig.allConfigs
96+
arguments: InstanceConfig.defaultConfigs
9697
)
9798
func generateContent_withGoogleMapsAndLanguageConfig_succeeds(_ config: InstanceConfig) async throws {
9899
let toolConfig = ToolConfig(
@@ -101,7 +102,7 @@ struct MapsGroundingIntegrationTests {
101102
)
102103
)
103104
let model = FirebaseAI.componentInstance(config).generativeModel(
104-
modelName: ModelNames.gemini2_5_Flash,
105+
modelName: ModelNames.gemini3_1_FlashLite,
105106
tools: [.googleMaps()],
106107
toolConfig: toolConfig
107108
)

0 commit comments

Comments
 (0)