From 094ed0aae91561beeb73e8693d2837f5f9fd1c94 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 15:48:48 -0400 Subject: [PATCH 01/10] [AI] Exclude Live API integration tests --- scripts/build.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 3b211a452d2..33d45dfee7f 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -567,14 +567,7 @@ case "$product-$platform-$method" in ;; FirebaseAIIntegration-*-*) - # Build - RunXcodebuild \ - -project 'FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj' \ - -scheme "FirebaseAITestApp-SPM" \ - "${xcb_flags[@]}" \ - build-for-testing - - # Run tests + # Run tests excluding LiveSessionTests RunXcodebuild \ -project 'FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj' \ -scheme "FirebaseAITestApp-SPM" \ @@ -582,7 +575,8 @@ case "$product-$platform-$method" in -parallel-testing-enabled NO \ -retry-tests-on-failure \ -test-iterations 3 \ - test-without-building + -skip-testing:IntegrationTests-SPM/LiveSessionTests \ + test ;; Sessions-*-integration) From d813c3cf73993863d7b111e7a729404a00c8e5ed Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 16:47:11 -0400 Subject: [PATCH 02/10] Add `NSUnbufferedIO=YES` to `xcbeautify` invocation --- scripts/build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index 33d45dfee7f..c966ae1cb22 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -123,7 +123,12 @@ function RunXcodebuild() { local xcbeautify_cmd if command -v xcbeautify &> /dev/null; then - xcbeautify_cmd=(xcbeautify --renderer github-actions --disable-logging) + xcbeautify_cmd=( + env NSUnbufferedIO=YES + xcbeautify + --renderer github-actions + --disable-logging + ) else echo "xcbeautify not found, using raw xcodebuild output." xcbeautify_cmd=(cat) From 9a88a89a9615de32a28dd6ce3d5e9a27f97acc80 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 17:13:44 -0400 Subject: [PATCH 03/10] Skip `scripts/repo.sh` so I can better understand what's going on --- .github/workflows/sdk.ai.yml | 17 +++++++++++++++-- scripts/build.sh | 10 +++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sdk.ai.yml b/.github/workflows/sdk.ai.yml index 1d691ed20b7..b6c39cfeed2 100644 --- a/.github/workflows/sdk.ai.yml +++ b/.github/workflows/sdk.ai.yml @@ -43,10 +43,12 @@ jobs: - os: macos-15 target: iOS xcode: Xcode_26.2 + action: build - os: macos-26 target: iOS # TODO: Bump to 26.4 when "Unable to find a destination matching the provided destination specifier" is resolved. xcode: Xcode_26.2 + action: test runs-on: ${{ matrix.os }} needs: spm env: @@ -60,8 +62,19 @@ jobs: with: path: .build key: ${{ needs.spm.outputs.cache_key }} - - name: Run integration tests - run: scripts/repo.sh tests run --secrets ./scripts/secrets/AI.json --platforms ${{ matrix.target }} --xcode ${{ matrix.xcode }} AI + - name: Install Secret GoogleService-Info.plist + run: scripts/decrypt_gha_secret.sh scripts/gha-encrypted/FirebaseAI/TestApp-GoogleService-Info.plist.gpg \ + FirebaseAI/Tests/TestApp/Resources/GoogleService-Info.plist "$secrets_passphrase" + - name: Install Secret GoogleService-Info-Spark.plist + run: scripts/decrypt_gha_secret.sh scripts/gha-encrypted/FirebaseAI/TestApp-GoogleService-Info-Spark.plist.gpg \ + FirebaseAI/Tests/TestApp/Resources/GoogleService-Info-Spark.plist "$secrets_passphrase" + - name: Install Secret Credentials.swift + run: scripts/decrypt_gha_secret.sh scripts/gha-encrypted/FirebaseAI/TestApp-Credentials.swift.gpg \ + FirebaseAI/Tests/TestApp/Tests/Integration/Credentials.swift "$secrets_passphrase" + - name: Xcode + run: sudo xcode-select -s /Applications/${{ matrix.xcode }}.app/Contents/Developer + - name: Run IntegrationTests + run: scripts/build.sh FirebaseAIIntegration ${{ matrix.target }} ${{ matrix.action }} - name: Upload xcodebuild logs if: failure() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 diff --git a/scripts/build.sh b/scripts/build.sh index c966ae1cb22..c119f40e703 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -571,7 +571,15 @@ case "$product-$platform-$method" in build ;; - FirebaseAIIntegration-*-*) + FirebaseAIIntegration-*-build) + RunXcodebuild \ + -project 'FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj' \ + -scheme "FirebaseAITestApp-SPM" \ + "${xcb_flags[@]}" \ + build + ;; + + FirebaseAIIntegration-*-test) # Run tests excluding LiveSessionTests RunXcodebuild \ -project 'FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj' \ From 036a01fadd8c323776a241cdb9933ac60370da21 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 17:16:17 -0400 Subject: [PATCH 04/10] Switch from `build` to `build-for-testing` to verify test target builds --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index c119f40e703..0f58523f181 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -576,7 +576,7 @@ case "$product-$platform-$method" in -project 'FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj' \ -scheme "FirebaseAITestApp-SPM" \ "${xcb_flags[@]}" \ - build + build-for-testing ;; FirebaseAIIntegration-*-test) From c5aee3dbc329904bdf4b0d72d8238dec3bd8d67a Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 17:44:34 -0400 Subject: [PATCH 05/10] Temporarily stop waiting for SPM before starting integration tests --- .github/workflows/sdk.ai.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdk.ai.yml b/.github/workflows/sdk.ai.yml index b6c39cfeed2..d29531d8d82 100644 --- a/.github/workflows/sdk.ai.yml +++ b/.github/workflows/sdk.ai.yml @@ -50,7 +50,8 @@ jobs: xcode: Xcode_26.2 action: test runs-on: ${{ matrix.os }} - needs: spm + # // Temporarily stop waiting for SPM before starting integration tests. + # needs: spm env: TEST_RUNNER_FIRAAppCheckDebugToken: ${{ secrets.VERTEXAI_INTEGRATION_FAC_DEBUG_TOKEN }} TEST_RUNNER_VTXIntegrationImagen: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} From 4dcbc2b96b18eef7b28038e9dc8339e7a94dd55f Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 17:47:33 -0400 Subject: [PATCH 06/10] Run Live API tests nightly only --- .github/workflows/sdk.ai.yml | 3 ++- .../Tests/Integration/ImagenIntegrationTests.swift | 4 ++-- .../TestApp/Tests/Integration/LiveSessionTests.swift | 8 +++++++- scripts/build.sh | 1 - 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sdk.ai.yml b/.github/workflows/sdk.ai.yml index d29531d8d82..c363623c31d 100644 --- a/.github/workflows/sdk.ai.yml +++ b/.github/workflows/sdk.ai.yml @@ -54,7 +54,8 @@ jobs: # needs: spm env: TEST_RUNNER_FIRAAppCheckDebugToken: ${{ secrets.VERTEXAI_INTEGRATION_FAC_DEBUG_TOKEN }} - TEST_RUNNER_VTXIntegrationImagen: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + TEST_RUNNER_FALIntegrationImagen: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + TEST_RUNNER_FALIntegrationLiveSession: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1 secrets_passphrase: ${{ secrets.GHASecretsGPGPassphrase1 }} steps: diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/ImagenIntegrationTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/ImagenIntegrationTests.swift index c8a288cc20c..133d0310c0b 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/ImagenIntegrationTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/ImagenIntegrationTests.swift @@ -25,8 +25,8 @@ import Testing @Suite( .enabled( - if: ProcessInfo.processInfo.environment["VTXIntegrationImagen"] != nil, - "Only runs if the environment variable VTXIntegrationImagen is set." + if: ProcessInfo.processInfo.environment["FALIntegrationImagen"] == "true", + "Only runs if the environment variable FALIntegrationImagen is set to true." ), .serialized ) diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift index d213fa51cb0..92951cf19af 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/LiveSessionTests.swift @@ -19,7 +19,13 @@ import Testing @testable import struct FirebaseAILogic.APIConfig -@Suite(.serialized) +@Suite( + .enabled( + if: ProcessInfo.processInfo.environment["FALIntegrationLiveSession"] == "true", + "Only runs if the environment variable FALIntegrationLiveSession is set to true." + ), + .serialized +) struct LiveSessionTests { private static let arguments = InstanceConfig.liveConfigs.flatMap { config in switch config.apiConfig.service { diff --git a/scripts/build.sh b/scripts/build.sh index 0f58523f181..bfbb077fdca 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -588,7 +588,6 @@ case "$product-$platform-$method" in -parallel-testing-enabled NO \ -retry-tests-on-failure \ -test-iterations 3 \ - -skip-testing:IntegrationTests-SPM/LiveSessionTests \ test ;; From 60f915494640d8dc60c94e265a4db5f7984884f9 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 17:49:44 -0400 Subject: [PATCH 07/10] Skip cache --- .github/workflows/sdk.ai.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sdk.ai.yml b/.github/workflows/sdk.ai.yml index c363623c31d..056a80bb133 100644 --- a/.github/workflows/sdk.ai.yml +++ b/.github/workflows/sdk.ai.yml @@ -60,10 +60,10 @@ jobs: secrets_passphrase: ${{ secrets.GHASecretsGPGPassphrase1 }} steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: actions/cache/restore@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 - with: - path: .build - key: ${{ needs.spm.outputs.cache_key }} + # - uses: actions/cache/restore@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + # with: + # path: .build + # key: ${{ needs.spm.outputs.cache_key }} - name: Install Secret GoogleService-Info.plist run: scripts/decrypt_gha_secret.sh scripts/gha-encrypted/FirebaseAI/TestApp-GoogleService-Info.plist.gpg \ FirebaseAI/Tests/TestApp/Resources/GoogleService-Info.plist "$secrets_passphrase" From 40876c5072934eadb6626eb120f4153256608669 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 18:09:58 -0400 Subject: [PATCH 08/10] Revert "Skip cache" This reverts commit 60f915494640d8dc60c94e265a4db5f7984884f9. --- .github/workflows/sdk.ai.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sdk.ai.yml b/.github/workflows/sdk.ai.yml index 056a80bb133..c363623c31d 100644 --- a/.github/workflows/sdk.ai.yml +++ b/.github/workflows/sdk.ai.yml @@ -60,10 +60,10 @@ jobs: secrets_passphrase: ${{ secrets.GHASecretsGPGPassphrase1 }} steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - # - uses: actions/cache/restore@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 - # with: - # path: .build - # key: ${{ needs.spm.outputs.cache_key }} + - uses: actions/cache/restore@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 + with: + path: .build + key: ${{ needs.spm.outputs.cache_key }} - name: Install Secret GoogleService-Info.plist run: scripts/decrypt_gha_secret.sh scripts/gha-encrypted/FirebaseAI/TestApp-GoogleService-Info.plist.gpg \ FirebaseAI/Tests/TestApp/Resources/GoogleService-Info.plist "$secrets_passphrase" From ae71184684262f65f15123eb5abe7e6a7f9cdd00 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 18:10:19 -0400 Subject: [PATCH 09/10] Revert "Temporarily stop waiting for SPM before starting integration tests" This reverts commit c5aee3dbc329904bdf4b0d72d8238dec3bd8d67a. --- .github/workflows/sdk.ai.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sdk.ai.yml b/.github/workflows/sdk.ai.yml index c363623c31d..019677a3d52 100644 --- a/.github/workflows/sdk.ai.yml +++ b/.github/workflows/sdk.ai.yml @@ -50,8 +50,7 @@ jobs: xcode: Xcode_26.2 action: test runs-on: ${{ matrix.os }} - # // Temporarily stop waiting for SPM before starting integration tests. - # needs: spm + needs: spm env: TEST_RUNNER_FIRAAppCheckDebugToken: ${{ secrets.VERTEXAI_INTEGRATION_FAC_DEBUG_TOKEN }} TEST_RUNNER_FALIntegrationImagen: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} From 58082c404ca703a2a75197bb338f21ac11e2c5e6 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 28 May 2026 18:18:17 -0400 Subject: [PATCH 10/10] Remove extraneous comment --- scripts/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index bfbb077fdca..d032e25880c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -580,7 +580,6 @@ case "$product-$platform-$method" in ;; FirebaseAIIntegration-*-test) - # Run tests excluding LiveSessionTests RunXcodebuild \ -project 'FirebaseAI/Tests/TestApp/FirebaseAITestApp.xcodeproj' \ -scheme "FirebaseAITestApp-SPM" \