integration_tests.yml

name: Integration tests

on:
  schedule:
    - cron: "0 9 * * *"  # 9am UTC = 1am PST / 2am PDT. for all testapps except firestore
    - cron: "0 10 * * *"  # 10am UTC = 2am PST / 3am PDT. for firestore test against release iOS/Android SDK
    - cron: "0 11 * * *"  # 11am UTC = 3am PST / 4am PDT. for firestore desktop test aginst tip-of-tree iOS repo

  pull_request:
    types: [ labeled, closed ]

  workflow_dispatch:
    inputs:
      platforms:
        description: 'CSV of Desktop, Android, iOS and/or tvOS'
        default: 'Desktop,Android,iOS,tvOS'
        required: true
      apis:
        description: 'CSV of apis to build and test'
        default: 'analytics,app_check,auth,database,dynamic_links,firestore,functions,gma,installations,messaging,remote_config,storage'
        required: true
      operating_systems:
        description: 'CSV of VMs to run on'
        default: 'ubuntu-22.04,windows-latest,macos-13'
        required: true
      desktop_ssl_variants:
        description: 'CSV of desktop SSL variants to use'
        default: 'openssl,boringssl'
        required: true
      mobile_test_on:
        description: 'Run mobile tests on real and/or virtual devices? (separated by commas)'
        default: 'real,virtual'
        required: true
      use_expanded_matrix:
        description: 'Use an expanded matrix? Note: above config will be ignored.'
        default: '0'
        required: true
      test_packaged_sdk:
        description: 'Optional: Packaging run # to build against?'
      test_pull_request:
        description: 'Optional: Pull request # to build and test? (With optional commit hash, separated by a colon. Specify the FULL hash.)'
      firestore_dep_source:
        description: 'Optional: Where to get firestore iOS SDK from: "RELEASED", "TIP" or "<git-commit> from firestore-ios-sdk"'

permissions: write-all

env:
  triggerLabelPrefix: "tests-requested: "
  triggerLabelFull: "tests-requested: full"
  triggerLabelQuick: "tests-requested: quick"
  pythonVersion: '3.8'
  xcodeVersion: '15.1'
  artifactRetentionDays: 2
  GITHUB_TOKEN: ${{ github.token }}
  # All self-hosted ARM Mac runners should have this label. Due to how
  # our custom reporting works, it must be exactly two words separated
  # by a hyphen. The first word must be "macos". The second word will
  # be omitted from the summary log.
  runnerLabelMacArm64: "macos-m1custom"

jobs:
  check_and_prepare:
    runs-on: ubuntu-22.04
    outputs:
      trigger: ${{ steps.set_outputs.outputs.trigger }}
      github_ref: ${{ steps.set_outputs.outputs.github_ref }}
      pr_number: ${{ steps.set_outputs.outputs.pr_number }}
      matrix_platform: ${{ steps.matrix_config.outputs.matrix_platform }}
      matrix_os: ${{ steps.matrix_config.outputs.matrix_os }}
      matrix_arch_macos: ${{ steps.matrix_config.outputs.matrix_arch_macos }}
      matrix_arch_windows_linux: ${{ steps.matrix_config.outputs.matrix_arch_windows_linux }}
      matrix_arch_combined: ${{ steps.matrix_config.outputs.matrix_arch_combined }}
      matrix_ssl: ${{ steps.matrix_config.outputs.matrix_ssl }}
      apis: ${{ steps.matrix_config.outputs.apis }}
      mobile_test_on: ${{ steps.matrix_config.outputs.mobile_test_on }}
      android_device: ${{ steps.matrix_config.outputs.android_device }}
      xcode_version: ${{ steps.matrix_config.outputs.xcode_version }}
      ios_device: ${{ steps.matrix_config.outputs.ios_device }}
      tvos_device: ${{ steps.matrix_config.outputs.tvos_device }}
      # Copy the runner label here because matrix specifiers cannot see env.
      runner_label_macos_arm64: ${{ env.runnerLabelMacArm64 }}
    steps:
    ### Fail the workflow if the user does not have admin access to run the tests.
    - name: Check if user has permission to trigger tests
      uses: lannonbr/repo-permission-check-action@2.0.0
      with:
        permission: "admin"
    ### It sets "github_ref,trigger,pr_number,requested_tests" outputs to control the following jobs and steps
    ### trigger value: manual_trigger, scheduled_trigger, label_trigger, postsubmit_trigger
    - id: set_outputs
      run: |
        if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
          if [[ "${{ github.event.inputs.test_pull_request }}" != "nightly-packaging" ]]; then
            # Triggered manually
            echo "trigger=manual_trigger" >> $GITHUB_OUTPUT
            if [[ "${{ github.event.inputs.use_expanded_matrix }}" == "1" ]]; then
              echo "requested_tests=expanded" >> $GITHUB_OUTPUT
            fi
            if [[ -z "${{github.event.inputs.test_pull_request}}" ]]; then
              # test_pull_request not specified
              echo "github_ref=$GITHUB_SHA" >> $GITHUB_OUTPUT
            elif [[ "${{github.event.inputs.test_pull_request}}" == *:* ]]; then
              # If specified as pr:commit_hash, split them.
              echo "github_ref=$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f2)" >> $GITHUB_OUTPUT
              echo "pr_number=$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f1)" >> $GITHUB_OUTPUT
            else
              # Just the PR specified, use refs/pull/<number>/merge as the ref.
              echo "github_ref=refs/pull/${{github.event.inputs.test_pull_request}}/merge" >> $GITHUB_OUTPUT
              echo "pr_number=${{ github.event.inputs.test_pull_request }}" >> $GITHUB_OUTPUT
            fi
          elif [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
            # Triggered by scheduled packaging SDK workflow.
            echo "trigger=scheduled_trigger" >> $GITHUB_OUTPUT
            echo "github_ref=$GITHUB_SHA" >> $GITHUB_OUTPUT
            echo "requested_tests=expanded" >> $GITHUB_OUTPUT
          fi
        elif [[ "${{ github.event_name }}" == "schedule" ]]; then
          echo "trigger=scheduled_trigger" >> $GITHUB_OUTPUT
          echo "github_ref=$GITHUB_SHA" >> $GITHUB_OUTPUT
          echo "requested_tests=expanded" >> $GITHUB_OUTPUT
        elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
          echo "github_ref=$GITHUB_SHA" >> $GITHUB_OUTPUT
          if [[ "${{ github.event.action }}" == "labeled" && "${{ github.event.label.name }}" == "${{ env.triggerLabelPrefix }}"* ]]; then
            echo "trigger=label_trigger" >> $GITHUB_OUTPUT
            echo "pr_number=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
            if [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelQuick }}" ]]; then
              echo "requested_tests=auto" >> $GITHUB_OUTPUT
            else
              echo "requested_tests=expanded" >> $GITHUB_OUTPUT
            fi
          elif [[ "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged == true}}" == "true" ]]; then
            echo "trigger=postsubmit_trigger" >> $GITHUB_OUTPUT
            echo "pr_number=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
            echo "requested_tests=auto" >> $GITHUB_OUTPUT
          fi
        fi
    ### If it's not a defined trigger, cancel workflow
    ### e.g. Triggered by non-"test-request" label; triggered by not merged PR close event.
    - name: Cancel workflow
      if: ${{ !steps.set_outputs.outputs.trigger }}
      uses: andymckay/cancel-action@0.2
    - name: Wait for workflow cancellation
      if: ${{ !steps.set_outputs.outputs.trigger }}
      run: |
        sleep 300
        exit 1  # fail out if the cancellation above somehow failed.
    - name: Cancel previous runs on the same PR
      if: steps.set_outputs.outputs.trigger == 'label_trigger'
      uses: styfle/cancel-workflow-action@0.8.0
      with:
        access_token: ${{ github.token }}
    - uses: actions/checkout@v3
      with:
        ref: ${{steps.set_outputs.outputs.github_ref}}
        fetch-depth: 0
        submodules: false
    - name: Setup python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.pythonVersion }}
    - name: Install python deps
      uses: nick-invision/retry@v2
      with:
        timeout_minutes: 1
        max_attempts: 3
        shell: bash
        command: pip install -r scripts/gha/python_requirements.txt
    - id: matrix_config
      env:
        HEAD_REF: ${{github.event.pull_request.head.ref}}
        BASE_REF: ${{github.event.pull_request.base.ref}}
      run: |
        if [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "expanded" ]]; then
          TEST_MATRIX_PARAM=-e=1
          echo "::warning ::Running on the expanded matrix"
        elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "minimal" ]]; then
          TEST_MATRIX_PARAM=-m=1
          echo "::warning ::Running on the minimal matrix"
        elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "auto" ]]; then
          # auto-diff only apply when running in a PR.
          # diff against the PR's base. "git merge-base main branch_name" will give the common ancestor of both branches.
          MERGE_BASE=$(git merge-base "origin/${HEAD_REF}" "origin/${BASE_REF}" || true)
          # If origin/<branch> is no longer valid, then just run all tests.
          if [[ -n "${MERGE_BASE}" ]]; then
            echo "::warning ::Auto-diff origin/${HEAD_REF}..${MERGE_BASE}"
            git diff --name-only "origin/${HEAD_REF}..${MERGE_BASE}"
            TEST_MATRIX_PARAM="--auto_diff origin/${HEAD_REF}..${MERGE_BASE}"
          fi
        fi

        # To feed input into the job matrix, we first need to convert to a JSON
        # list. Then we can use fromJson to define the field in the matrix for the tests job.
        if [[ "${{ github.event.schedule }}" == "0 9 * * *" ]]; then
          # at 1am PST/2am PDT. Running integration tests and generate test report for all testapps except firestore
          apis="analytics,app_check,auth,database,dynamic_links,functions,gma,installations,messaging,remote_config,storage"
          echo "::warning ::Running main nightly tests"
        elif [[ "${{ github.event.schedule }}" == "0 10 * * *" || "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
          # at 2am PST/3am PDT and 3am PST/4am PDT. Running integration tests for firestore and generate test report.
          echo "::warning ::Running Firestore nightly tests"
          apis="firestore"
        else
          echo "::warning ::Running pull request tests"
          apis=$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k apis -o "${{github.event.inputs.apis}}" ${TEST_MATRIX_PARAM} )
        fi
        if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
          # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
          echo "::warning ::Running against Firestore tip-of-tree"
          matrix_platform="Desktop"
          matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "ubuntu-22.04,macos-13")
        else
          matrix_platform=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )
          matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")
        fi
        echo "apis=${apis}" >> $GITHUB_OUTPUT
        echo "matrix_platform=${matrix_platform}" >> $GITHUB_OUTPUT
        echo "matrix_os=${matrix_os}" >> $GITHUB_OUTPUT
        echo "matrix_arch_macos=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos)" >> $GITHUB_OUTPUT
        echo "matrix_arch_windows_linux=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux)" >> $GITHUB_OUTPUT
        # Combine architecture_macos and architecture_windows_linux to get a list of all architectures for the build matrix.
        matrix_arch_combined=`echo $( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos | sed 's/[]\[,]//g') \
                                   $( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux | sed 's/[]\[,]//g' ) \
                                   | sed 's/ /\n/g' | sort | uniq`
        matrix_arch_combined_json=["$(echo ${matrix_arch_combined} | sed 's/ /,/g')"]
        echo "matrix_arch_combined=${matrix_arch_combined_json}" >> $GITHUB_OUTPUT
        
        # If building against a packaged SDK, consider it as using boringssl, as the packaged SDK uses boringssl under the hood.
        # This avoids trying to install openssl on the system when compiling against the packaged SDK.
        # As an added bonus, we ensure that the packaged SDK works even without openssl explicitly installed.
        if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then
          echo "::warning ::Downloading SDK package from previous run: https://github.com/${{github.repository}}/actions/runs/${{ github.event.inputs.test_packaged_sdk }}"
          echo "matrix_ssl=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o boringssl )" >> $GITHUB_OUTPUT
        else
          echo "matrix_ssl=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ssl_lib -o "${{github.event.inputs.desktop_ssl_variants}}" )" >> $GITHUB_OUTPUT
        fi
        mobile_test_on=$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k mobile_test_on -o "${{github.event.inputs.mobile_test_on}}")
        echo "mobile_test_on=${mobile_test_on}" >> $GITHUB_OUTPUT
        echo "android_device=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k android_device -t ${mobile_test_on} )" >> $GITHUB_OUTPUT
        echo "xcode_version=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k xcode_version)" >> $GITHUB_OUTPUT
        echo "ios_device=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ios_device -t ${mobile_test_on} )" >> $GITHUB_OUTPUT
        echo "tvos_device=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k tvos_device -t ${mobile_test_on} )" >> $GITHUB_OUTPUT
    - name: Update PR label and comment
      if: steps.set_outputs.outputs.pr_number
      shell: bash
      run: |
        #Add the in-progress label and remove any previous labels.
        python scripts/gha/it_workflow.py --stage start \
          --token ${{github.token}} \
          --issue_number ${{steps.set_outputs.outputs.pr_number}} \
          --actor ${{github.actor}} \
          --commit ${{steps.set_outputs.outputs.github_ref}} \
          --run_id ${{github.run_id}}

  build_desktop:
    name: build-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
    needs: [check_and_prepare]
    runs-on: ${{ matrix.os }}
    # Skip this if there is an empty matrix (which can happen if "auto" was set above).
    # But check cancelled() && !failure() so it runs even if check_trigger was skipped.
    if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && needs.check_and_prepare.outputs.apis != '' && !cancelled() && !failure()
    strategy:
      fail-fast: false
      matrix:
        os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }}
        arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }}
        ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }}
        # Because matrix_arch_combined combines Mac, Linux, and Windows, we need
        # a few exclusions. These are standard exclusions used in several
        # places.
        exclude:
        # Do not attempt to use arm64 on Windows or Linux.
        - os: windows-latest
          arch: arm64
        - os: ubuntu-22.04
          arch: arm64
        # Do not attempt to use x86 on Mac.
        - os: macos-13
          arch: x86
        # Until we support building openssl from source, we can't use the
        # system's openssl when cross-compiling, except on Linux. Builds all
        # happen on x64 machines, so arm64 and x86 are technically
        # cross-compiling.
        - os: windows-latest
          ssl_variant: openssl
          arch: x86
        - os: macos-13
          ssl_variant: openssl
          arch: arm64
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
          submodules: true
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: setup default Xcode version (macos)
        if: ${{ runner.os == 'macOS' }}
        run: |
          sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
          # Remove default Xcode version to prevent the wrong SDK from being used.
          rm -rf /Applications/Xcode.app
      - name: Install Desktop SDK & integration tests prerequisites
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 15
          max_attempts: 3
          command: scripts/gha/install_test_workflow_prereqs.sh -p Desktop -a '${{ matrix.arch }}' -s '${{ matrix.ssl_variant }}'
      - name: Add msbuild to PATH (Windows)
        if: startsWith(matrix.os, 'windows')
        uses: microsoft/setup-msbuild@v1.1
      - name: Cache vcpkg C++ dependencies
        id: cache_vcpkg
        uses: actions/cache@v3
        with:
          path: external/vcpkg/installed
          key: dev-vcpkg-${{ env.VCPKG_TRIPLET }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}
      - name: Fetch prebuilt packaged SDK from previous run
        uses: actions/download-artifact@v4
        if: ${{ github.event.inputs.test_packaged_sdk != '' }}
        with:
          name: 'firebase_cpp_sdk.zip'
          github-token: ${{ github.token }}
          run-id: ${{ github.event.inputs.test_packaged_sdk }}
      - name: Build integration tests
        shell: bash
        env:
          CCACHE_DIR: ${{ github.workspace }}/ccache_dir
        run: |
          python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
          declare -a additional_flags
          if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then
            # Building integration tests against a packaged SDK.
            mkdir downloaded_sdk
            unzip -q firebase_cpp_sdk.zip -d downloaded_sdk/
            additional_flags+=(--packaged_sdk downloaded_sdk/firebase_cpp_sdk)
          else
            # Building integration tests against the SDK source.
            #
            # When building the SDK from source, the default SSL is openssl.
            # To build using boringssl, a cmake flag must be added:
            if [[ "${{ matrix.ssl_variant }}" == "boringssl" ]]; then
              additional_flags+=(--cmake_flag=-DFIREBASE_USE_BORINGSSL=ON)
            fi
          fi
          if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
            # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
            additional_flags+=(--cmake_flag=-DFIRESTORE_DEP_SOURCE=TIP)
          elif [[ "${{ github.event.inputs.firestore_dep_source }}" ]]; then
            additional_flags+=(--cmake_flag=-DFIRESTORE_DEP_SOURCE=${{ github.event.inputs.firestore_dep_source }})
          fi
          python scripts/gha/build_testapps.py --p Desktop \
            --t ${{ needs.check_and_prepare.outputs.apis }} \
            --output_directory "${{ github.workspace }}" \
            --artifact_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" \
            --noadd_timestamp \
            --short_output_paths \
            --gha_build \
            --arch ${{ matrix.arch }} \
            ${additional_flags[*]}
      - name: Upload Desktop Cmake
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: cmake-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
          path: D:/a/firebase-cpp-sdk/firebase-cpp-sdk/ta/firestore/iti/CMakeFiles/
          retention-days: 1
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json ]; then
            # No summary was created, make a placeholder one.
            echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json
          fi
      - name: Upload Desktop integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
          path: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload Desktop build results artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: log-artifact-build-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
          path: build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: log-artifact-build-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          pushd ${{env.GCS_UPLOAD_DIR}}
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize build results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat build-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  build_android:
    name: build-android-${{ matrix.os }}
    needs: [check_and_prepare]
    runs-on: ${{ matrix.os }}
    if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Android') && needs.check_and_prepare.outputs.apis != '' && !cancelled() && !failure()
    strategy:
      fail-fast: false
      matrix:
        os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }}
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
          submodules: true
      - name: Force Java 11
        shell: bash
        run: echo "JAVA_HOME=${JAVA_HOME_11_X64}" >> $GITHUB_ENV
      - name: Add msbuild to PATH (Windows)
        if: startsWith(matrix.os, 'windows')
        uses: microsoft/setup-msbuild@v1.1
      - name: Cache NDK
        id: cache_ndk
        uses: actions/cache@v3
        with:
          path: /tmp/android-ndk-r21e
          key: android-ndk-${{ matrix.os }}-r21e
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: Install Android SDK & integration tests prerequisites
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 10
          max_attempts: 3
          command: scripts/gha/install_test_workflow_prereqs.sh -p Android
      - name: Fetch prebuilt packaged SDK from previous run
        uses: actions/download-artifact@v4
        if: ${{ github.event.inputs.test_packaged_sdk != '' }}
        with:
          name: 'firebase_cpp_sdk.zip'
          github-token: ${{ github.token }}
          run-id: ${{ github.event.inputs.test_packaged_sdk }}
      - name: Build integration tests
        shell: bash
        run: |
          python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
          declare -a additional_flags
          if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then
            # Building integration tests against a packaged SDK.
            mkdir downloaded_sdk
            cd downloaded_sdk
            unzip -q ../firebase_cpp_sdk.zip
            cd ..
            additional_flags+=(--packaged_sdk downloaded_sdk/firebase_cpp_sdk)
          fi
          python scripts/gha/build_testapps.py --p Android \
            --t ${{ needs.check_and_prepare.outputs.apis }}  \
            --output_directory "${{ github.workspace }}" \
            --artifact_name "android-${{ matrix.os }}" \
            --noadd_timestamp \
            --short_output_paths \
            --gha_build \
            ${additional_flags[*]}
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          if [ ! -f build-results-android-${{ matrix.os }}.log.json ]; then
            echo "__SUMMARY_MISSING__" > build-results-android-${{ matrix.os }}.log.json
          fi
      - name: Upload Android integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: testapps-android-${{ matrix.os }}
          path: testapps-android-${{ matrix.os }}
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload Android build results artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: log-artifact-build-android-${{ matrix.os }}
          path: build-results-android-${{ matrix.os }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: log-artifact-build-android-${{ matrix.os }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize build results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat build-results-android-${{ matrix.os }}.log
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  build_ios:
    name: build-ios-${{ matrix.os }}
    needs: [check_and_prepare]
    runs-on: ${{ matrix.os }}
    if: contains(needs.check_and_prepare.outputs.matrix_platform, 'iOS') && needs.check_and_prepare.outputs.apis != '' && !cancelled() && !failure()
    strategy:
      fail-fast: false
      matrix:
        os: [macos-13]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
          submodules: true
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: setup default Xcode version (macos)
        if: ${{ runner.os == 'macOS' }}
        run: |
          sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
          # Remove default Xcode version to prevent the wrong SDK from being used.
          rm -rf /Applications/Xcode.app
      - name: Install iOS SDK & integration tests prerequisites
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 3
          max_attempts: 3
          command: scripts/gha/install_test_workflow_prereqs.sh -p iOS
      - name: Fetch prebuilt packaged SDK from previous run
        uses: actions/download-artifact@v4
        if: ${{ github.event.inputs.test_packaged_sdk != '' }}
        with:
          name: 'firebase_cpp_sdk.zip'
          github-token: ${{ github.token }}
          run-id: ${{ github.event.inputs.test_packaged_sdk }}
      - name: Build integration tests
        shell: bash
        run: |
          python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
          declare -a additional_flags
          if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then
            # Building integration tests against a packaged SDK.
            mkdir downloaded_sdk
            cd downloaded_sdk
            unzip -q ../firebase_cpp_sdk.zip
            cd ..
            additional_flags+=(--packaged_sdk downloaded_sdk/firebase_cpp_sdk)
          fi
          python scripts/gha/build_testapps.py --p iOS \
            --t ${{ needs.check_and_prepare.outputs.apis }}  \
            --output_directory "${{ github.workspace }}" \
            --ios_sdk ${{ needs.check_and_prepare.outputs.mobile_test_on }} \
            --artifact_name "ios-${{ matrix.os }}" \
            --noadd_timestamp \
            --short_output_paths \
            --gha_build \
            ${additional_flags[*]}
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          if [ ! -f build-results-ios-${{ matrix.os }}.log.json ]; then
            echo "__SUMMARY_MISSING__" > build-results-ios-${{ matrix.os }}.log.json
          fi
      - name: Upload iOS integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: testapps-ios-${{ matrix.os }}
          path: testapps-ios-${{ matrix.os }}
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload iOS build results artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: log-artifact-build-ios-${{ matrix.os }}
          path: build-results-ios-${{ matrix.os }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: log-artifact-build-ios-${{ matrix.os }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize build results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat build-results-ios-${{ matrix.os }}.log
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  build_tvos:
    name: build-tvos-${{ matrix.os }}
    needs: [check_and_prepare]
    runs-on: ${{ matrix.os }}
    if: contains(needs.check_and_prepare.outputs.matrix_platform, 'tvOS') && needs.check_and_prepare.outputs.apis != '' && !cancelled() && !failure()
    strategy:
      fail-fast: false
      matrix:
        os: [macos-13]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
          submodules: true
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: setup default Xcode version (macos)
        if: ${{ runner.os == 'macOS' }}
        run: |
          sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
          # Remove default Xcode version to prevent the wrong SDK from being used.
          rm -rf /Applications/Xcode.app
      - name: Install tvOS SDK & integration tests prerequisites
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 3
          max_attempts: 3
          command: scripts/gha/install_test_workflow_prereqs.sh -p tvOS
      - name: Fetch prebuilt packaged SDK from previous run
        uses: actions/download-artifact@v4
        if: ${{ github.event.inputs.test_packaged_sdk != '' }}
        with:
          name: 'firebase_cpp_sdk.zip'
          github-token: ${{ github.token }}
          run-id: ${{ github.event.inputs.test_packaged_sdk }}
      - name: Build integration tests
        shell: bash
        run: |
          python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
          declare -a additional_flags
          if [[ -n "${{ github.event.inputs.test_packaged_sdk }}" ]]; then
            # Building integration tests against a packaged SDK.
            mkdir downloaded_sdk
            cd downloaded_sdk
            unzip -q ../firebase_cpp_sdk.zip
            cd ..
            additional_flags+=(--packaged_sdk downloaded_sdk/firebase_cpp_sdk)
          fi
          python scripts/gha/build_testapps.py --p tvOS \
            --t ${{ needs.check_and_prepare.outputs.apis }}  \
            --output_directory "${{ github.workspace }}" \
            --artifact_name "tvos-${{ matrix.os }}" \
            --noadd_timestamp \
            --short_output_paths \
            --gha_build \
            ${additional_flags[*]}
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          if [ ! -f build-results-tvos-${{ matrix.os }}.log.json ]; then
            echo "__SUMMARY_MISSING__" > build-results-tvos-${{ matrix.os }}.log.json
          fi
      - name: Upload tvOS integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: testapps-tvos-${{ matrix.os }}
          path: testapps-tvos-${{ matrix.os }}
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload tvOS build results artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: log-artifact-build-tvos-${{ matrix.os }}
          path: build-results-tvos-${{ matrix.os }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: log-artifact-build-tvos-${{ matrix.os }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize build results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat build-results-tvos-${{ matrix.os }}.log
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  test_desktop:
    name: test-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
    needs: [check_and_prepare, build_desktop]
    runs-on: ${{ matrix.os }}
    if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Desktop') && needs.check_and_prepare.outputs.apis != '' && !cancelled()
    strategy:
      fail-fast: false
      matrix:
        os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }}
        arch: ${{ fromJson(needs.check_and_prepare.outputs.matrix_arch_combined) }}
        ssl_variant: ${{ fromJson(needs.check_and_prepare.outputs.matrix_ssl) }}
        # Because matrix_arch_combined combines Mac, Linux, and Windows, we need
        # a few exclusions. These are standard exclusions used in several
        # places.
        exclude:
        # Do not attempt to use arm64 on Windows or Linux.
        - os: windows-latest
          arch: arm64
        - os: ubuntu-22.04
          arch: arm64
        # Do not attempt to use x86 on Mac.
        - os: macos-13
          arch: x86
        # Until we support building openssl from source, we can't use the
        # system's openssl when cross-compiling, except on Linux. Builds all
        # happen on x64 machines, so arm64 and x86 are technically
        # cross-compiling.
        - os: windows-latest
          ssl_variant: openssl
          arch: x86
        # Custom for this matrix: MacOS GitHub-hosted runner cannot test arm64
        # code. Exclude that scenario from running here; it will run in
        # test_desktop_custom_runners.
        - os: macos-13
          arch: arm64
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Download Desktop integration tests artifact
        uses: actions/download-artifact@v4
        with:
          path: testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
          name: testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: setup default Xcode version (macos)
        if: ${{ runner.os == 'macOS' }}
        run: |
          sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
          # Remove default Xcode version to prevent the wrong SDK from being used.
          rm -rf /Applications/Xcode.app
      - name: Install prerequisites for testing
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 15
          max_attempts: 3
          command: scripts/gha/install_test_workflow_prereqs.sh -p Desktop -t true -a '${{ matrix.arch }}' -s '${{ matrix.ssl_variant }}'
      - name: Set up Node (18)
        uses: actions/setup-node@v3
        with:
          node-version: 18.x
      - name: Setup Firestore Emulator
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 5
          max_attempts: 3
          command: npm install firebase-tools     
      - name: Add firebase-tools to PATH
        shell: bash
        run: |
          echo "$(npm root)" >> $GITHUB_PATH
          echo "$(npm root)/.bin" >> $GITHUB_PATH
      - name: Setup java
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '17'
      - name: Run Desktop integration tests
        env:
          USE_FIRESTORE_EMULATOR: true
        shell: bash
        run: |
          if [[ "${{ needs.check_and_prepare.outputs.apis }}" == *"firestore"* ]]; then
            firebase emulators:exec --only firestore --project demo-example 'python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}"'
          else
            python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}"
          fi
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          # If testapps do not exist, then it's a build error not test error.
          if [ -d "testapps/testapps-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}" ] && [ ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json ]; then
            mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log.json
          fi
      - name: Upload Desktop test results artifact
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: log-artifact-test-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
          path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: log-artifact-test-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize test results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}.log
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  test_android:
    name: test-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}
    needs: [check_and_prepare, build_android]
    runs-on: ubuntu-22.04
    if: contains(needs.check_and_prepare.outputs.matrix_platform, 'Android') && needs.check_and_prepare.outputs.apis != '' && !cancelled()
    strategy:
      fail-fast: false
      matrix:
        build_os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_os) }}
        android_device: ${{ fromJson(needs.check_and_prepare.outputs.android_device) }}
        test_type: ["gameloop"]
        exclude:
          -  android_device: "android_target"
             test_type: "uitest"
          -  android_device: "android_latest"
             test_type: "uitest"
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Download Android integration tests artifact
        uses: actions/download-artifact@v4
        with:
          path: testapps/testapps-android-${{ matrix.build_os }}
          name: testapps-android-${{ matrix.build_os }}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: Install prerequisites for testing
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 5
          max_attempts: 3
          command: scripts/gha/install_test_workflow_prereqs.sh -p Android -t true
      - name: Get Device Info
        id: device-info
        run: |
          echo "device_type=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_device_type)" >> $GITHUB_OUTPUT
          echo "device=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.android_device }} -get_ftl_device_list)" >> $GITHUB_OUTPUT
      - name: Setup java 11 for test_simulator.py
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '11'
      - name: Run Android integration tests on Emulator locally
        timeout-minutes: 180
        if: steps.device-info.outputs.device_type == 'virtual'
        run: |
          python scripts/gha/test_simulator.py --testapp_dir testapps \
            --test_type "${{ matrix.test_type }}" \
            --android_device "${{ matrix.android_device }}" \
            --logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}" \
            --ci
      - id: ftl_test
        if: steps.device-info.outputs.device_type == 'ftl'
        uses: FirebaseExtended/github-actions/firebase-test-lab@v1.4
        timeout-minutes: 240
        with:
          credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
          testapp_dir: testapps
          test_type: "game-loop"
          timeout: 1200
          test_devices: '${{ steps.device-info.outputs.device }}'
          test_device_selection: random
          max_attempts: 3
          validator: ${GITHUB_WORKSPACE}/scripts/gha/integration_testing/ftl_gha_validator.py
          additional_flags: '--client-details matrixLabel=android-${{ github.run_id }}-${{ matrix.build_os }}-${{ matrix.android_device }}'
      - name: Read FTL Test Result
        if: ${{ steps.device-info.outputs.device_type == 'ftl' && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/read_ftl_test_result.py --test_result '${{ steps.ftl_test.outputs.test_summary }}' \
            --output_path testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.log
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          # If testapps do not exist, then it's a build error not test error. 
          if [ -d "testapps/testapps-android-${{ matrix.build_os }}" ] && [ ! -f "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.log.json" ]; then
            mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.log.json"
          fi
      - name: Upload Android test results artifact
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: log-artifact-test-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}
          path: testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload Android test video artifact
        if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: mobile-simulator-test-video-artifact-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}
          path: testapps/video-*-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.mp4
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload Android test logcat artifact
        if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: mobile-simulator-test-logcat-artifact-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}
          path: testapps/logcat-*-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.txt
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: log-artifact-test-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize test results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}-${{ matrix.test_type }}.log"
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  test_ios:
    name: test-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}
    needs: [check_and_prepare, build_ios]
    runs-on: macos-13
    if: contains(needs.check_and_prepare.outputs.matrix_platform, 'iOS') && needs.check_and_prepare.outputs.apis != '' && !cancelled()
    strategy:
      fail-fast: false
      matrix:
        ios_device: ${{ fromJson(needs.check_and_prepare.outputs.ios_device) }}
        test_type: ["gameloop"]
        exclude:
          -  ios_device: "ios_min"
             test_type: "uitest"
          -  ios_device: "ios_target"
             test_type: "uitest"
          -  ios_device: "ios_latest"
             test_type: "uitest"
        build_os: [macos-13]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Download iOS integration tests artifact
        uses: actions/download-artifact@v4
        with:
          path: testapps/testapps-ios-${{ matrix.build_os }}
          name: testapps-ios-${{ matrix.build_os }}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: setup default Xcode version (macos)
        if: ${{ runner.os == 'macOS' }}
        run: |
          sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
          # Remove default Xcode version to prevent the wrong SDK from being used.
          rm -rf /Applications/Xcode.app
      - name: Install prerequisites for testing
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 3
          max_attempts: 3
          command: scripts/gha/install_test_workflow_prereqs.sh -p iOS -t true
      - name: Device Info
        id: device-info
        run: |
          echo "device_type=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_device_type)" >> $GITHUB_OUTPUT
          echo "device=$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.ios_device }} -get_ftl_device_list)" >> $GITHUB_OUTPUT
      - name: Set up Node (18)
        uses: actions/setup-node@v3
        with:
          node-version: 18.x
      - name: Setup java for Firestore emulator
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '17'
      - name: Setup Firestore Emulator
        if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 5
          max_attempts: 3
          command: npm install firebase-tools     
      - name: Add firebase-tools to PATH
        if: contains(needs.check_and_prepare.outputs.apis, 'firestore')
        shell: bash
        run: |
          echo "$(npm root)" >> $GITHUB_PATH
          echo "$(npm root)/.bin" >> $GITHUB_PATH
      - name: Start Firestore Emulator
        if: steps.device-info.outputs.device_type == 'virtual' && contains(needs.check_and_prepare.outputs.apis, 'firestore')
        run: |
          firebase emulators:start --only firestore --project demo-example &
      - name: Create keychain (macOS Simulator)
        if: ${{ runner.os == 'macOS' && steps.device-info.outputs.device_type == 'virtual'}}
        shell: bash
        run: |
          echo "Creating temporary keychain"
          # Create a local keychain on Mac:
          # Clean up previous temp keychain, if any.
          security delete-keychain tmp-keychain 2> /dev/null || true
          # Create temp keychain file and unlock it.
          # (Avoid passing in -p on command line by using interactive mode.)
          # Also set it to default settings so there is no unlock timeout.
          security -i <<EOF
          create-keychain -p ${{ secrets.TEST_SECRET }} tmp-keychain
          set-keychain-settings tmp-keychain
          unlock-keychain -p ${{ secrets.TEST_SECRET }} tmp-keychain
          EOF
          # Change the keychain list and default keychain to the temp keychain.
          security list-keychains -d user -s tmp-keychain
          security default-keychain -s tmp-keychain
      - name: Run iOS integration tests on Simulator locally
        timeout-minutes: 180
        if: steps.device-info.outputs.device_type == 'virtual'
        run: |
          python scripts/gha/test_simulator.py --testapp_dir testapps \
            --test_type "${{ matrix.test_type }}" \
            --ios_device "${{ matrix.ios_device }}" \
            --logfile_name "ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}" \
            --ci
      - id: ftl_test
        if: steps.device-info.outputs.device_type == 'ftl'
        uses: FirebaseExtended/github-actions/firebase-test-lab@v1.4
        timeout-minutes: 120
        with:
          credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
          testapp_dir: testapps
          test_type: "game-loop"
          timeout: 1200
          test_devices: '${{ steps.device-info.outputs.device }}'
          test_device_selection: random
          max_attempts: 3
          validator: ${GITHUB_WORKSPACE}/scripts/gha/integration_testing/ftl_gha_validator.py
          additional_flags: '--client-details matrixLabel=ios-${{ github.run_id }}-${{ matrix.build_os }}-${{ matrix.ios_device }}'
      - name: Read FTL Test Result
        if: ${{ steps.device-info.outputs.device_type == 'ftl' && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/read_ftl_test_result.py --test_result '${{ steps.ftl_test.outputs.test_summary }}' \
            --output_path testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}.log
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          # If testapps do not exist, then it's a build error not test error. 
          if [ -d "testapps/testapps-ios-${{ matrix.build_os }}" ] && [ ! -f "testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}.log.json" ]; then
            mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}.log.json"
          fi
      - name: Delete keychain (macOS Simulator)
        if: ${{ always() && runner.os == 'macOS' && steps.device-info.outputs.device_type == 'virtual' }}
        shell: bash
        run: |
          # Remove the local keychain on Mac:
          # Set back to the default login keychain.
          security list-keychains -d user -s login.keychain
          # Delete the temp keychain, if it exists.
          security delete-keychain tmp-keychain || true
      - name: Upload iOS test results artifact
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: log-artifact-test-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}
          path: testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload iOS test video artifact
        if: ${{ steps.device-info.outputs.device_type == 'virtual' && !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: mobile-simulator-test-video-artifact-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}
          path: testapps/video-*-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}.mp4
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: log-artifact-test-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize test results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat "testapps/test-results-ios-${{ matrix.build_os }}-${{ matrix.ios_device }}-${{ matrix.test_type }}.log"
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  test_tvos:
    name: test-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}
    needs: [check_and_prepare, build_tvos]
    runs-on: macos-13
    if: contains(needs.check_and_prepare.outputs.matrix_platform, 'tvOS') && needs.check_and_prepare.outputs.apis != '' && !cancelled()
    strategy:
      fail-fast: false
      matrix:
        tvos_device: ${{ fromJson(needs.check_and_prepare.outputs.tvos_device) }}
        build_os: [macos-13]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Download tvOS integration tests artifact
        uses: actions/download-artifact@v4
        with:
          path: testapps/testapps-tvos-${{ matrix.build_os }}
          name: testapps-tvos-${{ matrix.build_os }}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: setup default Xcode version (macos)
        if: ${{ runner.os == 'macOS' }}
        run: |
          sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
          # Remove default Xcode version to prevent the wrong SDK from being used.
          rm -rf /Applications/Xcode.app
      - name: Install prerequisites for testing
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 3
          max_attempts: 3
          command: scripts/gha/install_test_workflow_prereqs.sh -p tvOS  -t true
      - name: Setup java for Firestore emulator
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '17'
      - name: Setup Firestore Emulator
        if: contains(needs.check_and_prepare.outputs.apis, 'firestore')
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 5
          max_attempts: 3
          command: npm install firebase-tools     
      - name: Add firebase-tools to PATH
        if: contains(needs.check_and_prepare.outputs.apis, 'firestore')
        shell: bash
        run: |
          echo "$(npm root)" >> $GITHUB_PATH
          echo "$(npm root)/.bin" >> $GITHUB_PATH
      - name: Start Firestore Emulator
        if: contains(needs.check_and_prepare.outputs.apis, 'firestore')
        run: |
          firebase emulators:start --only firestore --project demo-example &
      - name: Create keychain (macOS Simulator)
        if: ${{ runner.os == 'macOS' && steps.device-info.outputs.device_type == 'virtual'}}
        shell: bash
        run: |
          echo "Creating temporary keychain"
          # Create a local keychain on Mac:
          # Clean up previous temp keychain, if any.
          security delete-keychain tmp-keychain 2> /dev/null || true
          # Create temp keychain file and unlock it.
          # (Avoid passing in -p on command line by using interactive mode.)
          # Also set it to default settings so there is no unlock timeout.
          security -i <<EOF
          create-keychain -p ${{ secrets.TEST_SECRET }} tmp-keychain
          set-keychain-settings tmp-keychain
          unlock-keychain -p ${{ secrets.TEST_SECRET }} tmp-keychain
          EOF
          # Change the keychain list and default keychain to the temp keychain.
          security list-keychains -d user -s tmp-keychain
          security default-keychain -s tmp-keychain
      - name: Run tvOS integration tests on Simulator locally
        timeout-minutes: 120
        run: |
          python scripts/gha/test_simulator.py --testapp_dir testapps \
            --tvos_device "${{ matrix.tvos_device }}" \
            --logfile_name "tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}" \
            --ci
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          # If testapps do not exist, then it's a build error not test error.
          if [ -d "testapps/testapps-tvos-${{ matrix.build_os }}" ] && [ ! -f "testapps/test-results-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}.log.json" ]; then
            mkdir -p testapps && echo "__SUMMARY_MISSING__" > "testapps/test-results-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}.log.json"
          fi
      - name: Delete keychain (macOS Simulator)
        if: ${{ always() && runner.os == 'macOS' && steps.device-info.outputs.device_type == 'virtual' }}
        shell: bash
        run: |
          # Remove the local keychain on Mac:
          # Set back to the default login keychain.
          security list-keychains -d user -s login.keychain
          # Delete the temp keychain, if it exists.
          security delete-keychain tmp-keychain || true
      - name: Upload tvOS test results artifact
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: log-artifact-test-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}
          path: testapps/test-results-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload tvOS test video artifact
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: mobile-simulator-test-video-artifact-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}
          path: testapps/video-*-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}.mp4
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: log-artifact-test-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize test results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat "testapps/test-results-tvos-${{ matrix.build_os }}-${{ matrix.tvos_device }}.log"
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  summarize_results:
    name: "summarize-results"
    needs: [check_and_prepare, test_desktop, test_android, test_ios, test_tvos]
    runs-on: ubuntu-22.04
    if: ${{ !cancelled() }}
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: Install python deps
        run: pip install -r scripts/gha/python_requirements.txt
      - name: Download log artifacts
        uses: actions/download-artifact@v4
        with:
          path: test_results
          pattern: log-artifact-*
          merge-multiple: true
      # Use a different token to remove the "in-progress" label,
      # to allow the removal to trigger the "Check Labels" workflow.
      - name: Generate token for GitHub API
        uses: tibdex/github-app-token@v1
        id: generate-token
        with:
          app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
          private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage end \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}} \
            --new_token ${{steps.generate-token.outputs.token}}
      - name: Update Daily Report
        if: needs.check_and_prepare.outputs.trigger == 'scheduled_trigger'
        shell: bash
        run: |
          if [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
            additional_flags=(--build_against sdk)
          else
            if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
              # at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
              additional_flags=(--build_against tip)
            else
              additional_flags=(--build_against repo)
            fi
          fi
          if [[ "${{ needs.check_and_prepare.outputs.apis }}" == "firestore" ]]; then
            additional_flags+=(--build_apis firestore)
          else
            additional_flags+=(--build_apis all_except_firestore)
          fi
          python scripts/gha/it_workflow.py --stage report \
            --token ${{github.token}} \
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}} \
            ${additional_flags[*]}
      - name: Summarize results into GitHub log
        run: python scripts/gha/summarize_test_results.py --dir test_results --github_log

  attempt_retry:
    name: "attempt-retry"
    needs: [check_and_prepare, summarize_results]
    runs-on: ubuntu-22.04
    if: ${{ failure() && needs.check_and_prepare.outputs.trigger == 'scheduled_trigger' }}
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: Install python deps
        run: pip install -r scripts/gha/python_requirements.txt
      # The default token can't run workflows, so get an alternate token.
      - name: Generate token for GitHub API
        uses: tibdex/github-app-token@v1
        id: generate-token
        with:
          app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
          private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
      - name: Retry failed tests
        run: |
          echo "::warning ::Attempting to retry failed tests"
          python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} \
          -w retry-test-failures.yml \
          -p run_id ${{ github.run_id }} \
          -s 10 \
          -A