android-device-test.yml

name: android-device-test

on:
  workflow_run:
    workflows: [android-ci]
    types:
      - completed

permissions:
  id-token: write         # needed for AWS

jobs:
  android-device-test:
    strategy:
      fail-fast: false
      matrix:
        test: [
          {
            artifactName: android-render-tests-opengl,
            testFile: RenderTests-opengl.apk,
            appFile: RenderTestsApp-opengl.apk,
            name: "Android Render Tests (OpenGL)",
            # Pixel 4a Android 12, Pixel 7 Pro Android 13
            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/55db2b56-930a-4485-bdc0-03409a5a6060",
            testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/c1fc7d3e-dfe3-4a31-9ee0-7b0f71b08872"
          },
          {
            artifactName: android-render-tests-vulkan,
            testFile: RenderTests-vulkan.apk,
            appFile: RenderTestsApp-vulkan.apk,
            name: "Android Render Tests (Vulkan)",
            # Pixel 4a Android 12, Pixel 7 Pro Android 13
            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/55db2b56-930a-4485-bdc0-03409a5a6060",
            # android-render-test.yml
            # see https://github.com/maplibre/ci-runners/tree/main/aws-device-farm/custom-test-envs
            testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/c1fc7d3e-dfe3-4a31-9ee0-7b0f71b08872"
          },
          {
            artifactName: benchmarkAPKs,
            testFile: "MapLibreAndroidTestApp-opengl-release-androidTest.apk",
            appFile: "MapLibreAndroidTestApp-opengl-release.apk",
            name: "Android Benchmark",
            testFilter: "org.maplibre.android.benchmark.Benchmark",
            # Google Pixel 7 Pro
            # query with `aws list-device-pools --arn <project_arn>`
            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564",
            # benchmark-android.yaml
            # see https://github.com/maplibre/ci-runners/tree/main/aws-device-farm/custom-test-envs
            testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/14862afb-cf88-44aa-9f1e-5131cbb22f01"
          },
          {
            artifactName: android-ui-test-opengl,
            testFile: InstrumentationTestsOpenGL.apk,
            appFile: InstrumentationTestAppOpenGL.apk,
            name: "Android Instrumentation Tests (OpenGL)",
            # Google Pixel 7 Pro
            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564",
            testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/09e0738e-c91e-4c5f-81e6-06a06cc340d8"
          },
          {
            artifactName: android-ui-test-vulkan,
            testFile: InstrumentationTestsVulkan.apk,
            appFile: InstrumentationTestAppVulkan.apk,
            name: "Android Instrumentation Tests (Vulkan)",
            # Google Pixel 7 Pro
            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564",
            testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/09e0738e-c91e-4c5f-81e6-06a06cc340d8"
          },
          {
            artifactName: android-cpp-tests,
            testFile: app-release-androidTest.apk,
            appFile: app-debug.apk,
            name: "Android C++ Unit Tests",
            # Google Pixel 7 Pro
            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564"
          }
        ]
    runs-on: ubuntu-latest
    if: github.repository_owner == 'maplibre' && github.event.workflow_run.conclusion == 'success'
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v4
        with:
          persist-credentials: false

      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v4
        with:
          node-version-file: '.nvmrc'

      - run: npm install

      - name: Check if android-build was skipped
        id: parent_workflow
        run: node .github/scripts/check-job-skipped.ts
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TEST_RUN_ID: ${{ github.event.workflow_run.id }}
          JOB_NAME: android-build

      # get comment from PR

      - uses: ./.github/actions/get-pr-number
        id: get-pr-number

      - name: Generate token
        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
        id: generate_token
        with:
          skip-token-revoke: true  # revoking will fail for long running workflows, because the token will already have expired
          app-id: ${{ secrets.MAPLIBRE_NATIVE_BOT_APP_ID }}
          private-key: ${{ secrets.MAPLIBRE_NATIVE_BOT_PRIVATE_KEY }}

      - name: Check if comment on PR contains '!benchmark android'
        if: matrix.test.name == 'Android Benchmark' && steps.get-pr-number.outputs.pr-number
        uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0
        id: benchmark_comment
        with:
          issue-number: ${{ steps.get-pr-number.outputs.pr-number }}
          body-regex: '^!benchmark.*android.*$'

      - name: Should we run this device test?
        # always run when something was merged into main
        # run benchmark when comment with '!benchmark android' exists in PR
        if: |
          steps.parent_workflow.outputs.was_skipped == 'false' &&
          ((github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.event == 'push') ||
          matrix.test.name == 'Android Benchmark' && steps.benchmark_comment.outputs.comment-id ||
          matrix.test.name != 'Android Benchmark')
        run:
          echo "run_device_test=true" >> "$GITHUB_ENV"

      - uses: LouisBrunner/checks-action@dfcbcf801bff1ea7f1414824fc28f2cd697b35da # v3.0.0
        id: create_check
        with:
          token: ${{ steps.generate_token.outputs.token }}
          details_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          status: in_progress
          name: ${{ matrix.test.name }}
          sha: ${{ github.event.workflow_run.head_sha }}

      - uses: ./.github/actions/download-workflow-run-artifact
        if: env.run_device_test == 'true'
        with:
          artifact-name: ${{ matrix.test.artifactName }}
          expect-files: "${{ matrix.test.testFile }}, ${{ matrix.test.appFile }}"

      - name: Configure AWS Credentials
        if: env.run_device_test == 'true'
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v4
        with:
          aws-region: us-west-2
          role-to-assume: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
          role-session-name: ${{ github.run_id }}
          role-duration-seconds: 14400

      - name: Run ${{ matrix.test.name }} on AWS Device Farm
        if: env.run_device_test == 'true'
        run: |
          export name="${{ matrix.test.name }}"
          export appType=ANDROID_APP
          export appFile="${{ matrix.test.appFile }}"
          export testFile="${{ matrix.test.testFile }}"
          export testPackageType=INSTRUMENTATION_TEST_PACKAGE
          export testType=INSTRUMENTATION
          export testFilter="${{ matrix.test.testFilter }}"
          export AWS_DEVICE_FARM_PROJECT_ARN="${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}"
          export AWS_DEVICE_FARM_DEVICE_POOL_ARN="${{ matrix.test.devicePool }}"
          export testSpecArn="${{ matrix.test.testSpecArn }}"
          export wait_for_completion=true

          set +e

          run_arn="$(./scripts/aws-device-farm/aws-device-farm-run.sh)"
          exit_status=$?
          echo "run_arn=$run_arn" >> "$GITHUB_ENV"

          # make sure workflow fails
          if [ $exit_status -ne 0 ]; then
            exit $exit_status
          fi

      - name: Store Test Artifacts
        if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
        run: |
          npm install
          results_dir="$(mktemp -d)"
          echo results_dir="$results_dir" >> "$GITHUB_ENV"
          node scripts/aws-device-farm/store-test-artifacts.mjs --runArn ${{ env.run_arn }} --outputDir "$results_dir"
          # unzip and delete .zip files so we don't have nested .zip
          find "$results_dir" -type f -name '*.zip' -exec sh -c 'unzip -o -d "$(dirname "$1")" "$1" && rm "$1"' _ {} \;
          zip -r test_artifacts.zip "$results_dir"

      - name: Store Benchmark Results
        if: matrix.test.name == 'Android Benchmark' && env.run_device_test == 'true'
        run: |
          benchmark_results_dir="$(mktemp -d)"
          node scripts/aws-device-farm/collect-benchmark-outputs.mjs --inputDir "${{ env.results_dir }}" --outputDir "$benchmark_results_dir"
          node scripts/aws-device-farm/upload-benchmark-outputs-to-s3.mjs --dir "$benchmark_results_dir"

      - name: Upload Test Artifacts
        if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: "Test Artifacts ${{ matrix.test.name }}"
          path: test_artifacts.zip

      - name: Generate another token (previous one could have expired)
        if: always()
        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
        id: generate_token_2
        with:
          app-id: ${{ secrets.MAPLIBRE_NATIVE_BOT_APP_ID }}
          private-key: ${{ secrets.MAPLIBRE_NATIVE_BOT_PRIVATE_KEY }}

      - uses: LouisBrunner/checks-action@dfcbcf801bff1ea7f1414824fc28f2cd697b35da # v3.0.0
        if: always()
        with:
          token: ${{ steps.generate_token_2.outputs.token }}
          check_id: ${{ steps.create_check.outputs.check_id }}
          conclusion: ${{ job.status }}
          sha: ${{ github.event.workflow_run.sha }}