benchmark-regression.yml

name: Benchmark Regression

# EI9 - manual trend-regression gate for the adb benchmark harness. This is
# workflow_dispatch only because the suite needs a booted emulator/device and
# writes candidate JSON that should be reviewed before becoming a new baseline.

on:
  workflow_dispatch:
    inputs:
      benchmark:
        description: "Benchmark slice to run"
        required: true
        default: all
        type: choice
        options:
          - all
          - ime-first-render
          - first-suggestion-latency
          - dictionary-load
          - candidate-row
          - theme-switch
          - backup-restore
      iterations:
        description: "Iterations per adb benchmark script"
        required: true
        default: "5"
      adb_serial:
        description: "Optional adb serial; leave blank for the workflow emulator"
        required: false
        default: ""

permissions:
  contents: read

concurrency:
  group: benchmark-regression-${{ github.ref }}
  cancel-in-progress: false

jobs:
  benchmark-regression:
    runs-on: ubuntu-latest
    timeout-minutes: 90

    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          submodules: recursive

      - uses: gradle/actions/wrapper-validation@48b5f213c81028ace310571dc5ec0fbbca0b2947 # v4

      - name: Set up JDK 17
        uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
        with:
          java-version: 17
          distribution: temurin

      - name: Set up CMake and Ninja
        uses: lukka/get-cmake@ea004816823209b8d1211e47b216185caee12cc5 # v4.0.2

      - name: Cache Gradle
        uses: gradle/actions/setup-gradle@48b5f213c81028ace310571dc5ec0fbbca0b2947 # v4

      - name: Build benchmark APKs
        run: ./gradlew :app:assembleBenchmark :benchmark:assembleBenchmark

      - name: Enable KVM
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm

      - name: Run adb benchmark slice
        uses: reactivecircus/android-emulator-runner@e89f39f1abbbd05b1113a29cf4db69e7540cae5a # v2
        env:
          BENCHMARK: ${{ inputs.benchmark }}
          ITERATIONS: ${{ inputs.iterations }}
          ADB_SERIAL: ${{ inputs.adb_serial }}
        with:
          api-level: 35
          arch: x86_64
          profile: pixel_6
          emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -camera-back none
          script: |
            set -euo pipefail
            mkdir -p build/benchmark-results

            serial_args=()
            if [ -n "$ADB_SERIAL" ]; then
              serial_args=(-Serial "$ADB_SERIAL")
            fi

            run_benchmark() {
              case "$1" in
                ime-first-render)
                  pwsh -NoProfile -ExecutionPolicy Bypass -File tools/benchmark-ime-first-render.ps1 -Iterations "$ITERATIONS" "${serial_args[@]}" -OutputPath build/benchmark-results/candidate-ime-first-render.json
                  ;;
                first-suggestion-latency)
                  pwsh -NoProfile -ExecutionPolicy Bypass -File tools/benchmark-ime-suggestion-latency.ps1 -Iterations "$ITERATIONS" "${serial_args[@]}" -OutputPath build/benchmark-results/candidate-ime-suggestion-latency.json
                  ;;
                dictionary-load)
                  pwsh -NoProfile -ExecutionPolicy Bypass -File tools/benchmark-ime-dictionary-load.ps1 -Iterations "$ITERATIONS" "${serial_args[@]}" -OutputPath build/benchmark-results/candidate-ime-dictionary-load.json
                  ;;
                candidate-row)
                  pwsh -NoProfile -ExecutionPolicy Bypass -File tools/benchmark-ime-candidate-row.ps1 -Iterations "$ITERATIONS" "${serial_args[@]}" -OutputPath build/benchmark-results/candidate-ime-candidate-row.json
                  ;;
                theme-switch)
                  pwsh -NoProfile -ExecutionPolicy Bypass -File tools/benchmark-ime-theme-switch.ps1 -Iterations "$ITERATIONS" "${serial_args[@]}" -OutputPath build/benchmark-results/candidate-ime-theme-switch.json
                  ;;
                backup-restore)
                  pwsh -NoProfile -ExecutionPolicy Bypass -File tools/benchmark-backup-restore.ps1 -Iterations "$ITERATIONS" "${serial_args[@]}" -OutputPath build/benchmark-results/candidate-backup-restore.json
                  ;;
                *)
                  echo "::error::Unknown benchmark '$1'"
                  exit 2
                  ;;
              esac
            }

            if [ "$BENCHMARK" = "all" ]; then
              for slice in ime-first-render first-suggestion-latency dictionary-load candidate-row theme-switch backup-restore; do
                run_benchmark "$slice"
              done
            else
              run_benchmark "$BENCHMARK"
            fi

      - name: Compare candidate results against baselines
        env:
          BENCHMARK: ${{ inputs.benchmark }}
        run: |
          set -euo pipefail
          require_args=()
          if [ "$BENCHMARK" = "all" ]; then
            require_args=(--require-all-baselines)
          fi
          python scripts/check-benchmark-trends.py \
            --baseline-dir docs/benchmark-results \
            --candidate-dir build/benchmark-results \
            --report build/benchmark-results/benchmark-trend-report.md \
            "${require_args[@]}"

      - name: Upload benchmark trend report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: benchmark-trend-report
          path: |
            build/benchmark-results/*.json
            build/benchmark-results/benchmark-trend-report.md