tests.yaml

name: tests

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:
  workflow_call:
    inputs:
      run_number:
        default: '1'
        required: true
        type: string

concurrency:
  group: tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  PYTHONWARNINGS: error
  BASE_IMAGE: openpilot-base
  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}

  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
  BUILD: selfdrive/test/docker_build.sh base

  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c

  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical

jobs:
  build_release:
    name: build release
    runs-on: ${{
      (github.repository == 'commaai/openpilot') &&
      ((github.event_name != 'pull_request') ||
      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
      || fromJSON('["ubuntu-24.04"]') }}
    env:
      STRIPPED_DIR: /tmp/releasepilot
    steps:
    - uses: actions/checkout@v6
      with:
        submodules: true
    - name: Getting LFS files
      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
      with:
        timeout_minutes: 2
        max_attempts: 3
        command: git lfs pull
    - name: Build devel
      timeout-minutes: 1
      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
    - uses: ./.github/workflows/setup-with-retry
    - name: Build openpilot and run checks
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "python3 system/manager/build.py"
    - name: Run tests
      timeout-minutes: 1
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "release/check-dirty.sh"
    - name: Check submodules
      if: github.repository == 'commaai/openpilot'
      timeout-minutes: 3
      run: release/check-submodules.sh

  build:
    runs-on: ${{
      (github.repository == 'commaai/openpilot') &&
      ((github.event_name != 'pull_request') ||
      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
      || fromJSON('["ubuntu-24.04"]') }}
    steps:
    - uses: actions/checkout@v6
      with:
        submodules: true
    - name: Setup docker push
      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
      run: |
        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
        $DOCKER_LOGIN
    - uses: ./.github/workflows/setup-with-retry
    - uses: ./.github/workflows/compile-openpilot
      timeout-minutes: 30

  build_mac:
    name: build macOS
    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
    steps:
    - uses: actions/checkout@v6
      with:
        submodules: true
    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
    - name: Homebrew cache
      uses: ./.github/workflows/auto-cache
      with:
        path: ~/Library/Caches/Homebrew
        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
        restore-keys: |
          brew-macos-${{ env.CACHE_COMMIT_DATE }}
          brew-macos
    - name: Install dependencies
      run: ./tools/mac_setup.sh
      env:
        PYTHONWARNINGS: default  # package install has DeprecationWarnings
        HOMEBREW_DISPLAY_INSTALL_TIMES: 1
    - run: git lfs pull
    - name: Getting scons cache
      uses: ./.github/workflows/auto-cache
      with:
        path: /tmp/scons_cache
        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
        restore-keys: |
          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
          scons-${{ runner.arch }}-macos
    - name: Building openpilot
      run: . .venv/bin/activate && scons -j$(nproc)

  static_analysis:
    name: static analysis
    runs-on: ${{
      (github.repository == 'commaai/openpilot') &&
      ((github.event_name != 'pull_request') ||
      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
      || fromJSON('["ubuntu-24.04"]') }}
    env:
      PYTHONWARNINGS: default
    steps:
    - uses: actions/checkout@v6
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Static analysis
      timeout-minutes: 1
      run: ${{ env.RUN }} "scripts/lint/lint.sh"

  unit_tests:
    name: unit tests
    runs-on: ${{
      (github.repository == 'commaai/openpilot') &&
      ((github.event_name != 'pull_request') ||
      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
      || fromJSON('["ubuntu-24.04"]') }}
    steps:
    - uses: actions/checkout@v6
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      id: setup-step
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run unit tests
      timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
      run: |
        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                        # Pre-compile Python bytecode so each pytest worker doesn't need to
                        $PYTEST --collect-only -m 'not slow' -qq && \
                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
                        chmod -R 777 /tmp/comma_download_cache"

  process_replay:
    name: process replay
    runs-on: ${{
      (github.repository == 'commaai/openpilot') &&
      ((github.event_name != 'pull_request') ||
      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
      || fromJSON('["ubuntu-24.04"]') }}
    steps:
    - uses: actions/checkout@v6
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      id: setup-step
    - name: Cache test routes
      id: dependency-cache
      uses: actions/cache@v5
      with:
        path: .ci_cache/comma_download_cache
        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run replay
      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
      run: |
        ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                        chmod -R 777 /tmp/comma_download_cache"
    - name: Print diff
      id: print-diff
      if: always()
      run: cat selfdrive/test/process_replay/diff.txt
    - uses: actions/upload-artifact@v6
      if: always()
      continue-on-error: true
      with:
        name: process_replay_diff.txt
        path: selfdrive/test/process_replay/diff.txt
    - name: Upload reference logs
      if: false # TODO: move this to github instead of azure
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
    - name: Run regen
      if: false
      timeout-minutes: 4
      run: |
        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
                        chmod -R 777 /tmp/comma_download_cache"

  simulator_driving:
    name: simulator driving
    runs-on: ${{
      (github.repository == 'commaai/openpilot') &&
      ((github.event_name != 'pull_request') ||
      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
      || fromJSON('["ubuntu-24.04"]') }}
    if: false  # FIXME: Started to timeout recently
    steps:
    - uses: actions/checkout@v6
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      id: setup-step
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Driving test
      timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }}
      run: |
        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                        source selfdrive/test/setup_vsound.sh && \
                        CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"

  create_raylib_ui_report:
    name: Create raylib UI Report
    runs-on: ${{
      (github.repository == 'commaai/openpilot') &&
      ((github.event_name != 'pull_request') ||
      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
      || fromJSON('["ubuntu-24.04"]') }}
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: true
      - uses: ./.github/workflows/setup-with-retry
      - name: Build openpilot
        run: ${{ env.RUN }} "scons -j$(nproc)"
      - name: Create raylib UI Report
        run: >
          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
                          source selfdrive/test/setup_xvfb.sh &&
                          python3 selfdrive/ui/tests/test_ui/raylib_screenshots.py"
      - name: Upload Raylib UI Report
        uses: actions/upload-artifact@v6
        with:
          name: raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
          path: selfdrive/ui/tests/test_ui/raylib_report/screenshots

  create_mici_raylib_ui_report:
    name: Create mici raylib UI Report
    runs-on: ${{
      (github.repository == 'commaai/openpilot') &&
      ((github.event_name != 'pull_request') ||
      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
      || fromJSON('["ubuntu-24.04"]') }}
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: true
      - uses: ./.github/workflows/setup-with-retry
      - name: Build openpilot
        run: ${{ env.RUN }} "scons -j$(nproc)"
      - name: Create mici raylib UI Report
        run: >
          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
                          source selfdrive/test/setup_xvfb.sh &&
                          WINDOWED=1 python3 selfdrive/ui/tests/diff/replay.py"
      - name: Upload Raylib UI Report
        uses: actions/upload-artifact@v6
        with:
          name: mici-raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
          path: selfdrive/ui/tests/diff/report