model-unittest-gpu.yml

name: Model Unittest GPU CI

on:
  pull_request:
  schedule:
    - cron: '0 16 * * *'  # 毎日凌晨1:00 JST
  workflow_dispatch:
    inputs:
      update_baseline_models:
        description: 'Update baseline'
        required: false
        default: 'false'
        type: choice
        options:
          - 'false'
          - 'all'
          - 'llama'
          - 'qwen3'
          - 'glm4_moe'
          - 'qwen3_next'
          - 'qwen3moe'
          - 'phi3'
          - 'qwen2'
          - 'gemma3_text'
          - 'paddleocr_vl'
          - 'qwen2_moe'
          - 'qwen3_vl'
          - 'qwen3_vl_moe'
          - 'qwen2_5_vl' 
          - 'deepseek_v3'
  workflow_call:
    inputs:
      runner:
        required: false
        type: string
      image_name:
        required: false
        type: string
permissions: 
    pull-requests: read
    contents: read
    actions: read
    checks: read

concurrency:
  group: model-unittest-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

env:
  PR_ID: ${{ github.event.pull_request.number || github.event.issue.number || '0' }}
  COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
  TASK: PaddleFormers-CI-${{ github.event.pull_request.number }}-model-unittest-gpu
  BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
  CI_JOB_NAME: model-unittest-gpu-ci
  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"

defaults:
  run:
    shell: bash

jobs:
  check_skip:
    name: Check skip-models-ci label
    runs-on: ubuntu-latest

    outputs:
      skip: ${{ steps.check_skip.outputs.skip }}

    steps:
      - name: Check skip-models-ci label
        id: check_skip
        shell: bash
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
            echo "PR labels: $labels"
            if echo "$labels" | grep -q "skip-models-ci"; then
              echo "skip=true" >> "$GITHUB_OUTPUT"
            else
              echo "skip=false" >> "$GITHUB_OUTPUT"
            fi
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Skip CI but mark success
        if: steps.check_skip.outputs.skip == 'true'
        run: |
          echo "skip-models-ci label found"
          echo "Downstream GPU jobs will be skipped"
  model-unittest-gpu-ci:
    needs: check_skip
    if: needs.check_skip.outputs.skip == 'false'
    name: model-unittest-gpu-ci
    runs-on: ${{ inputs.runner || 'ernie-8gpu' || 'distrbute' }}
    steps:
      - name: Determine Image Name
        env:
          IMAGE_NAME: ${{ inputs.image_name }}
        run: |
          if [[ -n "${IMAGE_NAME}" ]]; then
            echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
          else
            echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda129-dev" >> "$GITHUB_ENV"
          fi
      - name: Run Container
        env:
          FLAGS_dynamic_static_unified_comm: "True"
          python_version: "3.12"
          PIP_CACHE_DIR: /root/.cache/pip
          paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Training-Linux-Gpu-Cuda12.9-Cudnn9.9-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse/latest/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
        run: |
          work_dir="${{ github.workspace }}"
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> "$GITHUB_ENV"
          DOCKER_VER=$(docker version --format '{{.Server.Version}}' | cut -d. -f1,2)
          if (( $(echo "$DOCKER_VER < 19.03" | bc -l) )); then
            GPU_OPTION="--runtime=nvidia"
          else
            GPU_OPTION="--gpus all"
          fi
          echo "DOCKER_VER=${DOCKER_VER}"
          echo "GPU_OPTION=${GPU_OPTION}"
          docker run -d -t ${GPU_OPTION} --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
            -v ${work_dir}/../../..:${work_dir}/../../.. \
            -v ${work_dir}:/workspace \
            -v /home/.cache/:/root/.cache/ \
            -v /home/paddle-1/models/:/home/models/ \
            -e "BRANCH=$BRANCH" \
            -e "AGILE_COMPILE_BRANCH=$AGILE_COMPILE_BRANCH" \
            -e "PR_ID=$PR_ID" \
            -e "COMMIT_ID=$COMMIT_ID" \
            -e "work_dir=$work_dir" \
            -e "no_proxy=$no_proxy" \
            -e "CI_JOB_NAME=$CI_JOB_NAME" \
            -e "paddle_whl=$paddle_whl" \
            -e "FLAGS_dynamic_static_unified_comm=$FLAGS_dynamic_static_unified_comm" \
            -e "python_version=$python_version" \
            -e HF_PROXY_PATH=${work_dir}/../../../proxy_huggingface \
            -e AISTUDIO_PROXY_PATH=${work_dir}/../../../proxy_aistudio \
            -e PF_HOME=/home/models/ \
            -e PIP_CACHE_DIR \
            -w /workspace --privileged ${IMAGE_NAME}
            
      - name: Download Code
        run: |
          work_dir="${{ github.workspace }}"
          docker exec -i -e work_dir="$work_dir" -e PR_ID="$PR_ID" -e BRANCH="$BRANCH" $container_name /bin/bash <<-DOCKER_EOF
          rm -rf * .[^.]*
          echo "Downloading PaddleFormers.tar"
          wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleFormers.tar --no-check-certificate
          echo "Extracting PaddleFormers.tar"
          rm -rf PaddleFormers
          tar xf PaddleFormers.tar && rm -rf PaddleFormers.tar
          echo "work_dir = ${work_dir}"
          source ${work_dir}/../../../proxy
          cd PaddleFormers
          git config --global user.name "PaddleCI"
          git config --global user.email "paddle_ci@example.com"
          git pull
          git submodule update --init --recursive --force
          if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
            git fetch origin pull/${PR_ID}/head
            git checkout -b PR_${PR_ID} FETCH_HEAD
            git remote add upstream https://github.com/PaddlePaddle/PaddleFormers.git
            echo "Checking out ${BRANCH}..."
            git fetch upstream ${BRANCH}:${BRANCH}
            git merge ${BRANCH} --no-edit
            git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
          else
            echo "Not in a pull_request event. Skipping PR-specific operations."
          fi
          git log --pretty=oneline -10
          DOCKER_EOF
      - name: Determine trigger mode
        id: trigger_mode
        env:
          COMMENT_BODY: ${{ github.event.comment.body }}
          PR_BODY: ${{ github.event.pull_request.body }}
        run: |
          UPDATE_BASELINE_MODELS="false"
          MODE="pr"
          PR_NUMBER="${PR_ID}"
          
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            MODE="manual"
            UPDATE_BASELINE_MODELS="${{ github.event.inputs.update_baseline_models }}"
            PR_NUMBER="manual"
          elif [[ "${{ github.event_name }}" == "schedule" ]]; then
            MODE="schedule"
            UPDATE_BASELINE_MODELS="${{ github.event.inputs.update_baseline_models }}"
            PR_NUMBER=0 
          elif [[ "${{ github.event_name }}" == "issue_comment" ]]; then
            # Parse comment for [update-baseline: models]
            if grep -qE '\[update-baseline:[[:space:]]*[a-zA-Z0-9_,]+\]' <<< "$COMMENT_BODY"; then
              UPDATE_BASELINE_MODELS=$(grep -oE '\[update-baseline:[[:space:]]*([a-zA-Z0-9_,]+)\]' <<< "$COMMENT_BODY" | sed 's/.*\[\(.*\)\].*/\1/' | sed 's/update-baseline:[[:space:]]*//g')
              MODE="comment"
              PR_NUMBER="${{ github.event.issue.number }}"
              echo "Detected baseline update from comment: $UPDATE_BASELINE_MODELS"
            fi
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            # Parse PR description for [update-baseline: models]
            if grep -qE '\[update-baseline:[[:space:]]*[a-zA-Z0-9_,]+\]' <<< "$PR_BODY"; then
              UPDATE_BASELINE_MODELS=$(grep -oE '\[update-baseline:[[:space:]]*([a-zA-Z0-9_,]+)\]' <<< "$PR_BODY" | sed 's/.*\[\(.*\)\].*/\1/' | sed 's/update-baseline:[[:space:]]*//g')
              MODE="pr"
              echo "Detected baseline update from PR description: $UPDATE_BASELINE_MODELS"
            fi
          fi
          
          echo "mode=$MODE" >> "$GITHUB_OUTPUT"
          echo "update_baseline_models=$UPDATE_BASELINE_MODELS" >> "$GITHUB_OUTPUT"
          echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
      - name: Change python version
        env:
          work_dir: ${{ github.workspace }}
          python_version: "3.12"
        run: |
          docker exec -i -e "work_dir=$work_dir" -e "python_version=$python_version" $container_name /bin/bash <<-DOCKER_EOF
          echo "Change python version to $python_version"
          set -e
          cp ${work_dir}/../../../Miniconda3-latest-Linux-x86_64.sh /tmp/miniconda.sh
          bash /tmp/miniconda.sh -b -p /opt/conda
          export PATH=/opt/conda/bin:$PATH
          # source ${work_dir}/../../../proxy_huggingface
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
          conda create -n py_$python_version python=$python_version -y
          . /opt/conda/etc/profile.d/conda.sh
          conda activate py_$python_version
          python --version
          DOCKER_EOF
      - name: Test
        env:
          work_dir: ${{ github.workspace }}
          python_version: "3.12"
        run: |
          # Use step outputs from previous step instead of env vars
          MODE="${{ steps.trigger_mode.outputs.mode }}"
          UPDATE_BASELINE_MODELS="${{ steps.trigger_mode.outputs.update_baseline_models }}"
          PR_NUMBER="${{ steps.trigger_mode.outputs.pr_number }}"
          BRANCH="${{ github.event.pull_request.base.ref || github.ref_name }}"
          docker exec -i \
            -e python_version="$python_version" \
            -e work_dir="$work_dir" \
            -e MODE="$MODE" \
            -e UPDATE_BASELINE_MODELS="$UPDATE_BASELINE_MODELS" \
            -e PR_NUMBER="$PR_NUMBER" \
            -e BRANCH="$BRANCH" \
            -e GITHUB_REPOSITORY="$GITHUB_REPOSITORY" \
            $container_name /bin/bash <<-DOCKER_EOF
          . /opt/conda/etc/profile.d/conda.sh
          conda activate py_$python_version
          python --version
          ldconfig
          mkdir -p /root/.cache/pip
          pip cache dir
          set -e
          rm -rf /root/.cache/aistudio/
          cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
          echo "=== Trigger Mode: ${MODE} ==="
          echo "Update baseline: ${UPDATE_BASELINE_MODELS}"
          echo "BRANCH: ${BRANCH}"
          echo "PR number: ${PR_NUMBER}"
          if [[ "$MODE" == "pr" ]]; then
            bash -x scripts/regression/ci_model_unittest.sh True False "$UPDATE_BASELINE_MODELS" $BRANCH ${PR_NUMBER}
          elif [[ "$MODE" == "schedule" ]]; then
            bash -x scripts/regression/ci_model_unittest.sh True False False develop 0
          fi
          DOCKER_EOF
      - name: Upload Products
        if: always()
        env:
          home_path: ${{ github.workspace }}/../../..
          bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
          java_file: ${{ github.workspace }}/../../../jdk1.8.0_351
          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
        run: |
          docker exec -t $container_name /bin/bash -c '
          source ${work_dir}/../../../proxy_huggingface
          python -m pip install bce-python-sdk==0.8.74 -i https://pypi.org/simple
          if [ ! -f "${{ env.bos_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
            mkdir ${{ env.home_path }}/bos
            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
          fi
          if [ ! -f "${{ env.java_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/java_linux.tar.gz https://paddle-qa.bj.bcebos.com/java/java_linux.tar.gz --no-check-certificate
            tar xf ${{ env.home_path }}/java_linux.tar.gz
          fi
          if [ ! -f "${{ env.allure_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
            unzip -q ${{ env.home_path }}/allure-2.19.0.zip
          fi
          if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
            bos_prefix="${PR_ID}/${COMMIT_ID}"
          else
            bos_prefix="schedule/$(date +%Y%m%d)"
          fi
          # logs
          cd /workspace/PaddleFormers/model_unittest_logs
          for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do
            file=$(basename "$FILE")
            python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
            echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file"
          done
          # allure
          cd /workspace/PaddleFormers/
          export JAVA_HOME="${{ env.java_file }}"
          export JRE_HOME="${JAVA_HOME}/jre"
          export CLASSPATH="${JAVA_HOME}/lib"
          export PATH="${JAVA_HOME}/bin:${PATH}"
          ${{ env.allure_file }} generate result -o report        
          tar -czf report.tar.gz report
          python ${{ env.bos_file }} report.tar.gz paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
          echo "report: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/report.tar.gz"
          '
      - name: Terminate And Delete the Container
        if: always()
        run: |
          docker rm -f $container_name 2>/dev/null || true