model-unittest-gpu-ce-develop.yml

name: Model Unittest GPU CE Develop

on:
  schedule:
    - cron: "0 9 * * 0-4"
    - cron: "0 10 * * 0-4"
  workflow_dispatch:
    inputs:
      update_baseline_models:
        description: 'Update baseline'
        required: false
        default: 'false'
        type: choice
        options:
          - 'false'
          - 'all'
          - 'llama'
          - 'qwen3'
          - 'glm4_moe'
          - 'qwen3_next'
          - 'qwen3moe'
          - 'phi3'
          - 'deepseek_v3'
          - 'qwen2_moe'
          - 'qwen2'
          - 'gemma3_text'
          - 'paddleocr_vl'
      FLAGS_enable_CE:
        required: false
        default: 'CE_Develop_cu130_py312'
        type: choice
        options:
          - 'CE_Release_cu129_py312_nightly'
          - 'CE_Develop_cu130_py313'
          - 'CE_Develop_cu130_py312'
          - 'CE_Release_cu130_py313'
          - 'CE_Release_cu130_py312'
          - 'CE_Release_cu130_py311'
          - 'CE_Release_cu129_py313'
          - 'CE_Release_cu129_py312_weekly'
          - 'CE_Release_cu126_py310'
  workflow_call:
    inputs:
      runner:
        required: false
        type: string
      image_name:
        required: false
        type: string

permissions:
  contents: write
  pages: write
  id-token: write

# concurrency:
#   group: model-unittest-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
#   cancel-in-progress: true

env:
  PR_ID: ${{ github.event.pull_request.number || github.event.issue.number || '0' }}
  COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
  TASK: PaddleFormers-CE-${{ github.event.pull_request.number }}-model-unittest-gpu
  BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
  CE_JOB_NAME: model-unittest-gpu-ce-develop
  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"

defaults:
  run:
    shell: bash

jobs:
  check_skip:
    name: Check skip-models-ce label
    runs-on: ubuntu-latest

    outputs:
      skip: ${{ steps.check_skip.outputs.skip }}

    steps:
      - name: Check skip-models-ce label
        id: check_skip
        shell: bash
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
            echo "PR labels: $labels"
            if echo "$labels" | grep -q "skip-models-ce"; then
              echo "skip=true" >> "$GITHUB_OUTPUT"
            else
              echo "skip=false" >> "$GITHUB_OUTPUT"
            fi
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Skip CE but mark success
        if: steps.check_skip.outputs.skip == 'true'
        run: |
          echo "skip-models-ce label found"
          echo "Downstream GPU jobs will be skipped"
  model-unittest-gpu-ce-develop:
    needs: check_skip
    if: needs.check_skip.outputs.skip == 'false'
    name: model-unittest-gpu-ce-develop
    runs-on: 'models'
    steps:
      - name: Determine CE branch by time
        id: determine
        run: |
          schedule_time="${{ github.event.schedule || '' }}"
          if [[ "$schedule_time" == "0 9 "* ]]; then
            echo "FLAGS_enable_CE=CE_Develop_cu130_py313" >> $GITHUB_OUTPUT
          elif [[ "$schedule_time" == "0 10 "* ]]; then
            echo "FLAGS_enable_CE=CE_Develop_cu130_py312" >> $GITHUB_OUTPUT
          else
            echo "FLAGS_enable_CE=${{ github.event.inputs.FLAGS_enable_CE || 'CE_Develop_cu130_py313' }}" >> $GITHUB_OUTPUT
          fi
      - name: Determine Image Name
        env:
          IMAGE_NAME: ${{ inputs.image_name }}
          FLAGS_enable_CE: ${{ steps.determine.outputs.FLAGS_enable_CE }}
        run: |
          if [[ -n "${IMAGE_NAME}" ]]; then
            echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
          elif [[ "${FLAGS_enable_CE}" == *cu130* ]]; then
            echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda130-dev" >> "$GITHUB_ENV"
          elif [[ "${FLAGS_enable_CE}" == *cu129* ]]; then
            echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda129-dev" >> "$GITHUB_ENV"
          elif [[ "${FLAGS_enable_CE}" == *cu126* ]]; then
            echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev" >> "$GITHUB_ENV"
          else
            echo "IMAGE_NAME=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev" >> "$GITHUB_ENV"
          fi
      - name: Run Container
        env:
          FLAGS_dynamic_static_unified_comm: "True"
          PIP_CACHE_DIR: /root/.cache/pip
        run: |
          work_dir="${{ github.workspace }}"
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> "$GITHUB_ENV"
          DOCKER_VER=$(docker version --format '{{.Server.Version}}' | cut -d. -f1,2)
          if (( $(echo "$DOCKER_VER < 19.03" | bc -l) )); then
            GPU_OPTION="--runtime=nvidia"
          else
            GPU_OPTION="--gpus all"
          fi
          echo "DOCKER_VER=${DOCKER_VER}"
          echo "GPU_OPTION=${GPU_OPTION}"
          docker run -d -t ${GPU_OPTION} --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
            -v ${work_dir}/../../..:${work_dir}/../../.. \
            -v ${work_dir}:/workspace \
            -v /home/.cache/:/root/.cache/ \
            -v /home/paddle-1/models/:/home/models/ \
            -e "BRANCH=$BRANCH" \
            -e "AGILE_COMPILE_BRANCH=$AGILE_COMPILE_BRANCH" \
            -e "PR_ID=$PR_ID" \
            -e "COMMIT_ID=$COMMIT_ID" \
            -e "work_dir=$work_dir" \
            -e "no_proxy=$no_proxy" \
            -e "CE_JOB_NAME=$CE_JOB_NAME" \
            -e "FLAGS_dynamic_static_unified_comm=$FLAGS_dynamic_static_unified_comm" \
            -e HF_PROXY_PATH=${work_dir}/../../../proxy_huggingface \
            -e AISTUDIO_PROXY_PATH=${work_dir}/../../../proxy_aistudio \
            -e PF_HOME=/home/models/ \
            -e PIP_CACHE_DIR \
            -w /workspace --privileged ${IMAGE_NAME}
            
      - name: Download Code
        run: |
          work_dir="${{ github.workspace }}"
          docker exec -i -e work_dir="$work_dir" -e PR_ID="$PR_ID" -e BRANCH="$BRANCH" $container_name /bin/bash <<-DOCKER_EOF
          rm -rf * .[^.]*
          echo "Downloading PaddleFormers.tar"
          wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleFormers.tar --no-check-certificate
          echo "Extracting PaddleFormers.tar"
          rm -rf PaddleFormers
          tar xf PaddleFormers.tar && rm -rf PaddleFormers.tar
          echo "work_dir = ${work_dir}"
          source ${work_dir}/../../../proxy_huggingface
          cd PaddleFormers
          git config --global user.name "PaddleCI"
          git config --global user.email "paddle_ci@example.com"
          git pull
          git submodule update --init --recursive --force
          if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
            git fetch origin pull/${PR_ID}/head
            git checkout -b PR_${PR_ID} FETCH_HEAD
            git remote add upstream https://github.com/PaddlePaddle/PaddleFormers.git
            echo "Checking out ${BRANCH}..."
            git fetch upstream ${BRANCH}:${BRANCH}
            git merge ${BRANCH} --no-edit
            git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
          else
            echo "Not in a pull_request event. Skipping PR-specific operations."
          fi
          git log --pretty=oneline -10
          DOCKER_EOF
      - name: Determine trigger mode
        id: trigger_mode
        env:
          COMMENT_BODY: ${{ github.event.comment.body }}
          PR_BODY: ${{ github.event.pull_request.body }}
        run: |
          UPDATE_BASELINE_MODELS="false"
          MODE="pr"
          PR_NUMBER="${PR_ID}"
          
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            MODE="manual"
            UPDATE_BASELINE_MODELS="${{ github.event.inputs.update_baseline_models }}"
            PR_NUMBER="manual"
          elif [[ "${{ github.event_name }}" == "issue_comment" ]]; then
            # Parse comment for [update-baseline: models]
            if grep -qE '\[update-baseline:[[:space:]]*[a-zA-Z0-9_,]+\]' <<< "$COMMENT_BODY"; then
              UPDATE_BASELINE_MODELS=$(grep -oE '\[update-baseline:[[:space:]]*([a-zA-Z0-9_,]+)\]' <<< "$COMMENT_BODY" | sed 's/.*\[\(.*\)\].*/\1/' | sed 's/update-baseline:[[:space:]]*//g')
              MODE="comment"
              PR_NUMBER="${{ github.event.issue.number }}"
              echo "Detected baseline update from comment: $UPDATE_BASELINE_MODELS"
            fi
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            # Parse PR description for [update-baseline: models]
            if grep -qE '\[update-baseline:[[:space:]]*[a-zA-Z0-9_,]+\]' <<< "$PR_BODY"; then
              UPDATE_BASELINE_MODELS=$(grep -oE '\[update-baseline:[[:space:]]*([a-zA-Z0-9_,]+)\]' <<< "$PR_BODY" | sed 's/.*\[\(.*\)\].*/\1/' | sed 's/update-baseline:[[:space:]]*//g')
              MODE="pr"
              echo "Detected baseline update from PR description: $UPDATE_BASELINE_MODELS"
            fi
          fi
          
          echo "mode=$MODE" >> "$GITHUB_OUTPUT"
          echo "update_baseline_models=$UPDATE_BASELINE_MODELS" >> "$GITHUB_OUTPUT"
          echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
      
      - name: Change python version
        env:
          FLAGS_enable_CE: ${{ steps.determine.outputs.FLAGS_enable_CE }}
        run: |
          python_version=$(echo "${FLAGS_enable_CE##*_py}" | sed 's/_.*$//;s/\(.\)\(..\)/\1.\2/')
          work_dir="${{ github.workspace }}"
          docker exec -i -e "work_dir=$work_dir" -e "python_version=$python_version" $container_name /bin/bash <<-DOCKER_EOF
          echo "Change python version to $python_version"
          set -e
          cp ${work_dir}/../../../Miniconda3-latest-Linux-x86_64.sh /tmp/miniconda.sh
          bash /tmp/miniconda.sh -b -p /opt/conda
          export PATH=/opt/conda/bin:$PATH
          source ${work_dir}/../../../proxy_huggingface
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
          conda create -n py_$python_version python=$python_version -y
          . /opt/conda/etc/profile.d/conda.sh
          conda activate py_$python_version
          python --version
          DOCKER_EOF
      
      - name: Test
        env:
          FLAGS_enable_CE: ${{ steps.determine.outputs.FLAGS_enable_CE }}
        run: |
          python_version=$(echo "${FLAGS_enable_CE##*_py}" | sed 's/_.*$//;s/\(.\)\(..\)/\1.\2/')
          # Use step outputs from previous step instead of env vars
          MODE="${{ steps.trigger_mode.outputs.mode }}"
          UPDATE_BASELINE_MODELS="${{ steps.trigger_mode.outputs.update_baseline_models }}"
          PR_NUMBER="${{ steps.trigger_mode.outputs.pr_number }}"
          BRANCH="${{ github.event.pull_request.base.ref || github.ref_name }}"
          work_dir="${{ github.workspace }}"
          docker exec -i \
            -e python_version="$python_version" \
            -e FLAGS_enable_CE="$FLAGS_enable_CE" \
            -e work_dir="$work_dir" \
            -e MODE="$MODE" \
            -e UPDATE_BASELINE_MODELS="$UPDATE_BASELINE_MODELS" \
            -e PR_NUMBER="$PR_NUMBER" \
            -e BRANCH="$BRANCH" \
            -e GITHUB_REPOSITORY="$GITHUB_REPOSITORY" \
            $container_name /bin/bash <<-DOCKER_EOF
          . /opt/conda/etc/profile.d/conda.sh
          conda activate py_$python_version
          python --version
          ldconfig
          mkdir -p /root/.cache/pip
          pip cache dir
          set -e
          rm -rf /root/.cache/aistudio/
          cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
          echo "=== Trigger Mode: ${MODE} ==="
          echo "Update baseline: ${UPDATE_BASELINE_MODELS}"
          echo "BRANCH: ${BRANCH}"
          echo "PR number: ${PR_NUMBER}"
          source ${work_dir}/../../../proxy_huggingface
          bash -x scripts/regression/ci_model_unittest.sh False $FLAGS_enable_CE "$UPDATE_BASELINE_MODELS" $BRANCH ${PR_NUMBER}
          DOCKER_EOF
      - name: Upload Products
        if: always()
        env:
          home_path: ${{ github.workspace }}/../../..
          bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
          java_file: ${{ github.workspace }}/../../../jdk1.8.0_351
          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
          FLAGS_enable_CE: ${{ steps.determine.outputs.FLAGS_enable_CE }}
        run: |
          docker exec -t -e FLAGS_enable_CE="$FLAGS_enable_CE" $container_name /bin/bash -c '
          source ${work_dir}/../../../proxy_huggingface
          python -m pip install bce-python-sdk==0.8.74 -i https://pypi.org/simple
          if [ ! -f "${{ env.bos_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
            mkdir ${{ env.home_path }}/bos
            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
          fi
          if [ ! -f "${{ env.java_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/java_linux.tar.gz https://paddle-qa.bj.bcebos.com/java/java_linux.tar.gz --no-check-certificate
            tar xf ${{ env.home_path }}/java_linux.tar.gz
          fi
          if [ ! -f "${{ env.allure_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
            unzip -q ${{ env.home_path }}/allure-2.19.0.zip
          fi
          if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
            bos_prefix="${PR_ID}/${COMMIT_ID}"
          else
            bos_prefix="schedule_${FLAGS_enable_CE}/$(date +%Y%m%d)"
          fi
          # logs
          cd /workspace/PaddleFormers/model_unittest_logs
          for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do
            file=$(basename "$FILE")
            python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
            echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file"
          done
          # allure
          cd /workspace/PaddleFormers/
          export JAVA_HOME="${{ env.java_file }}"
          export JRE_HOME="${JAVA_HOME}/jre"
          export CLASSPATH="${JAVA_HOME}/lib"
          export PATH="${JAVA_HOME}/bin:${PATH}"
          ${{ env.allure_file }} generate result -o report        
          tar -czf report.tar.gz report
          python ${{ env.bos_file }} report.tar.gz paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
          echo "report: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/report.tar.gz"
          '
      - name: Terminate And Delete the Container
        if: always()
        run: |
          docker rm -f $container_name 2>/dev/null || true

  upload-allure:
    name: upload-allure
    needs: [model-unittest-gpu-ce-develop]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          repository: PaddlePaddle/PaddleFormers
          ref: develop
      - name: Checkout gh-pages
        uses: actions/checkout@v5
        with:
          ref: gh-pages
          path: gh-pages
        continue-on-error: true
      - name: Determine FLAGS_enable_CE
        id: determine
        run: |
          schedule_time="${{ github.event.schedule || '' }}"
          if [[ "$schedule_time" == "0 9 "* ]]; then
            echo "FLAGS_enable_CE=CE_Develop_cu130_py313" >> $GITHUB_OUTPUT
          elif [[ "$schedule_time" == "0 10 "* ]]; then
            echo "FLAGS_enable_CE=CE_Develop_cu130_py312" >> $GITHUB_OUTPUT
          else
            echo "FLAGS_enable_CE=${{ github.event.inputs.FLAGS_enable_CE || 'CE_Develop_cu130_py313' }}" >> $GITHUB_OUTPUT
          fi
      - name: Restore history
        env:
          FLAGS_enable_CE: ${{ steps.determine.outputs.FLAGS_enable_CE }}
        run: |          
          if [ -d "gh-pages/model_${FLAGS_enable_CE}/history" ]; then
            mkdir -p result
            cp -r gh-pages/model_${FLAGS_enable_CE}/history result/history
          fi
      - name: Download report
        id: download_report
        env:
          FLAGS_enable_CE: ${{ steps.determine.outputs.FLAGS_enable_CE }}
        run: |
          if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
            bos_prefix="${PR_ID}/${COMMIT_ID}"
          else
            bos_prefix="schedule_${FLAGS_enable_CE}/$(date +%Y%m%d)"
          fi
          echo "bos_prefix=${bos_prefix}" >> "$GITHUB_OUTPUT"

          wget -q --no-proxy \
            https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/report.tar.gz \
            --no-check-certificate
          echo "Current directory: $(pwd)"
          echo "Files after download:"
          ls -la
          tar -xzf report.tar.gz
          echo "Files after extraction:"
          ls -la
          if [ -d "report" ]; then
            echo "report directory exists"
          else
            echo "ERROR: report directory not found!"
            exit 1
          fi
      - name: Deploy to GitHub Pages
        if: always()
        env:
          FLAGS_enable_CE: ${{ steps.determine.outputs.FLAGS_enable_CE }}
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./report
          destination_dir: model_${{ env.FLAGS_enable_CE }}
      - name: Display GitHub Pages URL
        if: always()
        env:
          FLAGS_enable_CE: ${{ steps.determine.outputs.FLAGS_enable_CE }}
        run: |
          # GitHub Pages URL
          PAGES_URL="https://paddlepaddle.github.io/PaddleFormers/model_${FLAGS_enable_CE}/"
          echo "GitHub Allure Pages URL: ${PAGES_URL}"

          echo "## 🚀 Allure Report" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Item | Value |" >> $GITHUB_STEP_SUMMARY
          echo "|---|---|" >> $GITHUB_STEP_SUMMARY
          echo "| Workflow | ${{ github.workflow }} |" >> $GITHUB_STEP_SUMMARY
          echo "| CE Mode | $FLAGS_enable_CE |" >> $GITHUB_STEP_SUMMARY
          echo "| Report | [Open Report](${PAGES_URL}) |" >> $GITHUB_STEP_SUMMARY