unittest-gpu-ce.yml

name: Unittest GPU CE

on:
  schedule:
    - cron: "0 16 * * *"     # every day at 00:00 Beijing time (UTC+8)
  workflow_dispatch:        # allow to manually trigger the workflow

concurrency:
  group: unittest-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

env:
  PR_ID: ${{ github.event.pull_request.number || '0' }}
  COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
  BRANCH: ${{ github.event.pull_request.base.ref }}
  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
  work_dir: ${{ github.workspace }}
  CE_name: unittest-gpu-ce
  TASK: PaddleFormers_CE_unittest_gpu
  ce_scripts: workspace/PaddleFormers/scripts/unit_test
  IMAGE_NAME: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda129-dev
  python_version: 3.12
  paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Training-Linux-Gpu-Cuda12.9-Cudnn9.9-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse/latest/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
  PYTEST_EXECUTE_FLAG_FILE: ${{ github.workspace }}/../../../PYTEST_EXECUTE_FLAG_FILE/${{ github.event.pull_request.number || '0' }}/${{ github.event.pull_request.head.sha || github.sha }}/pytest_execute.flag
  PYTEST_EXECUTE_FLAG: false

defaults:
  run:
    shell: bash

jobs:
  check_skip:
    name: Check skip-unittest-ce label
    runs-on: 'ernie-cpu'

    outputs:
      skip: ${{ steps.check_skip.outputs.skip }}
    steps:
      - name: Check skip-unittest-ce label
        id: check_skip
        shell: bash
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
            echo "PR labels: $labels"

            if echo "$labels" | grep -q "skip-unittest-ce"; then
              echo "skip=true" >> "$GITHUB_OUTPUT"
            else
              echo "skip=false" >> "$GITHUB_OUTPUT"
            fi
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Skip CE but mark success
        if: steps.check_skip.outputs.skip == 'true'
        run: |
          echo "skip-unittest-ce label found"
          echo "Downstream GPU jobs will be skipped"

  unittest-gpu-ce:
    needs: check_skip
    if: needs.check_skip.outputs.skip == 'false'
    name: unittest-gpu-ce
    runs-on: 'ernie-unittest'
    outputs:  
      pytest_execute_flag: ${{ steps.set_pytest_flag.outputs.pytest_execute_flag }}
    steps:
      - name: Determine the runner
        run: |
          gpu_id=$(( $(echo $PWD | awk -F'/' '{print $4}' | awk -F'_' '{print $2}') ))
          echo $PWD
          echo GPU_DEVICES="$gpu_id" >> $GITHUB_ENV
      - name: Run Container
        env:
          FLAGS_dynamic_static_unified_comm: "True"
          PIP_CACHE_DIR: /root/.cache/pip
          GPU_DEVICES: ${{ env.GPU_DEVICES }}
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "GPU_DEVICES=$GPU_DEVICES"
          echo "container_name=${container_name}" >> "$GITHUB_ENV"
          echo "Workspace path: ${{ github.workspace }}"
          docker run -d -t --name ${container_name} --gpus device=${GPU_DEVICES} --runtime=nvidia --net=host -v /dev/shm:/dev/shm --shm-size=32G \
            -v $work_dir/../../..:$work_dir/../../.. \
            -v $work_dir:/workspace \
            -v /home/.cache/:/root/.cache/ \
            -v /home/paddle-1/models/:/home/models/ \
            -e PF_HOME=/home/models/ \
            -e "work_dir=$work_dir" \
            -e "PR_ID=$PR_ID" \
            -e "BRANCH=$BRANCH" \
            -e "COMMIT_ID=$COMMIT_ID" \
            -e "AGILE_COMPILE_BRANCH=$AGILE_COMPILE_BRANCH" \
            -e CUDA_VISIBLE_DEVICES=${GPU_DEVICES} \
            -e NVIDIA_VISIBLE_DEVICES=${GPU_DEVICES} \
            -e "ce_scripts=$ce_scripts" \
            -e "PYTEST_EXECUTE_FLAG_FILE=$PYTEST_EXECUTE_FLAG_FILE" \
            -e "no_proxy=$no_proxy" \
            -e "paddle_whl=$paddle_whl" \
            -e "FLAGS_dynamic_static_unified_comm=$FLAGS_dynamic_static_unified_comm" \
            -e "python_version=$python_version" \
            -e "HF_PROXY_PATH=$work_dir/../../../proxy_huggingface" \
            -e "AISTUDIO_PROXY_PATH=$work_dir/../../../proxy_aistudio" \
            -e "HF_DATASETS_CACHE=$work_dir/../../../paddlenlp/huggingface/datasets" \
            -e "TRANSFORMERS_CACHE=$work_dir/../../../paddlenlp/huggingface" \
            -e PIP_CACHE_DIR \
            -w /workspace --privileged $IMAGE_NAME
      - name: Check GPU
        run: |
          docker exec $container_name nvidia-smi
      - name: Download Code
        run: |
          docker exec -t $container_name /bin/bash -c '
            rm -rf * .[^.]*
            echo "Downloading PaddleFormers.tar"
            wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleFormers.tar --no-check-certificate
            echo "Extracting PaddleFormers.tar"
            rm -rf PaddleFormers
            tar xf PaddleFormers.tar && rm -rf PaddleFormers.tar
            source $work_dir/../../../proxy
            cd PaddleFormers
            git config --global user.name "PaddleCI"
            git config --global user.email "paddle_ci@example.com"
            git pull
            git submodule update --init --recursive --force
            if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
              git fetch origin pull/${PR_ID}/head
              git checkout -b PR_${PR_ID} FETCH_HEAD
              git remote add upstream https://github.com/PaddlePaddle/PaddleFormers.git
              git fetch upstream ${BRANCH}:${BRANCH}
              git merge ${BRANCH} --no-edit
              git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
            else
              echo "Not in a pull_request event. Skipping PR-specific operations."
            fi
            git log --pretty=oneline -10
            '
      - name: Change python version
        run: |
          docker exec -i -e "work_dir=$work_dir" -e "python_version=$python_version" $container_name /bin/bash <<-DOCKER_EOF
          echo "Change python version to $python_version"
          set -e
          cp ${work_dir}/../../../Miniconda3-latest-Linux-x86_64.sh /tmp/miniconda.sh
          bash /tmp/miniconda.sh -b -p /opt/conda
          export PATH=/opt/conda/bin:$PATH
          # source ${work_dir}/../../../proxy_huggingface
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
          conda create -n py_$python_version python=$python_version -y
          . /opt/conda/etc/profile.d/conda.sh
          conda activate py_$python_version
          python --version
          DOCKER_EOF
      - name: Test
        run: |
          docker exec -t $container_name /bin/bash -c '
          . /opt/conda/etc/profile.d/conda.sh
          conda activate py_$python_version
          python --version
          ldconfig
          mkdir -p /root/.cache/pip
          pip cache dir
          set -e
          rm -rf /root/.cache/aistudio/
          cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
          source $work_dir/../../../proxy
          source $work_dir/../../../AISTUDIO_ACCESS_TOKEN
          bash -x scripts/unit_test/ci_unittest.sh ${paddle_whl} true ${PYTEST_EXECUTE_FLAG_FILE} ${AGILE_COMPILE_BRANCH}
          '
      - name: Upload Products
        env:
          home_path: ${{ github.workspace }}/../../..
          bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
          java_file: ${{ github.workspace }}/../../../jdk1.8.0_351
          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
        run: |
          docker exec -t $container_name /bin/bash -c '
          source ${work_dir}/../../../proxy_huggingface
          python -m pip install bce-python-sdk==0.8.74 -i https://pypi.org/simple
          if [ ! -f "${{ env.bos_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
            mkdir ${{ env.home_path }}/bos
            tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
          fi
          if [ ! -f "${{ env.java_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/java_linux.tar.gz https://paddle-qa.bj.bcebos.com/java/java_linux.tar.gz --no-check-certificate
            tar xf ${{ env.home_path }}/java_linux.tar.gz
          fi
          if [ ! -f "${{ env.allure_file }}" ]; then
            wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
            unzip -q ${{ env.home_path }}/allure-2.19.0.zip
          fi
          
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            bos_prefix="manual/$(date +%Y%m%d)_${{ github.run_id }}"
          elif [[ "${{ github.event_name }}" == "schedule" ]]; then
            bos_prefix="schedule/$(date +%Y%m%d)_${{ github.run_id }}"
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            bos_prefix="${PR_ID}/${COMMIT_ID}"
          fi

          # coverage.xml
          if [ -f "${PYTEST_EXECUTE_FLAG_FILE}" ]; then
            echo "PYTEST_EXECUTE_FLAG_FILE found, uploading coverage.xml."
            cd /workspace/PaddleFormers
            python ${{ env.bos_file }} coverage.xml paddle-github-action/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs
            exit_code=$?
            if [ $exit_code -eq 0 ]; then
              echo "cov-report: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs/coverage.xml"
            else
              echo "ERROR: Failed to upload $file (exit code: $exit_code)"
            fi
          else
            echo "PYTEST_EXECUTE_FLAG_FILE not found, skipping coverage.xml upload."
          fi
          # logs
          cd /workspace/PaddleFormers/unittest_logs
          for FILE in /workspace/PaddleFormers/unittest_logs/*; do
            file=$(basename "$FILE")
            python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs
            exit_code=$?
            if [ $exit_code -eq 0 ]; then
              echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs/$file"
            else
              echo "ERROR: Failed to upload $file (exit code: $exit_code)"
            fi
            done
          # allure
          cd /workspace/PaddleFormers/
          export JAVA_HOME="${{ env.java_file }}"
          export JRE_HOME="${JAVA_HOME}/jre"
          export CLASSPATH="${JAVA_HOME}/lib"
          export PATH="${JAVA_HOME}/bin:${PATH}"
          ${{ env.allure_file }} generate result -o report
          tar -czf report.tar.gz report
          python ${{ env.bos_file }} report.tar.gz paddle-github-action/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs
          exit_code=$?
          if [ $exit_code -eq 0 ]; then
            echo "report: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs/report.tar.gz"
          else
            echo "ERROR: Failed to upload report.tar.gz (exit code: $exit_code)"
          fi
          '
      - name: Set pytest execute flag output
        id: set_pytest_flag  # 专门设置输出的步骤
        run: |
          # 检查标志文件是否存在，并设置作业输出
          if [ -f "${{ env.PYTEST_EXECUTE_FLAG_FILE }}" ]; then
            echo "pytest_execute_flag=true" >> $GITHUB_OUTPUT
            echo "PYTEST_EXECUTE_FLAG_FILE exists, setting flag to true"
          else
            echo "pytest_execute_flag=false" >> $GITHUB_OUTPUT
            echo "PYTEST_EXECUTE_FLAG_FILE does not exist, setting flag to false"
          fi
      - name: Terminate And Delete the Container
        if: always()
        run: |
          docker rm -f $container_name 2>/dev/null || true
          
  upload-coverage:
    name: upload-coverage
    needs: [unittest-gpu-ce]
    if: always()
    runs-on: 'ernie-cpu'
    steps:
      - name: Checkout Code
        uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - name: Check if should upload coverage
        id: flag_check
        run: |
          echo "needs.unittest-gpu-ce.outputs.pytest_execute_flag = ${{ needs.unittest-gpu-ce.outputs.pytest_execute_flag }}"
          if [ "${{ needs.unittest-gpu-ce.outputs.pytest_execute_flag }}" = "true" ]; then
            echo "pytest_execute_flag is true, proceeding to upload coverage."
            echo "should_upload=true" >> $GITHUB_OUTPUT
          else
            echo "pytest_execute_flag is false, skipping coverage upload."
            echo "should_upload=false" >> $GITHUB_OUTPUT
          fi
      - name: Download coverage.xml
        if: steps.flag_check.outputs.should_upload == 'true'
        env:
          PR_ID: ${{ github.event.pull_request.number || '0' }}
          COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
        run: |
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            bos_prefix="manual/$(date +%Y%m%d)_${{ github.run_id }}"
          elif [[ "${{ github.event_name }}" == "schedule" ]]; then
            bos_prefix="schedule/$(date +%Y%m%d)_${{ github.run_id }}"
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            bos_prefix="${PR_ID}/${COMMIT_ID}"
          fi
          echo "cov-report: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs/coverage.xml"
          wget -q --no-proxy \
            https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs/coverage.xml \
            --no-check-certificate -O coverage.xml
      - name: Fix coverage.xml paths
        if: steps.flag_check.outputs.should_upload == 'true'
        run: |
          echo "Before fix:"
          head -n 10 coverage.xml || true

          old_source=$(grep -oPm1 '(?<=<source>).*?(?=</source>)' coverage.xml || true)
          if [ -n "$old_source" ]; then
            echo "Replacing source '$old_source' with 'paddleformers'"
            sed -i "s|<source>$old_source</source>|<source>paddleformers</source>|g" coverage.xml
          else
            echo "No <source> found, injecting <source>paddleformers</source>"
            sed -i 's|<sources>|<sources>\n        <source>paddleformers</source>|' coverage.xml
          fi

          echo "After fix:"
          head -n 10 coverage.xml || true
      - name: Upload coverage to Codecov
        if: steps.flag_check.outputs.should_upload == 'true'
        uses: codecov/codecov-action@v4
        with:
          files: coverage.xml
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  upload-allure:
    name: upload-allure
    needs: [unittest-gpu-ce]
    if: always()
    runs-on: 'ernie-cpu'
    steps:
      - name: Checkout gh-pages
        uses: actions/checkout@v5
        with:
          ref: gh-pages
          path: gh-pages
        continue-on-error: true
      - name: Determine trigger mode && Restore history
        id: trigger_mode
        run: |
          MODE="pr"
          PR_NUMBER="${PR_ID}"
          mkdir -p result
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            MODE="manual"
            PR_NUMBER="manual"
            cp -r gh-pages/unit_CE/manual/history result/ || true
            destination_dir="unit_CE/manual"
          elif [[ "${{ github.event_name }}" == "schedule" ]]; then
            MODE="schedule"
            PR_NUMBER="schedule"
            cp -r gh-pages/unit_CE/schedule/history result/ || true
            destination_dir="unit_CE/schedule"
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            MODE="pr"
            PR_NUMBER="${PR_ID}"
            cp -r gh-pages/unit_CE/$PR_NUMBER/history result/ || true
            destination_dir="unit_CE/$PR_NUMBER"
          fi
          
          echo "mode=$MODE" >> "$GITHUB_OUTPUT"
          echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
          echo "destination_dir=$destination_dir" >> "$GITHUB_OUTPUT"
      - name: Download report
        id: download_report
        env:
          MODE: ${{ steps.trigger_mode.outputs.mode }}
          PR_NUMBER: ${{ steps.trigger_mode.outputs.pr_number }}
          RUN_ID: ${{ github.run_id }}
        run: |
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            bos_prefix="manual/$(date +%Y%m%d)_${{ github.run_id }}"
          elif [[ "${{ github.event_name }}" == "schedule" ]]; then
            bos_prefix="schedule/$(date +%Y%m%d)_${{ github.run_id }}"
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            bos_prefix="${PR_ID}/${COMMIT_ID}"
          fi
          echo "report: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs/report.tar.gz"
          wget -q --no-proxy \
            https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/ce-unittest-gpu/${bos_prefix}/logs/report.tar.gz \
            --no-check-certificate
          tar -xzf report.tar.gz
          if [ -d "report" ]; then
            echo "report directory exists"
            rm -rf report/data/attachments/*
          else
            echo "ERROR: report directory not found!"
            exit 1
          fi
      - name: Deploy to GitHub Pages
        if: always()
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./report
          destination_dir: ${{ steps.trigger_mode.outputs.destination_dir }}
      - name: Display GitHub Pages URL
        if: always()
        run: |
          DEST_DIR="${{ steps.trigger_mode.outputs.destination_dir }}"
          PAGES_URL="https://paddlepaddle.github.io/PaddleFormers/${DEST_DIR}/"
          echo "GitHub Allure Pages URL: ${PAGES_URL}"

          echo "## 🚀 Allure Report" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Item | Value |" >> $GITHUB_STEP_SUMMARY
          echo "|---|---|" >> $GITHUB_STEP_SUMMARY
          echo "| Workflow | ${{ github.workflow }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Mode | ${{ steps.trigger_mode.outputs.mode }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Time | $(date +%Y%m%d) |" >> $GITHUB_STEP_SUMMARY
          echo "| Report | [Open Report](${PAGES_URL}) |" >> $GITHUB_STEP_SUMMARY