|
| 1 | +name: Coverage Combine |
| 2 | +description: "Coverage Combine And Check" |
| 3 | + |
| 4 | +on: |
| 5 | + workflow_call: |
| 6 | + inputs: |
| 7 | + DOCKER_IMAGE: |
| 8 | + description: "Build Images" |
| 9 | + required: true |
| 10 | + type: string |
| 11 | + default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310" |
| 12 | + FASTDEPLOY_ARCHIVE_URL: |
| 13 | + description: "URL of the compressed FastDeploy code archive." |
| 14 | + required: true |
| 15 | + type: string |
| 16 | + GPU_COV_FILE_URL: |
| 17 | + description: "URL of the compressed GPU Coverage Data archive." |
| 18 | + required: true |
| 19 | + type: string |
| 20 | + XPU_COV_FILE_URL: |
| 21 | + description: "URL of the compressed GPU Coverage Data archive." |
| 22 | + required: true |
| 23 | + type: string |
| 24 | + CACHE_DIR: |
| 25 | + description: "Cache Dir Use" |
| 26 | + required: false |
| 27 | + type: string |
| 28 | + default: "" |
| 29 | + secrets: |
| 30 | + github-token: |
| 31 | + required: true |
| 32 | + |
| 33 | + |
| 34 | +jobs: |
| 35 | + coverage_combine: |
| 36 | + name: Coverage Combine And Check |
| 37 | + env: |
| 38 | + docker_image: ${{ inputs.DOCKER_IMAGE }} |
| 39 | + fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }} |
| 40 | + gpu_cov_file_url: ${{ inputs.GPU_COV_FILE_URL }} |
| 41 | + xpu_cov_file_url: ${{ inputs.XPU_COV_FILE_URL }} |
| 42 | + IS_PR: ${{ github.event_name == 'pull_request' }} |
| 43 | + runs-on: [self-hosted, CovCombine] |
| 44 | + outputs: |
| 45 | + diff_cov_file_url: ${{ steps.cov_upload.outputs.diff_cov_file_url }} |
| 46 | + diff_cov_result_json_url: ${{ steps.cov_upload.outputs.diff_cov_result_json_url }} |
| 47 | + steps: |
| 48 | + - name: coverage file download and combine |
| 49 | + shell: bash |
| 50 | + run: | |
| 51 | + set -x |
| 52 | + REPO="https://github.com/${{ github.repository }}.git" |
| 53 | + FULL_REPO="${{ github.repository }}" |
| 54 | + REPO_NAME="${FULL_REPO##*/}" |
| 55 | + BASE_BRANCH="${{ github.base_ref }}" |
| 56 | + docker pull ${docker_image} |
| 57 | +
|
| 58 | + CACHE_DIR="${CACHE_DIR:-$(dirname "$(dirname "${{ github.workspace }}")")}" |
| 59 | + echo "CACHE_DIR is set to ${CACHE_DIR}" |
| 60 | + if [ ! -f "${CACHE_DIR}/gitconfig" ]; then |
| 61 | + touch "${CACHE_DIR}/gitconfig" |
| 62 | + fi |
| 63 | +
|
| 64 | + # Clean the repository directory before starting |
| 65 | + docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ |
| 66 | + -v "${CACHE_DIR}/gitconfig:/etc/gitconfig:ro" \ |
| 67 | + -e "REPO_NAME=${REPO_NAME}" \ |
| 68 | + -e "fd_archive_url=${fd_archive_url}" \ |
| 69 | + -e "gpu_cov_file_url=${gpu_cov_file_url}" \ |
| 70 | + -e "xpu_cov_file_url=${xpu_cov_file_url}" \ |
| 71 | + -e "BASE_REF=${BASE_REF}" \ |
| 72 | + -e "IS_PR=${IS_PR}" \ |
| 73 | + ${docker_image} /bin/bash -c ' |
| 74 | + if [ -d ${REPO_NAME} ]; then |
| 75 | + echo "Directory ${REPO_NAME} exists, removing it..." |
| 76 | + rm -rf ${REPO_NAME}* |
| 77 | + fi |
| 78 | +
|
| 79 | + wget -q ${fd_archive_url} |
| 80 | + tar -xf FastDeploy.tar.gz |
| 81 | + rm -rf FastDeploy.tar.gz |
| 82 | + cd FastDeploy |
| 83 | + git config --global --add safe.directory /workspace/FastDeploy |
| 84 | + git config --global user.name "FastDeployCI" |
| 85 | + git config --global user.email "fastdeploy_ci@example.com" |
| 86 | +
|
| 87 | + git log -n 3 --oneline |
| 88 | +
|
| 89 | + git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt |
| 90 | + mkdir coveragedata |
| 91 | + if [ -z "${gpu_cov_file_url}" ]; then |
| 92 | + echo "No diff coverage file URL provided." |
| 93 | + else |
| 94 | + wget -q ${gpu_cov_file_url} |
| 95 | + gpu_cov_file=$(basename "$gpu_cov_file_url") |
| 96 | + tar -xf ${gpu_cov_file} -C coveragedata |
| 97 | + fi |
| 98 | +
|
| 99 | + if [ -z "${xpu_cov_file_url}" ]; then |
| 100 | + echo "No diff coverage file URL provided." |
| 101 | + else |
| 102 | + wget -q ${xpu_cov_file_url} |
| 103 | + xpu_cov_file=$(basename "$xpu_cov_file_url") |
| 104 | + tar -xf ${xpu_cov_file} -C coveragedata |
| 105 | + fi |
| 106 | + export COVERAGE_FILE=coveragedata/.coverage |
| 107 | + coverage combine coveragedata/ |
| 108 | + coverage report --ignore-errors |
| 109 | + coverage xml -o python_coverage_all.xml --ignore-errors |
| 110 | + COVERAGE_EXIT_CODE=0 |
| 111 | + if [[ "$IS_PR" == "true" ]]; then |
| 112 | + diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9 |
| 113 | + python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml |
| 114 | + else |
| 115 | + echo "Not a PR, skipping diff-cover" |
| 116 | + fi |
| 117 | + echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env |
| 118 | + ' |
| 119 | + if [ -f FastDeploy/exit_code.env ]; then |
| 120 | + cat FastDeploy/exit_code.env >> $GITHUB_ENV |
| 121 | + fi |
| 122 | + - name: Upload unit resule and diff coverage to bos |
| 123 | + id: cov_upload |
| 124 | + shell: bash |
| 125 | + run: | |
| 126 | + cd FastDeploy |
| 127 | + commit_id=${{ github.event.pull_request.head.sha }} |
| 128 | + pr_num=${{ github.event.pull_request.number }} |
| 129 | + target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/Combine |
| 130 | + wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py -O bos_tools.py |
| 131 | + push_file=$(realpath bos_tools.py) |
| 132 | + python -m pip install bce-python-sdk==0.9.29 |
| 133 | + diff_cov_file="diff_coverage.xml" |
| 134 | + if [ -f ${diff_cov_file} ];then |
| 135 | + python ${push_file} ${diff_cov_file} ${target_path}/CoverageData |
| 136 | + target_path_stripped="${target_path#paddle-github-action/}" |
| 137 | + DIFF_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_file} |
| 138 | + echo "diff_cov_file_url=${DIFF_COV_FILE_URL}" >> $GITHUB_OUTPUT |
| 139 | + echo "diff_cov_file_url=${DIFF_COV_FILE_URL}" >> $GITHUB_ENV |
| 140 | + export diff_cov_file_url=${DIFF_COV_FILE_URL} |
| 141 | + fi |
| 142 | + diff_cov_result_json="diff_coverage.json" |
| 143 | + if [ -f ${diff_cov_result_json} ];then |
| 144 | + python ${push_file} ${diff_cov_result_json} ${target_path}/CoverageData |
| 145 | + target_path_stripped="${target_path#paddle-github-action/}" |
| 146 | + DIFF_COV_JSON_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_result_json} |
| 147 | + echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_OUTPUT |
| 148 | + echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_ENV |
| 149 | + export diff_cov_result_json_url=${DIFF_COV_JSON_URL} |
| 150 | + fi |
| 151 | +
|
| 152 | + if [ "$COVERAGE_EXIT_CODE" -eq 9 ]; then |
| 153 | + echo "Coverage generation failed (exit code 9)" |
| 154 | + filename=$(basename "$diff_cov_result_json_url") |
| 155 | + if [ -z "${diff_cov_result_json_url}" ]; then |
| 156 | + echo "No diff cov result file URL provided." |
| 157 | + else |
| 158 | + rm -rf "${filename}" |
| 159 | + wget -O ${filename} ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..." |
| 160 | + fi |
| 161 | + if [ -f "${filename}" ];then |
| 162 | + echo "Failed test cases:" |
| 163 | + if command -v jq >/dev/null 2>&1; then |
| 164 | + jq . "${filename}" |
| 165 | + else |
| 166 | + cat "${filename}" |
| 167 | + fi |
| 168 | + fi |
| 169 | + exit "$COVERAGE_EXIT_CODE" |
| 170 | + fi |
| 171 | + echo "coverage passed" |
| 172 | + exit 0 |
| 173 | +
|
| 174 | + diff_coverage_report: |
| 175 | + needs: coverage_combine |
| 176 | + if: always() |
| 177 | + runs-on: ubuntu-latest |
| 178 | + env: |
| 179 | + fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }} |
| 180 | + steps: |
| 181 | + - name: coverage diff file download |
| 182 | + shell: bash |
| 183 | + env: |
| 184 | + diff_cov_file_url: ${{ needs.coverage_combine.outputs.diff_cov_file_url }} |
| 185 | + run: | |
| 186 | + mkdir FastDeploy |
| 187 | + cd FastDeploy |
| 188 | + if [ -z "${diff_cov_file_url}" ]; then |
| 189 | + echo "No diff coverage file URL provided." |
| 190 | + exit 0 |
| 191 | + fi |
| 192 | + wget "${diff_cov_file_url}" -O ./diff_coverage.xml || echo "Download cov file failed, but continuing..." |
| 193 | + - name: Upload diff coverage report |
| 194 | + if: ${{ needs.coverage_combine.outputs.diff_cov_file_url != null && needs.coverage_combine.outputs.diff_cov_file_url != '' }} |
| 195 | + uses: codecov/codecov-action@v5 |
| 196 | + with: |
| 197 | + files: ./FastDeploy/diff_coverage.xml |
| 198 | + name: python diff coverage |
| 199 | + verbose: true |
| 200 | + flags: diff |
0 commit comments