ce_job.yml

name: CE Compile Job

on:
  workflow_dispatch:
  push:
    branches:
      - develop
      - 'release/*'
permissions: read-all

concurrency:
  group: CE-Job-${{ github.ref }}-${{ github.sha }}
  cancel-in-progress: true

jobs:
  ce_job_pre_check:
    runs-on: ubuntu-latest
    env:
      COMPILE_BRANCH: ${{ vars.COMPILE_BRANCH }}
      CE_COMPILE_SELECTION: ${{ vars.CE_COMPILE_SELECTION }}
      COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ vars.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }}
    outputs:
      branch_match: ${{ steps.set_output.outputs.branch_match }}
      compile_use_paddle_whl_url: ${{ steps.set_output.outputs.compile_use_paddle_whl_url }}
      sm8689_match: ${{ steps.set_output.outputs.sm8689_match }}
      sm8090_match: ${{ steps.set_output.outputs.sm8090_match }}

    steps:
      - name: Set Version
        id: set_output
        env:
          COMPILE_BRANCH: ${{ env.COMPILE_BRANCH }}
          CE_COMPILE_SELECTION: ${{ env.CE_COMPILE_SELECTION }}
          COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ env.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }}
          GITHUB_REF_NAME: ${{ github.ref_name }}
        run: |
          # 选择要触发编译任务的分支 done
          # 选择指定分支要编译的任务 8090或者8689
          # 指定分支编译要使用的Paddle的安装包,默认使用nightly最新的

          IFS=',' read -ra BRANCHES <<< "$COMPILE_BRANCH"
          MATCH=false
          for b in "${BRANCHES[@]}"; do
            if [[ "$b" == "${GITHUB_REF_NAME}" ]]; then
              MATCH=true
              break
            fi
          done
          echo "branch_match=$MATCH" >> $GITHUB_OUTPUT

          # 通过变量CE_COMPILE_SELECTION中的映射关系,决定分支是编译sm8090还是sm8689
          for pair in $(echo "$CE_COMPILE_SELECTION" | tr ';' ' '); do
            branch=$(echo "$pair" | cut -d',' -f1)
            compile_task_list=$(echo "$pair" | cut -d',' -f2)

            if [[ "$branch" == "$GITHUB_REF_NAME" ]]; then

              # 判断里面是否包含 sm8090 或 sm8689
              if [[ "$compile_task_list" == *"sm8090"* ]]; then
                echo "sm8090_match=true" >> $GITHUB_OUTPUT
              fi
              if [[ "$compile_task_list" == *"sm8689"* ]]; then
                echo "sm8689_match=true" >> $GITHUB_OUTPUT
              fi
              break
            fi
          done

          # 通过变量COMPILE_USE_PADDLE_WHL_URL_MAPPINGS中的映射关系,决定是否是安装指定版本的Paddle还是直接安装URL
          for pair in $(echo $COMPILE_USE_PADDLE_WHL_URL_MAPPINGS | tr ';' ' '); do
            branch=$(echo "$pair" | cut -d',' -f1)
            paddle_whl_url=$(echo "$pair" | cut -d',' -f2)
            if [[ "$branch" == "${{ github.ref_name }}" ]]; then
              FOUND_PADDLE_URL="$paddle_whl_url"
              echo "compile_use_paddle_whl_url=${FOUND_PADDLE_URL}" >> $GITHUB_OUTPUT
              break
            fi
          done

  print_ce_job_pre_check_outputs:
    runs-on: ubuntu-latest
    needs: ce_job_pre_check
    steps:
      - name: Print outputs as JSON
        run: |
          echo '${{ toJSON(needs.ce_job_pre_check.outputs) }}'


  clone:
    environment: CodeSync
    name: FD-Clone-Linux
    runs-on: ubuntu-latest
    needs: ce_job_pre_check
    if: ${{ needs.ce_job_pre_check.outputs.branch_match == 'true' }}
    outputs:
      repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }}
    steps:
      - name: Clone FastDeploy
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event_name == 'pull_request'
                && github.event.pull_request.base.ref
                || github.ref_name }}
          submodules: 'recursive'
          fetch-depth: 1000

      - name: Python Setup
        uses: actions/setup-python@v6
        with:
          python-version: '3.10'
      - name: Code Info Show and Upload
        id: set_output
        env:
          AK: ${{ secrets.BOS_AK }}
          SK: ${{ secrets.BOS_SK }}
        run: |
          git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
          echo "Current HEAD Log:"
          git log --oneline -n 5
          ls
          cd ..
          tar -zcf FastDeploy.tar.gz FastDeploy
          commit_id=${{ github.sha }}
          branch_name=${{ github.ref_name }}

          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29

          filename="FastDeploy.tar.gz"
          target_paths=(
            "paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id}"
            "paddle-qa/BRANCH/FastDeploy/${branch_name}/latest"
          )

          for target_path in "${target_paths[@]}"; do
            echo "Uploading ${filename} to ${target_path}"
            python "${push_file}" "${filename}" "${target_path}"
          done

          base_prefix="paddle-qa/"

          commit_path_stripped="${target_paths[0]#${base_prefix}}"
          latest_path_stripped="${target_paths[1]#${base_prefix}}"

          REPO_ARCHIVE_URL="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}"
          LATEST_REPO_ARCHIVE_URL="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}"

          echo "commit archive url is ${REPO_ARCHIVE_URL}"
          echo "latest archive url is ${LATEST_REPO_ARCHIVE_URL}"

          echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT

  resultshow:
    name: Show Code Archive Output
    needs: clone
    runs-on: ubuntu-latest
    steps:
      - name: Print repo_archive_url path
        run: |
          echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}"

  build_sm8090:
    name: BUILD_SM8090
    needs: [clone, ce_job_pre_check]
    if: ${{ needs.ce_job_pre_check.outputs.sm8090_match == 'true' }}
    uses: ./.github/workflows/_build_linux.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "80,90"
      WITH_NIGHTLY_BUILD: OFF
      FD_VERSION: 0.0.0
      PADDLE_WHL_URL: ${{ needs.ce_job_pre_check.outputs.compile_use_paddle_whl_url }}
    secrets:
      github-token: ${{ secrets.GITHUB_TOKEN }}

  build_sm8090_rl:
    name: BUILD_SM8090_RL
    needs: [clone, ce_job_pre_check]
    if: ${{ needs.ce_job_pre_check.outputs.sm8090_match == 'true' }}
    uses: ./.github/workflows/_build_linux_rl.yml
    with:
      DOCKER_IMAGE: iregistry.baidu-int.com/new_rl_infra/base-images:paddlecloud-ubuntu24.04-gcc13.3-cuda12.9-cudnn9.9-bccl1.4.1.4-nccl2.26.5-openmpi4.1.5-FleetY13.0.0-v2.4.0-rc1
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "80,90"
      WITH_NIGHTLY_BUILD: OFF
      FD_VERSION: 0.0.0
      PADDLE_WHL_URL: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Training-Linux-Gpu-Cuda12.9-Cudnn9.9-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl

  build_sm90_rl:
    name: BUILD_SM90_RL
    needs: [clone, ce_job_pre_check]
    if: ${{ needs.ce_job_pre_check.outputs.sm8090_match == 'true' }}
    uses: ./.github/workflows/_build_linux_rl.yml
    with:
      DOCKER_IMAGE: iregistry.baidu-int.com/paddlepaddle/base-images:paddlecloud-ubuntu24.04-gcc13.3-cuda12.9-cudnn9.9-bccl1.4.1.4-nccl2.26.5-openmpi4.1.5-FleetY13.4.1H-rc2
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "90"
      WITH_NIGHTLY_BUILD: OFF
      FD_VERSION: 0.0.0

  build_sm8689:
    name: BUILD_SM8689
    needs: [clone, ce_job_pre_check]
    if: ${{ needs.ce_job_pre_check.outputs.sm8689_match == 'true' }}
    uses: ./.github/workflows/_build_linux.yml
    with:
      DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux
      FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
      COMPILE_ARCH: "86,89"
      WITH_NIGHTLY_BUILD: OFF
      FD_VERSION: 0.0.0
      PADDLE_WHL_URL: ${{ needs.ce_job_pre_check.outputs.compile_use_paddle_whl_url }}
    secrets:
      github-token: ${{ secrets.GITHUB_TOKEN }}

  ce_upload_sm8090:
    environment: CodeSync
    name: CE_UPLOAD
    needs: build_sm8090
    runs-on: ubuntu-latest
    env:
      AK: ${{ secrets.BOS_AK }}
      SK: ${{ secrets.BOS_SK }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }}
      COMPILE_ARCH: "80,90"
    steps:
      - uses: actions/setup-python@v6
        with:
          python-version: '3.10'
      - name: Wheel Info Show and Upload
        run: |
          echo "The wheel is located at: ${{ needs.build_sm8090.outputs.wheel_path }}"
          wget -q --no-check-certificate ${{ needs.build_sm8090.outputs.wheel_path }}
          filename=$(basename ${{ needs.build_sm8090.outputs.wheel_path }})

          commit_id=${{ github.sha }}
          branch_name=${{ github.ref_name }}

          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29

          target_paths=(
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/${commit_id}"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_80/${branch_name}/${commit_id}"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_90/${branch_name}/${commit_id}"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/latest"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_80/${branch_name}/latest"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_90/${branch_name}/latest"
          )

          for target_path in "${target_paths[@]}"; do
            echo "Uploading ${filename} to ${target_path}"
            python "${push_file}" "${filename}" "${target_path}"
          done

          base_prefix="paddle-qa/"
          commit_path_stripped="${target_paths[0]#${base_prefix}}"
          latest_path_stripped="${target_paths[3]#${base_prefix}}"
          WHEEL_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}"
          WHEEL_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}"

          echo "commit wheel url is ${WHEEL_PATH}"
          echo "latest wheel url is ${WHEEL_PATH_LATEST}"

  ce_upload_sm8090_rl:
    environment: CodeSync
    name: CE_UPLOAD_RL
    needs: build_sm8090_rl
    runs-on: ubuntu-latest
    env:
      AK: ${{ secrets.BOS_AK }}
      SK: ${{ secrets.BOS_SK }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090_rl.outputs.wheel_path_rl }}
      COMPILE_ARCH: "80,90"
    steps:
      - uses: actions/setup-python@v6
        with:
          python-version: '3.10'
      - name: Wheel Info Show and Upload
        run: |
          echo "The wheel is located at: ${{ needs.build_sm8090_rl.outputs.wheel_path_rl }}"
          wget -q --no-check-certificate ${{ needs.build_sm8090_rl.outputs.wheel_path_rl }}
          filename=$(basename ${{ needs.build_sm8090_rl.outputs.wheel_path_rl }})

          commit_id=${{ github.sha }}
          branch_name=${{ github.ref_name }}

          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29

          target_paths=(
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE_RL/cu129/SM_8090/${branch_name}/${commit_id}"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE_RL/cu129/SM_8090/${branch_name}/latest"
          )

          for target_path in "${target_paths[@]}"; do
            echo "Uploading ${filename} to ${target_path}"
            python "${push_file}" "${filename}" "${target_path}"
          done

          base_prefix="paddle-qa/"
          commit_path_stripped="${target_paths[0]#${base_prefix}}"
          latest_path_stripped="${target_paths[1]#${base_prefix}}"
          WHEEL_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}"
          WHEEL_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}"

          echo "commit wheel url is ${WHEEL_PATH}"
          echo "latest wheel url is ${WHEEL_PATH_LATEST}"

  ce_upload_sm90_rl:
    environment: CodeSync
    name: CE_UPLOAD_90RL
    needs: build_sm90_rl
    runs-on: ubuntu-latest
    env:
      AK: ${{ secrets.BOS_AK }}
      SK: ${{ secrets.BOS_SK }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm90_rl.outputs.wheel_path_rl }}
      COMPILE_ARCH: "90,100"
    steps:
      - uses: actions/setup-python@v6
        with:
          python-version: '3.10'
      - name: Wheel Info Show and Upload
        run: |
          echo "The wheel is located at: ${{ needs.build_sm90_rl.outputs.wheel_path_rl }}"
          wget -q --no-check-certificate ${{ needs.build_sm90_rl.outputs.wheel_path_rl }}
          filename=$(basename ${{ needs.build_sm90_rl.outputs.wheel_path_rl }})

          commit_id=${{ github.sha }}
          branch_name=${{ github.ref_name }}

          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29

          target_paths=(
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE_RL/cu129/SM_90/${branch_name}/${commit_id}"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE_RL/cu129/SM_90/${branch_name}/latest"
          )

          for target_path in "${target_paths[@]}"; do
            echo "Uploading ${filename} to ${target_path}"
            python "${push_file}" "${filename}" "${target_path}"
          done

          base_prefix="paddle-qa/"
          commit_path_stripped="${target_paths[0]#${base_prefix}}"
          latest_path_stripped="${target_paths[1]#${base_prefix}}"
          WHEEL_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}"
          WHEEL_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}"

          echo "commit wheel url is ${WHEEL_PATH}"
          echo "latest wheel url is ${WHEEL_PATH_LATEST}"

  ce_upload_sm8689:
    environment: CodeSync
    name: CE_UPLOAD
    needs: build_sm8689
    runs-on: ubuntu-latest
    env:
      AK: ${{ secrets.BOS_AK }}
      SK: ${{ secrets.BOS_SK }}
      FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8689.outputs.wheel_path }}
      COMPILE_ARCH: "86,89"
    steps:
      - uses: actions/setup-python@v6
        with:
          python-version: '3.10'
      - name: Wheel Info Show and Upload
        run: |
          echo "The wheel is located at: ${{ needs.build_sm8689.outputs.wheel_path }}"
          wget -q --no-check-certificate ${{ needs.build_sm8689.outputs.wheel_path }}
          filename=$(basename ${{ needs.build_sm8689.outputs.wheel_path }})

          commit_id=${{ github.sha }}
          branch_name=${{ github.ref_name }}

          wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
          push_file=$(realpath bos_tools.py)
          python -m pip install bce-python-sdk==0.9.29

          target_paths=(
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/${commit_id}"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_86/${branch_name}/${commit_id}"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_89/${branch_name}/${commit_id}"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/latest"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_86/${branch_name}/latest"
            "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_89/${branch_name}/latest"
          )

          for target_path in "${target_paths[@]}"; do
            echo "Uploading ${filename} to ${target_path}"
            python "${push_file}" "${filename}" "${target_path}"
          done

          base_prefix="paddle-qa/"
          commit_path_stripped="${target_paths[0]#${base_prefix}}"
          latest_path_stripped="${target_paths[3]#${base_prefix}}"
          WHEEL_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}"
          WHEEL_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}"

          echo "commit wheel url is ${WHEEL_PATH}"
          echo "latest wheel url is ${WHEEL_PATH_LATEST}"