deploy-test.yml

name: Deploy Model Test

on:
  workflow_dispatch:

# If there is a new commit, the previous jobs will be canceled
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true
env:
  OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/script/models
  SCRIPT_PATH: /intel-extension-for-transformers/.github/workflows/script
  DOCKER_CONFIG_NAME: "commonDockerConfig"
  REPO_NAME: "intel-extension-for-transformers"
  REPO_TAG: "py38"
  DOCKER_FILE_NAME: "devel"
  CONTAINER_NAME: "modelTest"
  EXTRA_CONTAINER_NAME: "utTest"
  EXTRA_CONTAINER_NAME2: "codeScan"

permissions:
  contents: read

jobs:
  Deploy-Workflow:
    permissions:
      actions: read  # for dawidd6/action-download-artifact to query and download artifacts
      contents: read  # for actions/checkout to fetch code
      pull-requests: read  # for dawidd6/action-download-artifact to query commit hash
    runs-on: itrex-node
    strategy:
      matrix:
        include:
          - modelName: "bert_mini_sparse"
            framework: "engine"
            mode: "accuracy,performance"
            precision: "int8"
          - modelName: "distilbert_base_squad_ipex"
            framework: "ipex"
            mode: "accuracy,performance"
            precision: "int8"
    steps:
      - name: Docker Clean Up
        run: |
          docker ps -a
          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
              docker start ${{ env.CONTAINER_NAME }}
              echo "remove left files through container ..."
              docker exec ${{ env.CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
          fi
          docker ps -a
          if [[ $(docker ps -a | grep -i '${{ env.EXTRA_CONTAINER_NAME }}'$) ]]; then
              docker start ${{ env.EXTRA_CONTAINER_NAME }}
              echo "remove left files through container ..."
              docker exec ${{ env.EXTRA_CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
          fi

      - name: Checkout out Repo
        uses: actions/checkout@v4
        with:
          submodules: "recursive"
          fetch-tags: true
      # We need this because GitHub needs to clone the branch to pipeline
      - name: Docker Build
        run: |
          docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} .

      - name: Docker Run
        run: |
          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
            docker stop ${{ env.CONTAINER_NAME }}
            docker rm -vf ${{ env.CONTAINER_NAME }} || true
          fi
          docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
          -v ${{ github.workspace }}:/intel-extension-for-transformers \
          -v ~/.cache/oneAPI:/cache \
          -v /dataset/cache/huggingface:/root/.cache/huggingface \
          ${{ env.REPO_NAME }}:${{ env.REPO_TAG }}

      - name: Binary build
        run: |
          docker exec ${{ env.CONTAINER_NAME }} \
          bash -c "cd /intel-extension-for-transformers/.github/workflows/script \
          && bash install_binary.sh"

      - name: Download Reference Artifact
        id: download-artifact
        uses: dawidd6/action-download-artifact@v3.1.2
        with:
          workflow: deploy-test.yml
          name: ${{ matrix.framework }}-${{ matrix.modelName }}
          run_id: ${{ vars.DEPLOY_REF_ID }}
          path: ${{ github.workspace }}/${{ matrix.framework }}_${{ matrix.modelName }}_refer_log
          name_is_regexp: true
          repo: ${{ github.repository }}
          check_artifacts: false
          search_artifacts: false
          skip_unpack: false
          if_no_artifact_found: warn

      - name: Display structure of downloaded files
        run: ls -R

      - name: FP32 Benchmark
        run: |
          docker exec ${{ env.CONTAINER_NAME }} \
          bash -c "cd /intel-extension-for-transformers/.github/workflows/script/models \
          && bash run_deploy.sh --model=${{ matrix.modelName }} --framework=${{ matrix.framework }} --mode=${{ matrix.mode }} --precision=fp32 --PERF_STABLE_CHECK=${{ vars.PERF_STABLE_CHECK }}"

      - name: INT8 Benchmark
        run: |
          docker exec ${{ env.CONTAINER_NAME }} \
          bash -c "cd /intel-extension-for-transformers/.github/workflows/script/models \
          && bash run_deploy.sh --model=${{ matrix.modelName }} --framework=${{ matrix.framework }} --mode=${{ matrix.mode }} --precision=int8 --PERF_STABLE_CHECK=${{ vars.PERF_STABLE_CHECK }}"

      - name: Collect Log
        run: |
          docker exec ${{ env.CONTAINER_NAME }} \
          bash -c "cd /intel-extension-for-transformers/.github/workflows/script/models \
          && python collect_model_log.py --model=${{ matrix.modelName }} \
           --framework=${{ matrix.framework }} \
           --logs_dir=/intel-extension-for-transformers/${{matrix.framework}}_${{matrix.modelName}} \
           --output_dir=/intel-extension-for-transformers/${{matrix.framework}}_${{matrix.modelName}} \
           --build_id=${{ github.run_id }} \
           --model_test_type=deploy"

      - name: Publish pipeline artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: ${{ matrix.framework }}-${{ matrix.modelName }}
          path: ${{ github.workspace }}/${{ matrix.framework }}_${{ matrix.modelName }}
          if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
          retention-days: 60 # 1 <= retention-days <= 90

  Genreate-Report:
    permissions:
      actions: read  # for dawidd6/action-download-artifact to query and download artifacts
      contents: read  # for actions/checkout to fetch code
      pull-requests: read  # for dawidd6/action-download-artifact to query commit hash
    runs-on: itrex-node-spell
    needs: [Deploy-Workflow]
    steps:
      - name: Docker Clean Up
        run: |
          docker ps -a
          if [[ $(docker ps -a | grep -i '${{ env.EXTRA_CONTAINER_NAME2 }}-${{ runner.name }}'$) ]]; then
              docker start ${{ env.EXTRA_CONTAINER_NAME2 }}-${{ runner.name }}
              echo "remove left files through container ..."
              docker exec ${{ env.EXTRA_CONTAINER_NAME2 }}-${{ runner.name }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
          fi
      - name: Checkout out Repo
        uses: actions/checkout@v4

      - name: Download Summary Log
        uses: actions/download-artifact@v4
        with:
          path: ${{ env.OUT_SCRIPT_PATH }}/log

      - name: Analysis Summary
        run: |
          cd ${{ env.OUT_SCRIPT_PATH }}
          mkdir generated
          pip install requests
          python summary.py --logs_dir ${{ env.OUT_SCRIPT_PATH }}/log --output_dir=generated

      - name: Download Reference Artifact
        id: download-artifact
        uses: dawidd6/action-download-artifact@v3.1.2
        with:
          workflow: deploy-test.yml
          name: FinalReport
          run_id: ${{ vars.DEPLOY_REF_ID }}
          path: ${{ env.OUT_SCRIPT_PATH }}
          name_is_regexp: true
          repo: ${{ github.repository }}
          check_artifacts: false
          search_artifacts: false
          skip_unpack: false
          if_no_artifact_found: warn

      - name: Display structure of downloaded files
        run: cd ${{ env.OUT_SCRIPT_PATH }}/log && ls -R

      - name: Generate report
        run: |
          echo "------ Generating final report.html ------"
          cd ${{ env.OUT_SCRIPT_PATH }}
          /usr/bin/bash generate_report.sh --workflow=deploy
        env:
          RUN_DISPLAY_URL: https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
          BUILD_NUMBER: ${{ github.run_id }}
          JOB_STATUS: succeed
          MR_source_branch: ${{ github.event.pull_request._links.html.href }}
          ghprbActualCommit: ${{ github.event.pull_request.head.sha }}

      - name: Publish Report
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: FinalReport
          path: ${{ env.OUT_SCRIPT_PATH }}/generated

      - name: Specify performance regression
        if: ${{ !cancelled() }}
        run: |
          if [ ${{ env.is_perf_reg }} == 'true' ]; then
            echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports."
            exit 1
          fi