utils.yaml

# Define workflow name.
name: MatrixOne Utils CI

on:
  workflow_call:
    secrets:
      TOKEN_ACTION:
        description: "A token passed from the caller workflow"
        required: true
      TEST_S3FS_ALIYUN:
        description: "TEST_S3FS_ALIYUN For UT Test"
        required: true
      TEST_S3FS_QCLOUD:
        description: "TEST_S3FS_QCLOUD For UT Test"
        required: true
      DOCU_GROUP_HOOK:
        description: "DOCU_GROUP_HOOK For Notice"
        required: true
      OPENAI_API_KEY:
        description: "OPENAI_API_KEY For get PR Suggestion"
        required: false
      TENCENT_AK:
        description: "TENCENT_AK For UT Test"
        required: true
      TENCENT_SK:
        description: "TENCENT_SK For UT Test"
        required: true
      EE_DB_ADDR:
        description: "matrixone cloud host"
        required: true
      EE_DB_PORT:
        description: "matrixone cloud port"
        required: true
      EE_DB_USER:
        description: "matrixone cloud user"
        required: true
      EE_DB_PASSWORD:
        description: "matrixone cloud password"
        required: true
      EE_DB_DB:
        description: "matrixone cloud db"
        required: true

jobs:
  check_organization_user:
    environment: ci
    runs-on: ubuntu-22.04
    outputs:
      in_org: ${{ steps.check_in_org.outputs.in_org }}
      safe_label: ${{ steps.check_safe_label.outputs.safe_label }}
    steps:
      - id: check_in_org
        name: CHECK ORGANIZATION USER
        run: |
          PAGE=1;
          PER_PAGE=100;
          USER_IN_ORG="0";
          while true; do
            users=$(curl -L \
              -H "Accept: application/vnd.github+json" \
              -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \
              -H "X-GitHub-Api-Version: 2022-11-28" \
              "https://api.github.com/orgs/matrixorigin/members?page=$PAGE&per_page=$PER_PAGE");
            if [ $(echo $users | jq ".[].login" | grep -c "${{ github.event.pull_request.user.login }}" ) -eq 1 ]; then
              USER_IN_ORG="1";
              break;
            fi
            if [ $(echo $users | jq ".[].login" | wc -l ) -eq $PER_PAGE ]; then
              PAGE=$(($PAGE+1));
              users="";
            else
              break;
            fi
          done
          echo "in_org=$USER_IN_ORG" >> $GITHUB_OUTPUT;

      - id: check_safe_label
        name: CHECK PULL REQUEST LABEL
        run: |
          labels=$(curl -L \
            -H "Accept: application/vnd.github+json" \
            -H "Authorization: Bearer ${{ secrets.TOKEN_ACTION }}" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            '${{ github.event.pull_request.url}}' | jq ".labels");
          if [ $(echo $labels | jq ".[].name" | grep -c "safe-to-test" ) -ge 1 ]; then
            echo "safe_label=1" >> $GITHUB_OUTPUT;
          else
            echo "safe_label=0" >> $GITHUB_OUTPUT;
          fi

  pr_coverage:
    if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }}
    name: Coverage
    needs: [check_organization_user]
    environment: ci
    # if you change runner, must modify L117
    runs-on: amd64-mo-guangzhou-2xlarge16
    steps:
      - uses: actions/checkout@v6
        with:
          token: ${{ secrets.TOKEN_ACTION }}
          fetch-depth: "1"
          path: ./matrixone
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
      - name: Clone test-tool repository
        uses: actions/checkout@v6
        with:
          repository: matrixorigin/mo-tester
          path: ./mo-tester
          ref: main

      - name: Set up Go and JAVA
        uses: matrixorigin/CI/actions/setup-env@main
        with:
          setup-java: false
          go-version-file: "${{ github.workspace }}/matrixone/go.mod"

      - name: Set JAVA PATH
        run: |
          echo "$JAVA_HOME/bin" >> $GITHUB_PATH

      - name: Generate diff.patch
        env:
          GH_TOKEN: ${{ secrets.TOKEN_ACTION }}
          IS_PUB_REPO: ${{ vars.IS_PUB_REPO }}
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          retry_times=5
          delay=3

          # If this is a public repo, fetch the PR diff directly from the .diff URL without token/API.
          if [ -n "${IS_PUB_REPO:-}" ] && [ "${IS_PUB_REPO}" != "0" ] && [ "${IS_PUB_REPO}" != "false" ]; then
            url="https://github.com/${{ github.event.pull_request.base.repo.full_name }}/pull/${{ github.event.pull_request.number }}.diff"
            for ((i = 1; i <= retry_times; i++)); do
              if curl -f --max-time 10 -L -o diff.patch "$url"; then
                exit 0
              fi
              echo "Get diff.patch from public URL failed...Retrying in ${delay} seconds..."
              sleep $delay
            done
            echo "Max retries reached (public URL). Failed."
            exit 1
          fi

          # Private repo or no IS_PUB_REPO flag: use GitHub API with TOKEN_ACTION.
          url="https://api.github.com/repos/${{ github.event.pull_request.base.repo.full_name }}/pulls/${{ github.event.pull_request.number }}"
          for ((i = 1; i <= retry_times; i++)); do
            if curl -f --max-time 10 -L \
              -H "Accept: application/vnd.github.v3.diff" \
              -H "Authorization: Bearer $GH_TOKEN" \
              -o diff.patch "$url"; then
              exit 0
            fi
            echo "Get diff.patch failed...Retrying in ${delay} seconds..."
            sleep $delay
          done
          echo "Max retries reached. Failed."
          exit 1


      - name: Set Variables
        run: |
          echo "ut_report=UT-Report.out" >> $GITHUB_ENV
          echo "raw_ut_coverage='ut_coverage.raw'" >> $GITHUB_ENV
          echo "raw_bvt_coverage='bvt_coverage.raw'" >> $GITHUB_ENV
          echo "ut_coverage='ut_coverage.out'" >> $GITHUB_ENV
          echo "bvt_coverage='bvt_coverage.out'" >> $GITHUB_ENV
          echo "ut_html_coverage='ut_coverage.html'" >> $GITHUB_ENV
          echo "bvt_html_coverage='bvt_coverage.html'" >> $GITHUB_ENV
          echo "ut_excluded_pkg='pkg/pb|pkg/sql/parsers/goyacc|yaccpar'" >> $GITHUB_ENV
          echo "bvt_excluded_pkg='pkg/pb|yaccpar'" >> $GITHUB_ENV
          echo 'GOPROXY=http://goproxy.goproxy.svc.cluster.local|https://goproxy.cn|direct' >> $GITHUB_ENV

#      - uses: matrixorigin/setup-hdfs@master
#        with:
#          hdfs-download-url: "3.4.1"
      - name: Add cn.txn
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          echo "[cn.txn]" >> ./etc/launch/cn.toml
          echo "  enable-leak-check = 1" >> ./etc/launch/cn.toml
          echo '  max-active-ages = "2m"'>> ./etc/launch/cn.toml
      - name: Start Unit Test
        timeout-minutes: 45
        run: |
          set -o pipefail
          cd $GITHUB_WORKSPACE/matrixone
          export TEST_S3FS_ALIYUN=${{ secrets.TEST_S3FS_ALIYUN }}
          export TEST_S3FS_QCLOUD=${{ secrets.TEST_S3FS_QCLOUD }}
          test_scope=$(go list ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog')
          make clean && make config && make cgo
          CGO_CFLAGS="-I${GITHUB_WORKSPACE}/matrixone/cgo"
          CGO_LDFLAGS="-L${GITHUB_WORKSPACE}/matrixone/cgo -lmo"
          THIRDPARTIES_DIR=$GITHUB_WORKSPACE/matrixone/thirdparties
          THIRDPARTIES_INSTALL_DIR=$THIRDPARTIES_DIR/install
          if [ -d $THIRDPARTIES_DIR ] ; then
            make thirdparties
            CGO_CFLAGS="${CGO_CFLAGS} -I${THIRDPARTIES_INSTALL_DIR}/include"
            CGO_LDFLAGS="${CGO_LDFLAGS} -L${THIRDPARTIES_INSTALL_DIR}/lib -lusearch_c -Wl,-rpath,${THIRDPARTIES_INSTALL_DIR}/lib -lm"
          fi

          echo "ut start"
          CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go test -json -short -v -tags matrixone_test -p 6 -covermode=set -coverprofile=${{ env.raw_ut_coverage }} -coverpkg=./... ${test_scope} > ${{ env.ut_report }}
          echo "ut finished"
      - name: Analysis Fail UT Cases
        if: ${{ failure() }}
        run: |
          go install github.com/matrixorigin/go-ut-analysis@latest
          go-ut-analysis test -f "$GITHUB_WORKSPACE/matrixone/${{ env.ut_report }}" --first 10 --report-path  "$GITHUB_WORKSPACE/ut-report" --stdout=false;
          if find ut-report > /dev/null 2>&1 && ! find ut-report/failed/outputs > /dev/null 2>&1; then
            logger "INF" "UNIT TESTING SUCCEEDED !!!"
          else
            logger "INF" "UNIT TESTING FAILED !!!"
            exit 1;
          fi
      - name: Parse Time for moc
        if: ${{ failure() }}
        id: time
        run: |
          export TZ='Asia/Shanghai'

          #utcTime="2024-11-15T08:33:52Z"
          action_time=${{ github.event.pull_request.head.repo.pushed_at }}
          ACTION_TIME=$(date -d "${action_time}" '+%Y-%m-%d %H:%M:%S')

          # 获取格式化的日期时间字符串
          time=$(date '+%Y-%m-%d %H:%M:%S')

          # action link
          action_link="https://github.com/${{ github.event.pull_request.base.repo.full_name }}/actions/runs/${{ github.run_id }}"

          echo "ACTION_TIME=${ACTION_TIME}"
          echo "START_TIME=${time}"
          echo "ACTION_LINK=${action_link}"

          echo "ACTION_TIME=${ACTION_TIME}" >> $GITHUB_OUTPUT
          echo "START_TIME=${time}" >> $GITHUB_OUTPUT
          echo "ACTION_LINK=${action_link}" >> $GITHUB_OUTPUT

      - name: Get failed ut cases
        if: ${{ failure() }}
        id: failed
        run: |
          cd $GITHUB_WORKSPACE;
          if [ "$(find ./ut-report/failed | wc -l)" -le "2" ]; then
            exit 0;
          fi
          find ./ut-report/failed -name result.txt -exec cat {} \;
          fail_ut_cases=''
          files=($(find ./ut-report/failed | grep -v result.txt))
          for file in "${files[@]}"; do
            if [ -d "$file" ]; then
              continue
            fi
            echo "============================================"
            cat "${file}";
            fail_ut_cases=${fail_ut_cases}$(basename "${file}")','
          done

          echo "steps.ut.conclusion: ${{ steps.ut.conclusion }}"
          echo "FAIL_UT_CASES=${fail_ut_cases}"
          echo "FAIL_UT_CASES=${fail_ut_cases}" >> $GITHUB_OUTPUT
      - name: Start BVT Test Service
        timeout-minutes: 10
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          sed -i.bak 's/level = "info"/level = "error"/g'  ./etc/launch/log.toml
          if  [ -d  "./store"  ]; then rm -rf ./store; fi
          export GOCOVERDIR=$GITHUB_WORKSPACE/matrixone
          make GOBUILD_OPT=-cover build
          ./optools/run_bvt.sh $GITHUB_WORKSPACE/matrixone launch
      - name: Update BVT SQL Timeout
        run: |
          cd $GITHUB_WORKSPACE/mo-tester
          sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml
      - name: Start BVT Test
        timeout-minutes: 30
        id: bvt_on_pr_version
        run: |
          export LC_ALL="C.UTF-8"
          locale

          cd $GITHUB_WORKSPACE/mo-tester
          sed -i 's/  port: [0-9]*/  port: 12345/g' mo.yml
          cat mo.yml
          echo "============================"

          ./run.sh -n -g -o -p $GITHUB_WORKSPACE/matrixone/test/distributed/cases -e optimistic 2>&1

      - name: Generate UT and BVT Coverage File
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 10
        run: |
          set -uo pipefail
          set -ex
          cd $GITHUB_WORKSPACE/matrixone
          ls -lh;
          echo "====================================="
          echo ""
          
          grep -Ev ${{ env.ut_excluded_pkg }} ${{ env.raw_ut_coverage }} > ${{ env.ut_coverage }}
          echo "ut_coverage finished"
          go tool cover -o ${{ env.ut_html_coverage }} -html=${{ env.ut_coverage }}
          echo "ut_html_coverage finished"
          rm -rf ${{ env.raw_ut_coverage }}
          pids=$(ps -aux | grep 'mo-service' | grep -v 'grep' | awk '{print $2}')
          if [ -n "$pids" ]; then
              kill -TERM $pids
          fi
  
          i=1 && while [ -n "$(ps -aux|grep 'mo-service' | grep -v 'grep' | awk '{print $2}')" ] && [ $i -le 300 ]; do echo "mo-service kill not finished......$i/300"; i=$(($i+1)); sleep 1s; done
          if [ $i -gt 300 ]; then echo 'stop mo-service failed...'; exit 1; else echo 'stop mo-service successed...'; fi
          go tool covdata textfmt -i="$GITHUB_WORKSPACE/matrixone" -o ${{ env.raw_bvt_coverage }}
          grep -Ev ${{ env.bvt_excluded_pkg }} ${{ env.raw_bvt_coverage }} > ${{ env.bvt_coverage }}
          echo "bvt_coverage finished"
          go tool cover -o ${{ env.bvt_html_coverage }} -html=${{ env.bvt_coverage }}
          echo "bvt_html_coverage finished"
          
#          # generate sorted ut coverage file
#          sort ut_coverage.out | uniq > ut_coverage_sort.out

      - name: Upload Compressed Files
        uses: actions/upload-artifact@v7
        if: ${{ failure() || cancelled() }}
        continue-on-error: true
        with:
          name: coverage-files
          path: |
            ${{ github.workspace }}/matrixone/mo-service.log
            ${{ github.workspace }}/matrixone/ut_coverage.out
            ${{ github.workspace }}/matrixone/bvt_coverage.out
            ${{ github.workspace }}/matrixone/diff.patch
          retention-days: 7
      - name: Checkout for Python file
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v6
        with:
          token: ${{ secrets.TOKEN_ACTION }}
          repository: matrixorigin/CI
          fetch-depth: "1"
          path: CI
      - name: Get Python file
        if: ${{ always() && !cancelled() }}
        run: |
          cd $GITHUB_WORKSPACE
          cp $GITHUB_WORKSPACE/CI/scripts/parse_coverage.py $GITHUB_WORKSPACE/parse_coverage.py
          cp $GITHUB_WORKSPACE/CI/scripts/.ignore $GITHUB_WORKSPACE/.ignore
      - name: Run coverage Python script
        run: |
          cd $GITHUB_WORKSPACE
          python $GITHUB_WORKSPACE/parse_coverage.py -coverage_files $GITHUB_WORKSPACE/matrixone/ut_coverage.out $GITHUB_WORKSPACE/matrixone/bvt_coverage.out -diff_path $GITHUB_WORKSPACE/matrixone/diff.patch

      - name: Show Dmesg For Debug
        continue-on-error: true
        if: ${{ always() }}
        run: |
          sudo dmesg -T

      - name: Compress final result files
        id: compress_result
        if: ${{ always() && !cancelled() }}
        continue-on-error: true
        run: |
          cd $GITHUB_WORKSPACE
          if [ ! -f "$GITHUB_WORKSPACE/pr_coverage.out" ]; then
            echo "pr_coverage.out does not exist.The code block has not changed,skip the remaining steps."
            echo "file_exists=false" >> $GITHUB_OUTPUT
            exit 0
          fi

          echo "file_exists=true" >> $GITHUB_OUTPUT

          cd $GITHUB_WORKSPACE/matrixone
          cp $GITHUB_WORKSPACE/pr_coverage.out $GITHUB_WORKSPACE/matrixone/pr_coverage.out
          go tool cover -o pr_coverage.html -html=pr_coverage.out

      - name: Upload final result files
        uses: actions/upload-artifact@v7
        if: ${{ always() && !cancelled() && steps.compress_result.outputs.file_exists == 'true' }}
        continue-on-error: true
        with:
          name: final-result-files
          path: |
            ${{ github.workspace }}/matrixone/pr_coverage.html
            ${{ github.workspace }}/pr_coverage.out
            ${{ github.workspace }}/merged_coverage.out
          retention-days: 7

  pr-size-label:
    environment: ci
    runs-on: ubuntu-22.04
    needs: [check_organization_user]
    steps:
      - name: size-label
        uses: "matrixorigin/CI/actions/label-size-action@main"
        with:
          size_token: ${{ secrets.TOKEN_ACTION }}
          ignore: ".md,.pb.go"
          sizes: >
            {
              "XS":0,
              "S":10,
              "M":100,
              "L":500,
              "XL":1000,
              "XXL":2000
            }

  pr_basic_benchmarks:
    if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }}
    name: Pull Request Basic Benchmark Test
    needs:
      - check_organization_user
    environment: ci
    runs-on: amd64-mo-guangzhou-2xlarge16
    steps:
      - uses: actions/checkout@v6
        with:
          token: ${{ secrets.TOKEN_ACTION }}
          path: ./matrixone
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
      - name: Clone mo-load-data Repo
        if: ${{ !cancelled() && !failure() }}
        uses: actions/checkout@v6
        with:
          repository: matrixorigin/mo-load-data
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-load-data
          ref: main
      - name: Clone mo-load Repo
        if: ${{ !cancelled() && !failure() }}
        uses: actions/checkout@v6
        with:
          repository: matrixorigin/mo-load
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-load
          ref: "1.0"
      - name: Clone mo-tpcc Repo
        if: ${{ !cancelled() && !failure() }}
        uses: actions/checkout@v6
        with:
          repository: matrixorigin/mo-tpcc
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-tpcc
          ref: main
      - name: Clone mo-ssb Repo
        if: ${{ !cancelled() && !failure() }}
        uses: actions/checkout@v6
        with:
          repository: matrixorigin/mo-ssb
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-ssb
          ref: main
      - name: Clone mo-tpch Repo
        if: ${{ !cancelled() && !failure() }}
        uses: actions/checkout@v6
        with:
          repository: matrixorigin/mo-tpch
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-tpch
          ref: main

      - name: Set up Go and JAVA
        uses: matrixorigin/CI/actions/setup-env@main
        with:
          setup-java: false
          go-version-file: "${{ github.workspace }}/matrixone/go.mod"

      - name: Set Variables and PATH
        run: |
          echo 'GOPROXY=http://goproxy.goproxy.svc.cluster.local|https://goproxy.cn|direct' >> $GITHUB_ENV
          echo "$JAVA_HOME/bin" >> $GITHUB_PATH

      - name: Build MatrixOne Service
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          make clean;
          make config;
          make build;

      - name: Add cn.txn
        run: |
          cd $GITHUB_WORKSPACE/matrixone
          echo "[cn.txn]" >> ./etc/launch/cn.toml
          echo "  enable-leak-check = 1" >> ./etc/launch/cn.toml
          echo '  max-active-ages = "2m"'>> ./etc/launch/cn.toml

      - name: Start MatrixOne Service
        id: setup_mo_test_env
        run: |
          # start mo-service
          cd $GITHUB_WORKSPACE/matrixone
          ./optools/run_bvt.sh $GITHUB_WORKSPACE/matrixone launch
          
          echo "mo_svc_ip=127.0.0.1" >> $GITHUB_OUTPUT;
          sleep 60;

      - name: Create Account for Test
        if: ${{ !cancelled() && !failure() }}
        timeout-minutes: 5
        id: create_account
        run: |
          mysql -h 127.0.0.1 -P 6001 -udump -p111 -e "create account tpcc_test admin_name = 'admin' identified by '111' comment 'for tpcc test';"
          echo "tpcc_account=tpcc_test:admin" >> $GITHUB_OUTPUT
          echo "tpcc_password=111" >> $GITHUB_OUTPUT
          
          echo "sysbench_account=dump" >> $GITHUB_OUTPUT
          echo "sysbench_password=111" >> $GITHUB_OUTPUT
          
          echo "ssb_tpch_account=dump" >> $GITHUB_OUTPUT
          echo "ssb_tpch_password=111" >> $GITHUB_OUTPUT

      - name: Modify Tool's Setting And Set PATH
        if: ${{ !cancelled() && !failure() }}
        run: |
          cd $GITHUB_WORKSPACE/mo-load-data
          sed -i '/.*ENDPOINT: ""*/c\ENDPOINT: "cos.ap-guangzhou.myqcloud.com"' access.yml
          sed -i '/.*ACCESS_KEY_ID: ""*/c\ACCESS_KEY_ID: "${{secrets.TENCENT_AK}}"' access.yml
          sed -i '/.*SECRET_ACCESS_KEY: ""*/c\SECRET_ACCESS_KEY: "${{secrets.TENCENT_SK}}"' access.yml

      # start tpcc test
      - name: Load TPCC 10 Warehouse Data
        if: ${{ !cancelled() && !failure() }}
        timeout-minutes: 20
        id: load_tpcc_data
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s
          cd $GITHUB_WORKSPACE/mo-load-data
          mkdir -p report
          ./load.sh -u ${{ steps.create_account.outputs.tpcc_account }} -p ${{ steps.create_account.outputs.tpcc_password }} -h 127.0.0.1 -c cases/02_from_cos/tpcc_10 -r -m | tee report/mo-load-data-tpcc.log

      - name: Start TPCC 10 Warehouses 10 terminals Test
        if: ${{ !cancelled() && !failure() }}
        timeout-minutes: 25
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale

          cd $GITHUB_WORKSPACE/mo-tpcc
          mkdir -p report
          cp props.mo props_10.mo
          sed -i '/.*terminals=*/c\terminals=10' props_10.mo
          sed -i '/.*warehouses=*/c\warehouses=10' props_10.mo
          sed -i 's/tpcc/tpcc_10/g' props_10.mo
          sed -i 's/127.0.0.1/${{ needs.setup_mo_test_env.outputs.mo_svc_ip }}/g' props_10.mo
          sed -i '/runMins=*/c\runMins=5' props_10.mo
          sed -i '/.*expectedErrorCodes=*/c\expectedErrorCodes=20619,1062' props_10.mo
          sed -i 's/user=.*/user=${{ steps.create_account.outputs.tpcc_account }}/g' props_10.mo
          sed -i 's/password=.*/password=${{ steps.create_account.outputs.tpcc_password }}/g' props_10.mo
          ./runBenchmark.sh props_10.mo | tee report/tpcc-benchmark-10-10.log
          ./runVerify.sh props_10.mo | tee report/tpcc-verify-10-10.log

      - name: TPCC Result Summary
        if: ${{ !cancelled() && !failure() }}
        continue-on-error: true
        run: |
          cd $GITHUB_WORKSPACE/mo-tpcc
          files=("report/tpcc-benchmark-10-10.log")
          for filename in "${files[@]}"
          do
              if [ ! -f $filename ]; then
                  continue
              fi
              echo -e "\n[$filename]"
              grep "Running" $filename > tmp
              N=$(wc -l < tmp)
              head -n 3 tmp
              echo "..."
              middle_start=$((N / 2 - 1))
              middle_end=$((N / 2 + 1))
              head -n $middle_end tmp | tail -n +$middle_start
              echo "..."
              tail -n 3 tmp 
          done

      # start sysbench test
      - name: Prepare Sysbench Database and Tables
        if: ${{ !cancelled() && !failure() }}
        timeout-minutes: 5
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale

          cd $GITHUB_WORKSPACE/mo-load
          mkdir -p report
          sed -i 's/127.0.0.1/${{ needs.setup_mo_test_env.outputs.mo_svc_ip }}/g' mo.yml
          sed -i 's/name:.*$/name: "${{ steps.create_account.outputs.sysbench_account }}"/g' mo.yml
          sed -i 's/password:.*$/password: "${{ steps.create_account.outputs.sysbench_password }}"/g' mo.yml

          echo 'Start Prepare Data of 10-tables-100000-per-table'
          ./start.sh -m SYSBENCH -n 10 -s 100000 -t 1 | tee report/load-oltp-mix.log
          sleep 10

      # start sysbench test
      - name: Start OLTP Mixed Test With 10 Threads
        if: ${{ !cancelled() && !failure() }}
        timeout-minutes: 40
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale

          cd $GITHUB_WORKSPACE/mo-load
          mkdir -p report
          echo 'Start OLTP Insert Test With 10 threads'
          ./start.sh -c cases/sysbench/mixed_10_100000 -t 10 -d 5 -g -b sbtest | tee report/oltp-mix.log

      - name: Sysbench Result Summary
        if: ${{ !cancelled() && !failure() }}
        run: |
          cd $GITHUB_WORKSPACE/mo-load

          files=("report/oltp-mix.log")
          for filename in "${files[@]}"
          do
              if [ ! -f $filename ]; then
                  continue
              fi
              echo -e "\n[$filename]"
              grep "tps=" $filename > tmp
              N=$(wc -l < tmp)
              head -n 3 tmp
              echo "..."
              middle_start=$((N / 2 - 1))
              middle_end=$((N / 2 + 1))
              head -n $middle_end tmp | tail -n +$middle_start
              echo "..."
              tail -n 3 tmp 
          done


      - name: Start Load SSB 1G Data And Test
        if: ${{ !cancelled() && !failure() }}
        timeout-minutes: 20
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s
          cd $GITHUB_WORKSPACE/mo-load-data
          mkdir -p report
          ./load.sh -u ${{ steps.create_account.outputs.ssb_tpch_account }} -p ${{ steps.create_account.outputs.ssb_tpch_password }} -h 127.0.0.1 -c cases/02_from_cos/ssb_1 -r -m | tee report/mo-load-data-ssb.log
          # start test
          cd $GITHUB_WORKSPACE/mo-ssb
          mkdir -p report
          ./run.sh -u ${{ steps.create_account.outputs.ssb_tpch_account }} -p ${{ steps.create_account.outputs.ssb_tpch_password }} -h 127.0.0.1 -q all -s 1 -t 5 | tee report/mo-ssb.log

      - name: Load TPCH 1G Test Data
        if: ${{ !cancelled() && !failure() }}
        timeout-minutes: 20
        id: load_tpch_data
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s
          cd $GITHUB_WORKSPACE/mo-load-data
          mkdir -p report
          ./load.sh -u ${{ steps.create_account.outputs.ssb_tpch_account }} -p ${{ steps.create_account.outputs.ssb_tpch_password }} -h 127.0.0.1 -c cases/02_from_cos/tpch_1 -r -m | tee report/mo-load-data-tpch.log

      - name: Start TPCH 1G Test
        if: ${{ !cancelled() && !failure() && steps.load_tpch_data.conclusion == 'success'}}
        id: step_tpch_test
        timeout-minutes: 20
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s
          cd $GITHUB_WORKSPACE/mo-tpch
          mkdir -p report
          ./run.sh -u ${{ steps.create_account.outputs.ssb_tpch_account }} -p ${{ steps.create_account.outputs.ssb_tpch_password }} -h 127.0.0.1 -q all -s 1 -t 5 | tee report/tpch.log

      - name: Clean mo-service
        if: ${{ always() }}
        run: |
          ps -aux | grep 'mo-service' | grep -v 'grep' | awk '{print $2}' | xargs kill -s SIGINT
          i=1 && while [ -n "$(ps -aux|grep 'mo-service' | grep -v 'grep' | awk '{print $2}')" ] && [ $i -le 300 ]; do echo "mo-service kill not finished......$i/300"; i=$(($i+1)); sleep 1s; done
          if [ $i -gt 300 ]; then echo 'stop mo-service failed...'; exit 1; else echo 'stop mo-service successed...'; fi

      - name: Collect Upload files
        if: ${{ always() }}
        run: |
          mkdir -p $GITHUB_WORKSPACE/reports
          mkdir -p $GITHUB_WORKSPACE/reports/mo-ssb
          mkdir -p $GITHUB_WORKSPACE/reports/mo-load-data
          mkdir -p $GITHUB_WORKSPACE/reports/mo-tpch
          mkdir -p $GITHUB_WORKSPACE/reports/mo-load
          mkdir -p $GITHUB_WORKSPACE/reports/mo-tpcc
          mkdir -p $GITHUB_WORKSPACE/mo-ssb/report && cp -r $GITHUB_WORKSPACE/mo-ssb/report $GITHUB_WORKSPACE/reports/mo-ssb
          mkdir -p $GITHUB_WORKSPACE/mo-load-data/report && cp -r $GITHUB_WORKSPACE/mo-load-data/report $GITHUB_WORKSPACE/reports/mo-load-data
          mkdir -p $GITHUB_WORKSPACE/mo-tpch/report && cp -r $GITHUB_WORKSPACE/mo-tpch/report $GITHUB_WORKSPACE/reports/mo-tpch
          mkdir -p $GITHUB_WORKSPACE/mo-load/report && cp -r $GITHUB_WORKSPACE/mo-load/report $GITHUB_WORKSPACE/reports/mo-load
          mkdir -p $GITHUB_WORKSPACE/mo-tpcc/report && cp -r $GITHUB_WORKSPACE/mo-tpcc/report $GITHUB_WORKSPACE/reports/mo-tpcc
          
          cp $GITHUB_WORKSPACE/matrixone/mo-service.log $GITHUB_WORKSPACE/reports/mo-service.log


      - uses: actions/upload-artifact@v7
        if: ${{ failure() || cancelled() }}
        continue-on-error: true
        with:
          name: reports
          path: |
            ${{ github.workspace }}/reports
          retention-days: 7