VLMEvalKit/.github/workflows/pr-run-test.yml at main · open-compass/VLMEvalKit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
name: pr_run_test

on:
  pull_request:
    branches:
      - "main"
    paths-ignore:
      - "docs/**"
      - "**.md"
  workflow_dispatch:
  schedule:
    - cron:  '56 01 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  BASE_SCORE: '{"MMBench_V11_MINI":{"Qwen2.5-VL-7B-Instruct":0.76363636,"InternVL3-8B":0.92727273,"llava-onevision-qwen2-0.5b-ov-hf":0.45454545},"MMStar_MINI":{"Qwen2.5-VL-7B-Instruct":0.6133333333333333,"InternVL3-8B":0.7,"llava-onevision-qwen2-0.5b-ov-hf":0.36},"AI2D_MINI":{"Qwen2.5-VL-7B-Instruct":0.7651821862348178,"InternVL3-8B":0.8218623481781376,"llava-onevision-qwen2-0.5b-ov-hf":0.48582995951417},"OCRBench_MINI":{"Qwen2.5-VL-7B-Instruct":15.7,"InternVL3-8B":17.3,"llava-onevision-qwen2-0.5b-ov-hf":5.5}}'
  HF_HUB_CACHE: /mnt/shared-storage-gpfs2/gpfs2-shared-public/huggingface/hub
  HF_HUB_OFFLINE: 1
  CONDA_PATH: /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/miniconda3
  CONDA_ENV: vlm_pr_test
  KUBEBRAIN_CLUSTER_ENTRY: https://h.pjlab.org.cn
  KUBEBRAIN_NAMESPACE: ailab-opencompass

jobs:
  vlm_test:
    if: ${{!cancelled()}}
    runs-on: [yidian_cu12_mllm]
    strategy:
      fail-fast: false
      matrix:
        dataset: ["MMBench_V11_MINI MMStar_MINI AI2D_MINI","OCRBench_MINI"]
        model: ['llava-onevision-qwen2-0.5b-ov-hf', 'InternVL3-8B', 'Qwen2.5-VL-7B-Instruct']
        include:
          - model: llava-onevision-qwen2-0.5b-ov-hf
            model_name: llava
          - model: Qwen2.5-VL-7B-Instruct
            model_name: qwen
          - model: InternVL3-8B
            model_name: internvl
          - dataset: MMBench_V11_MINI MMStar_MINI AI2D_MINI
            dataset_name: mmbench
          - dataset: OCRBench_MINI
            dataset_name: ocrbench
    steps:
      - name: Clean workdir
        run: sudo git clean -ffdx
      - name: clone_repo
        uses: actions/checkout@v3
      - name: reinstall vlmeval
        run: |
          . ${{env.CONDA_PATH}}/bin/activate
          conda activate ${{env.CONDA_ENV}}
          pip uninstall vlmeval -y
          pip install .
          pip install numpy==1.23.0 transformers==4.57.1
      - name: evaluation_model
        run: |
          . ${{env.CONDA_PATH}}/bin/activate
          conda activate ${{env.CONDA_ENV}}
          pip list

          rjob submit --metadata-name=vllm-pr-test-${{ github.run_id }}-${{matrix.model_name}}-${{matrix.dataset_name}}-${{ github.run_attempt }} --charged-group=llmeval_gpu --private-machine=group --namespace=ailab-llmeval --group=llmeval_gpu --gpu=1 --cpu=16 --memory=32568 --private-machine=group --image=registry.h.pjlab.org.cn/ailab-puyu-puyu_wsp_cpu/vlmevalkit:auto-v0.0.10 --env=HF_HUB_CACHE=/mnt/shared-storage-gpfs2/gpfs2-shared-public/huggingface/hub --env=HF_HUB_OFFLINE=1 --env=LMUData=/mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/LMUData --mount=gpfs://gpfs1/qa-llm-cicd:/mnt/shared-storage-user/qa-llm-cicd --mount=gpfs://gpfs1/opencompass-shared:/mnt/shared-storage-user/opencompass-shared --mount=gpfs://gpfs1/auto-eval-pipeline:/mnt/shared-storage-user/auto-eval-pipeline --mount=gpfs://gpfs2/gpfs2-shared-public:/mnt/shared-storage-gpfs2/gpfs2-shared-public --mount=gpfs://gpfs1/mllm:/mnt/shared-storage-user/mllm -- bash -exc 'cd ${{github.workspace}}; source /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/miniconda3/bin/activate; conda activate ${{env.CONDA_ENV}}; python run.py --data ${{matrix.dataset}} --model ${{matrix.model}} --work-dir /mnt/shared-storage-user/mllm/qa-llm-cicd/eval_report/${{ github.run_id }}/${{matrix.model}} --reuse --judge exact_matching 2>&1'

          for i in {1..1200}; do
            current_status=$(rjob get --namespace=ailab-llmeval vllm-pr-test-${{ github.run_id }}-${{matrix.model_name}}-${{matrix.dataset_name}}-${{ github.run_attempt }} | grep -oP 'rjob [^:]+: \K[^ ]+')
            echo "Current status: $current_status, stop checking"
            if [[ $current_status == "Succeeded" ]]; then
              echo "Task succeeded"
              rjob logs job --namespace=ailab-llmeval vllm-pr-test-${{ github.run_id }}-${{matrix.model_name}}-${{matrix.dataset_name}}-${{ github.run_attempt }} -n 100
              exit 0
            elif [[ $current_status == "Failed" || $current_status == "Stopped" ]]; then
              echo "Task failed or stopped, fetching logs"
              rjob logs job --namespace=ailab-llmeval vllm-pr-test-${{ github.run_id }}-${{matrix.model_name}}-${{matrix.dataset_name}}-${{ github.run_attempt }}
              exit 1
            fi
            sleep 6
          done
          rjob logs job --namespace=ailab-llmeval vllm-pr-test-${{ github.run_id }}-${{matrix.model_name}}-${{matrix.dataset_name}}-${{ github.run_attempt }} -n 100
          echo "Task timeout"
          exit 1
      - name: assert_result
        run: |
          . ${{env.CONDA_PATH}}/bin/activate
          conda activate ${{env.CONDA_ENV}}
          cp -r /mnt/shared-storage-user/mllm/qa-llm-cicd/eval_report/${{ github.run_id }}/${{matrix.model}} outputs
          python .github/scripts/assert_score.py --dataset "${{matrix.dataset}}" --base_score $BASE_SCORE --model-name ${{matrix.model}}
      - name: stop job
        if: always()
        run: |
          rjob stop vllm-pr-test-${{ github.run_id }}-${{matrix.model_name}}-${{matrix.dataset_name}}-${{ github.run_attempt }}

  vlm_api_test:
    if: ${{!cancelled()}}
    runs-on: [yidian_cu12_mllm]
    strategy:
      fail-fast: false
      matrix:
        answer_type: ["choice", "bool", "score", "text"]
        include:
          - answer_type: choice
            dataset: XLRS-Bench-lite MMMU_DEV_VAL MathVista_MINI AI2D_TEST MMStar
            mem_resoure: 2048
          - answer_type: bool
            dataset: HallusionBench
            mem_resoure: 1024
          - answer_type: score
            dataset: MMVet
            mem_resoure: 1024
          - answer_type: text
            dataset: OCRBench
            mem_resoure: 16384
    steps:
      - name: Clean workdir
        run: sudo git clean -ffdx
      - name: clone_repo
        uses: actions/checkout@v3
      - name: reinstall vlmeval
        run: |
          . ${{env.CONDA_PATH}}/bin/activate
          conda activate ${{env.CONDA_ENV}}
          pip uninstall vlmeval -y
          pip install .
          pip install numpy==1.23.0 transformers==4.57.1
      - name: evaluation_model
        run: |
          . ${{env.CONDA_PATH}}/bin/activate
          conda activate ${{env.CONDA_ENV}}
          pip list

          rjob submit --metadata-name=vllm-api-pr-test-${{ github.run_id }}-mock-api-${{matrix.answer_type}}-${{ github.run_attempt }} --charged-group=llmeval_gpu --private-machine=group --group=llmeval_gpu --gpu=0 --cpu=4 --memory=${{matrix.mem_resoure}} --private-machine=group --namespace=ailab-llmeval --image=registry.h.pjlab.org.cn/ailab-puyu-puyu_wsp_cpu/vlmevalkit:auto-v0.0.10 --env=HF_HUB_CACHE=/mnt/shared-storage-gpfs2/gpfs2-shared-public/huggingface/hub --env=HF_HUB_OFFLINE=1 --env=LMUData=/mnt/shared-storage-user/auto-eval-pipeline/vlmeval/LMUData --env=NO_PROXY=localhost,127.0.0.1,0.0.0.0 --mount=gpfs://gpfs1/qa-llm-cicd:/mnt/shared-storage-user/qa-llm-cicd --mount=gpfs://gpfs1/opencompass-shared:/mnt/shared-storage-user/opencompass-shared --mount=gpfs://gpfs1/auto-eval-pipeline:/mnt/shared-storage-user/auto-eval-pipeline --mount=gpfs://gpfs2/gpfs2-shared-public:/mnt/shared-storage-gpfs2/gpfs2-shared-public --mount=gpfs://gpfs1/mllm:/mnt/shared-storage-user/mllm -- bash -exc 'cd ${{github.workspace}}; source /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/miniconda3/bin/activate; conda activate ${{env.CONDA_ENV}}; nohup python /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/mock_chat_api.py --type ${{matrix.answer_type}} --port 24334 > mock_${{matrix.answer_type}}.log 2>&1 & sleep 3; python run.py --data ${{matrix.dataset}} --model mock/mock-api --api-mode --base-url http://0.0.0.0:24334/v1 --api-nproc 128 --local-media --judge mock-judge-api --judge-base-url http://0.0.0.0:24334/v1 --judge-api-nproc 32 --judge-timeout 60 --judge-retry 1 --work-dir /mnt/shared-storage-user/mllm/qa-llm-cicd/eval_report/${{ github.run_id }}/mock_api_${{matrix.answer_type}} --reuse 2>&1'


          for i in {1..1200}; do
            current_status=$(rjob get --namespace=ailab-llmeval vllm-api-pr-test-${{ github.run_id }}-mock-api-${{matrix.answer_type}}-${{ github.run_attempt }} | grep -oP 'rjob [^:]+: \K[^ ]+')
            echo "Current status: $current_status, stop checking"
            if [[ $current_status == "Succeeded" ]]; then
              echo "Task succeeded"
              rjob logs job --namespace=ailab-llmeval vllm-api-pr-test-${{ github.run_id }}-mock-api-${{matrix.answer_type}}-${{ github.run_attempt }}  -n 100
              exit 0
            elif [[ $current_status == "Failed" || $current_status == "Stopped" ]]; then
              echo "Task failed or stopped, fetching logs"
              rjob logs job --namespace=ailab-llmeval vllm-api-pr-test-${{ github.run_id }}-mock-api-${{matrix.answer_type}}-${{ github.run_attempt }}
              exit 1
            fi
            sleep 6
          done

          rjob logs job --namespace=ailab-llmeval vllm-api-pr-test-${{ github.run_id }}-mock-api-${{matrix.answer_type}}-${{ github.run_attempt }} -n 100
          echo "Task timeout"
          exit 1
      - name: assert_result
        run: |
          . ${{env.CONDA_PATH}}/bin/activate
          conda activate ${{env.CONDA_ENV}}
          CURRENT_DIR=/mnt/shared-storage-user/mllm/qa-llm-cicd/eval_report/${{ github.run_id }}/mock_api_${{matrix.answer_type}}/mock--mock-api
          BASELINE_DIR=/mnt/shared-storage-user/mllm/qa-llm-cicd/eval_report/baseline/mock_api_${{matrix.answer_type}}/mock--mock-api
          python .github/scripts/compare_dirs.py "$BASELINE_DIR" "$CURRENT_DIR"
      - name: stop job
        if: always()
        run: |
          rjob stop --namespace=ailab-llmeval vllm-api-pr-test-${{ github.run_id }}-mock-api-${{matrix.answer_type}}-${{ github.run_attempt }}