unit_tests.yml

name: Unit Tests

run-name: "${{ github.event.inputs.title }}"

defaults:
  run:
    shell: bash -le {0}

on:
  repository_dispatch:
  workflow_dispatch:
    inputs:
      title:
        description: 'set a title for this run'
        required: false
        default: ''
      repo:
        description: 'GitHub repo {owner}/{repo}'
        required: false
        default: ''
      ref:
        description: 'GitHub ref: Branch, Tag or Commit SHA'
        required: false
        default: ''
      pr_number:
        description: 'PR Number'
        required: false
        type: number
      test_regex:
        description: 'Regex to filter test files'
        required: false
        default: ''
      max-parallel:
        description: 'Parallel jobs'
        required: false
        default: '4'
      model-test-mode:
        description: 'Model compat test mode'
        type: choice
        required: false
        default: 'fast'
        options:
          - 'fast'
          - 'slow'
      server:
        description: 'Wheel Build Server'
        type: choice
        default: '["self-hosted", "xeon5"]'
        options:
          - '["self-hosted", "xeon5"]'
          - '["self-hosted", "zen4"]'

env:
  CUDA_DEVICE_ORDER: PCI_BUS_ID
  CUDA_VISIBLE_DEVICES: 0
  # Force fresh torch.ops JIT builds in CI so heterogeneous GPUs never reuse
  # cached kernels compiled by earlier jobs/tests.
  GPTQMODEL_KERNEL_REBUILD: '1'
  # CI can allocate A100-class devices (sm_80), so the shared JIT arch list
  # must include 8.0 in addition to the newer targets.
  TORCH_CUDA_ARCH_LIST: 8.0;8.6;8.9;9.0;12.0
  PYTORCH_ALLOC_CONF: 'expandable_segments:True'
  RUNNER: 10.0.13.31
  XEON5: 10.0.14.249
  GPU_ALLOCATOR_URL: http://10.0.13.31/gpu
  LOGBAR_ANIMATION: '0'
  CUDA_VERSION: 130
  UV_TORCH_BACKEND: cu130
  TORCH_VERSION: 2.11.0
  PYTHON_VERSION: 3.14t
  UV_PYTHON: 3.14t
  # PYTHON_GIL: 0 // test libs don't support yet
  IGNORED_TEST_FILES: "test_tgi.py,test_gptneox.py,models/test_mixtral.py,models/test_phi_3_moe.py,test_bits_new.py,models/test_internlm.py,models/test_internlm2_5.py,models/test_xverse.py"
  MODEL_TEST_MODE: ${{ github.event.inputs['model-test-mode'] || 'fast' }}
  MODEL_TEST_GPU_COUNT: '1'
  DEFUSER_GIT_URL: git+https://github.com/modelcloud/Defuser.git
  PYPCRE_GIT_URL: git+https://github.com/modelcloud/PyPcre.git
  TOKENICER_GIT_URL: git+https://github.com/modelcloud/Tokenicer.git
  LOGBAR_GIT_URL: git+https://github.com/modelcloud/LogBar.git
  EVALUTION_GIT_URL: git+https://github.com/modelcloud/Evalution.git
  repo: ${{ github.event.inputs.repo || github.repository }}
  ref: ${{ github.event.inputs.ref || github.ref }}
  HF_TOKEN: ${{ secrets.HF_TOKEN }}

concurrency:
  group: ${{ github.event.inputs.ref || github.ref }}-workflow-unit-tests #-${{ github.event.inputs.test_names }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  check-vm:
    runs-on: ubuntu-latest
    outputs:
      ip: ${{ steps.get_ip.outputs.ip }}
      run_id: ${{ steps.get_ip.outputs.run_id }}
      max-parallel: ${{ steps.get_ip.outputs['max-parallel'] }}
      cuda_version: ${{ env.CUDA_VERSION }}
    steps:
      - name: Checkout Codes
        uses: actions/checkout@v6
        with:
          repository: ${{ env.repo }}
          ref: ${{ env.ref }}

      - name: Print env
        run: |
          echo "repo: ${{ env.repo }}"
          echo "ref: ${{ env.ref }}"
          echo "selected server: ${{ github.event.inputs.server }}"

      - name: Set server
        id: get_ip
        run: |
          bash .github/scripts/ci_write_runner_outputs.sh \
            "$RUNNER" \
            "${{ github.run_id }}" \
            "${{ github.event.inputs['max-parallel'] }}"

  list-test-files:
    runs-on: ubuntu-latest
    outputs:
      cpu-files: ${{ steps.files.outputs.cpu-files }}
      torch-files: ${{ steps.files.outputs.torch-files }}
      model-files: ${{ steps.files.outputs.model-files }}
      m4-files: ${{ steps.files.outputs.m4-files }}
      cpu-matrix: ${{ steps.files.outputs.cpu-matrix }}
      torch-matrix: ${{ steps.files.outputs.torch-matrix }}
      model-matrix: ${{ steps.files.outputs.model-matrix }}

    steps:
      - name: Checkout Codes
        uses: actions/checkout@v6
        with:
          repository: ${{ env.repo }}
          ref: ${{ env.ref }}

      - name: Prepare checkout
        run: |
          .github/scripts/ci_prepare_checkout.sh "${{ github.event.inputs.pr_number }}"

      - name: List files
        id: files
        run: |
          test_plan=$(python3 .github/scripts/ci_workflow.py list-tests \
          --ignored-test-files "$IGNORED_TEST_FILES" \
          --test-regex "${{ github.event.inputs.test_regex }}")
          echo "Test plan: $test_plan"

          TEST_PLAN="$test_plan" python3 - <<'PY' >> "$GITHUB_OUTPUT"
          import json
          import os

          plan = json.loads(os.environ["TEST_PLAN"])
          print(f"cpu-files={json.dumps(plan['cpu_files'])}")
          print(f"torch-files={json.dumps(plan['torch_files'])}")
          print(f"model-files={json.dumps(plan['model_files'])}")
          print(f"m4-files={json.dumps(plan['mlx_files'])}")
          print(f"cpu-matrix={json.dumps(plan['cpu_matrix'])}")
          print(f"torch-matrix={json.dumps(plan['torch_matrix'])}")
          print(f"model-matrix={json.dumps(plan['model_matrix'])}")
          PY

          TEST_PLAN="$test_plan" python3 - <<'PY'
          import json
          import os

          plan = json.loads(os.environ["TEST_PLAN"])
          print(f"CPU Test files: {json.dumps(plan['cpu_files'])}")
          print(f"Torch Test files: {json.dumps(plan['torch_files'])}")
          print(f"Model Compat Test files: {json.dumps(plan['model_files'])}")
          print(f"MLX Test files: {json.dumps(plan['mlx_files'])}")
          print(f"CPU Matrix: {json.dumps(plan['cpu_matrix'])}")
          print(f"Torch Matrix: {json.dumps(plan['torch_matrix'])}")
          print(f"Model Matrix: {json.dumps(plan['model_matrix'])}")
          PY
          echo "Ignored Test files: $IGNORED_TEST_FILES"

  cpu:
    needs:
      - list-test-files
      - check-vm
    if: always() && !cancelled() && needs.list-test-files.outputs.cpu-matrix != '[]'
    uses: ./.github/workflows/unit_tests_reusable.yml
    secrets: inherit
    with:
      repo: ${{ github.event.inputs.repo || github.repository }}
      ref: ${{ github.event.inputs.ref || github.ref }}
      pr_number: ${{ fromJSON(github.event.inputs.pr_number || '0') }}
      check_vm_ip: ${{ needs.check-vm.outputs.ip }}
      check_vm_max_parallel: ${{ needs.check-vm.outputs['max-parallel'] }}
      check_vm_cuda_version: ${{ needs.check-vm.outputs.cuda_version }}
      matrix_json: ${{ needs.list-test-files.outputs.cpu-matrix || '[]' }}
      model_test_mode: ${{ github.event.inputs['model-test-mode'] || 'fast' }}

  torch:
    needs:
      - list-test-files
      - check-vm
    if: always() && !cancelled() && needs.list-test-files.outputs.torch-matrix != '[]'
    uses: ./.github/workflows/unit_tests_reusable.yml
    secrets: inherit
    with:
      repo: ${{ github.event.inputs.repo || github.repository }}
      ref: ${{ github.event.inputs.ref || github.ref }}
      pr_number: ${{ fromJSON(github.event.inputs.pr_number || '0') }}
      check_vm_ip: ${{ needs.check-vm.outputs.ip }}
      check_vm_max_parallel: ${{ needs.check-vm.outputs['max-parallel'] }}
      check_vm_cuda_version: ${{ needs.check-vm.outputs.cuda_version }}
      matrix_json: ${{ needs.list-test-files.outputs.torch-matrix || '[]' }}
      model_test_mode: ${{ github.event.inputs['model-test-mode'] || 'fast' }}

  models:
    needs:
      - list-test-files
      - check-vm
    if: always() && !cancelled() && needs.list-test-files.outputs.model-matrix != '[]'
    uses: ./.github/workflows/unit_tests_reusable.yml
    secrets: inherit
    with:
      repo: ${{ github.event.inputs.repo || github.repository }}
      ref: ${{ github.event.inputs.ref || github.ref }}
      pr_number: ${{ fromJSON(github.event.inputs.pr_number || '0') }}
      check_vm_ip: ${{ needs.check-vm.outputs.ip }}
      check_vm_max_parallel: ${{ needs.check-vm.outputs['max-parallel'] }}
      check_vm_cuda_version: ${{ needs.check-vm.outputs.cuda_version }}
      matrix_json: ${{ needs.list-test-files.outputs.model-matrix || '[]' }}
      model_test_mode: ${{ github.event.inputs['model-test-mode'] || 'fast' }}

  check-torch:
    runs-on: [ self-hosted, xeon5 ]
    container:
      image: 10.0.13.31:5000/nvidia/cuda:130-ubuntu24.04_0415
      options: --device /dev/dri --ipc=host --runtime=nvidia --gpus all
      volumes:
        - /monster/ci/env/entrypoint.sh:/entrypoint.sh
        - /monster/ci/env/entrypoint.sh:/etc/profile.d/01-entrypoint.sh
        - /dev/dri/by-path:/dev/dri/by-path
        - /monster/ci/uv:/opt/uv
        - /monster/ci/env:/opt/env
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Test pypi pip
            env_name: pypi_pip_env_${{ github.run_id }}_${{ github.run_attempt }}
            use_pip: 'true'
            install_cmd: pip install gptqmodel torch -U -vvv
          - name: Test pypi uv
            env_name: pypi_uv_env_${{ github.run_id }}_${{ github.run_attempt }}
            use_pip: 'false'
            install_cmd: uv pip install gptqmodel torch -U -vvv
          - name: Test local pip
            env_name: local_pip_env_${{ github.run_id }}_${{ github.run_attempt }}
            use_pip: 'true'
            install_cmd: pip install . torch -U -vvv
          - name: Test local uv
            env_name: local_uv_env_${{ github.run_id }}_${{ github.run_attempt }}
            use_pip: 'false'
            install_cmd: uv pip install . torch -U -vvv
    steps:
      - name: Checkout Codes
        uses: actions/checkout@v6
        with:
          repository: ${{ env.repo }}
          ref: ${{ env.ref }}

      - name: Prepare checkout
        run: |
          .github/scripts/ci_prepare_checkout.sh "${{ github.event.inputs.pr_number }}"

      - name: ${{ matrix.name }}
        run: |
          uv venv "${{ matrix.env_name }}"
          source "${{ matrix.env_name }}/bin/activate"
          if [[ "${{ matrix.use_pip }}" == "true" ]]; then
            uv pip install pip -U
          fi
          if python -VV 2>&1 | grep -q '3\.14.*free-threading'; then
            echo "[INFO] Python 3.14t detected, installing torchao..."
            uv pip install http://10.0.13.31/files/torchao-0.18.0+git13cd013d6-cp314-cp314t-linux_x86_64.whl
          else
            echo "[INFO] Current Python is not 3.14t, skip torchao."
          fi

          ${{ matrix.install_cmd }} || echo "RETRY_INSTALL=true" >> $GITHUB_ENV

      - name: Clear installer cache on failure
        if: always() && env.RETRY_INSTALL == 'true'
        run: |
          source "${{ matrix.env_name }}/bin/activate"

          INSTALL_CMD='${{ matrix.install_cmd }}'

          if [[ "$INSTALL_CMD" == uv* || "$INSTALL_CMD" == *" uv "* ]]; then
            echo "[INFO] Previous install command uses uv, clearing uv cache..."
            uv cache clean
          elif [[ "$INSTALL_CMD" == pip* || "$INSTALL_CMD" == *" pip "* ]]; then
            echo "[INFO] Previous install command uses pip, clearing pip cache..."
            pip cache purge
          fi

      - name: retry ${{ matrix.name }}
        if: always() && env.RETRY_INSTALL == 'true'
        run: |
          source "${{ matrix.env_name }}/bin/activate"
          if [[ "${{ matrix.use_pip }}" == "true" ]]; then
            uv pip install pip -U
          fi
          if python -VV 2>&1 | grep -q '3\.14.*free-threading'; then
            echo "[INFO] Python 3.14t detected, installing torchao..."
            uv pip install http://10.0.13.31/files/torchao-0.18.0+git13cd013d6-cp314-cp314t-linux_x86_64.whl
          else
            echo "[INFO] Current Python is not 3.14t, skip torchao."
          fi
          
          ${{ matrix.install_cmd }}

  setuptools-compatibility:
    uses: ./.github/workflows/setuptools_compatibility_reusable.yml
    with:
      repo: ${{ github.event.inputs.repo || github.repository }}
      ref: ${{ github.event.inputs.ref || github.ref }}
      pr_number: ${{ fromJSON(github.event.inputs.pr_number || '0') }}