full-sweep-test.yml

name: Test - Full Sweep

on:
    workflow_dispatch:
        inputs:
            run_1k1k:
                type: boolean
                required: false
            run_8k1k:
                type: boolean
                required: false
            run_1k8k:
                type: boolean
                required: false
            use_h100:
                type: boolean
                required: false
            use_h200:
                type: boolean
                required: false
            use_b200:
                type: boolean
                required: false
            use_mi300x:
                type: boolean
                required: false
            use_mi325x:
                type: boolean
                required: false
            use_mi355x:
                type: boolean
                required: false
            use_gb200:
                type: boolean
                required: false

jobs:
    get-configs:
        runs-on: ubuntu-latest
        outputs:
            dsr1-1k1k: ${{ steps.generate-configs.outputs.dsr1-1k1k }}
            dsr1-1k8k: ${{ steps.generate-configs.outputs.dsr1-1k8k }}
            dsr1-8k1k: ${{ steps.generate-configs.outputs.dsr1-8k1k }}
            gptoss-1k1k: ${{ steps.generate-configs.outputs.gptoss-1k1k }}
            gptoss-1k8k: ${{ steps.generate-configs.outputs.gptoss-1k8k }}
            gptoss-8k1k: ${{ steps.generate-configs.outputs.gptoss-8k1k }}
        steps:
            - name: Checkout code
              uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0

              # This looks complicated, but it is just calling generate_sweep_configs.py conditioned on 
              # discrete inputs (i.e., run_1k1k, run_h100, etc.) to split the test sweep into discrete jobs 
            - id: generate-configs
              run: |
                  pip install pydantic

                  set -x
                  # Build runner type filters based on inputs
                  RUNNER_TYPES="${{ inputs.use_h100 && 'h100' || '' }} ${{ inputs.use_h200 && 'h200' || '' }} ${{ inputs.use_b200 && 'b200 b200-trt' || '' }} ${{ inputs.use_mi300x && 'mi300x' || '' }} ${{ inputs.use_mi325x && 'mi325x' || '' }} ${{ inputs.use_mi355x && 'mi355x' || '' }}"

                  # DSR1 doesn't support H100, so exclude it
                  DSR1_RUNNER_TYPES=$(echo $RUNNER_TYPES | sed 's/\bh100\b//g' | xargs)

                  # Generate dsr1 configs (only if we have valid runner types for DSR1)
                  if [ "${{ inputs.run_1k1k }}" = "true" ] && [ -n "$DSR1_RUNNER_TYPES" ]; then
                      DSR1_1K1K=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix dsr1 --runner-type $DSR1_RUNNER_TYPES --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml)
                      echo "dsr1-1k1k=$DSR1_1K1K" >> $GITHUB_OUTPUT
                  else
                      echo "dsr1-1k1k=[]" >> $GITHUB_OUTPUT
                  fi

                  if [ "${{ inputs.run_1k8k }}" = "true" ] && [ -n "$DSR1_RUNNER_TYPES" ]; then
                      DSR1_1K8K=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix dsr1 --runner-type $DSR1_RUNNER_TYPES --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml)
                      echo "dsr1-1k8k=$DSR1_1K8K" >> $GITHUB_OUTPUT
                  else
                      echo "dsr1-1k8k=[]" >> $GITHUB_OUTPUT
                  fi

                  if [ "${{ inputs.run_8k1k }}" = "true" ] && [ -n "$DSR1_RUNNER_TYPES" ]; then
                      DSR1_8K1K=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 8k1k --model-prefix dsr1 --runner-type $DSR1_RUNNER_TYPES --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml)
                      echo "dsr1-8k1k=$DSR1_8K1K" >> $GITHUB_OUTPUT
                  else
                      echo "dsr1-8k1k=[]" >> $GITHUB_OUTPUT
                  fi

                  # Generate gptoss configs (only if we have runner types selected)
                  if [ "${{ inputs.run_1k1k }}" = "true" ] && [ -n "$RUNNER_TYPES" ]; then
                      GPTOSS_1K1K=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix gptoss --runner-type $RUNNER_TYPES --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml)
                      echo "gptoss-1k1k=$GPTOSS_1K1K" >> $GITHUB_OUTPUT
                  else
                      echo "gptoss-1k1k=[]" >> $GITHUB_OUTPUT
                  fi

                  if [ "${{ inputs.run_1k8k }}" = "true" ] && [ -n "$RUNNER_TYPES" ]; then
                      GPTOSS_1K8K=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix gptoss --runner-type $RUNNER_TYPES --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml)
                      echo "gptoss-1k8k=$GPTOSS_1K8K" >> $GITHUB_OUTPUT
                  else
                      echo "gptoss-1k8k=[]" >> $GITHUB_OUTPUT
                  fi

                  if [ "${{ inputs.run_8k1k }}" = "true" ] && [ -n "$RUNNER_TYPES" ]; then
                      GPTOSS_8K1K=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/generate_sweep_configs.py full-sweep --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 8k1k --model-prefix gptoss --runner-type $RUNNER_TYPES --runner-config ${GITHUB_WORKSPACE}/.github/configs/runners.yaml)
                      echo "gptoss-8k1k=$GPTOSS_8K1K" >> $GITHUB_OUTPUT
                  else
                      echo "gptoss-8k1k=[]" >> $GITHUB_OUTPUT
                  fi

    # DSR1 1K1K Benchmarks
    benchmark-dsr1-1k1k:
        needs: get-configs
        if: ${{ needs.get-configs.outputs.dsr1-1k1k != '[]' }}
        uses: ./.github/workflows/benchmark-tmpl.yml
        name: dsr1 1k1k /
        strategy:
            fail-fast: false
            matrix:
                config: ${{ fromJson(needs.get-configs.outputs.dsr1-1k1k) }}
        secrets: inherit
        with:
            exp-name: ${{ matrix.config.exp-name }}
            isl: ${{ matrix.config.isl }}
            osl: ${{ matrix.config.osl }}
            max-model-len: ${{ matrix.config.max-model-len }}
            runner: ${{ matrix.config.runner }}
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            tp: ${{ matrix.config.tp }}
            ep: ${{ matrix.config.ep }}
            dp-attn: ${{ matrix.config.dp-attn }}
            conc: ${{ matrix.config.conc }}

    collect-dsr1-1k1k-results:
        needs: benchmark-dsr1-1k1k
        if: ${{ always() && needs.get-configs.outputs.dsr1-1k1k != '[]' }}
        uses: ./.github/workflows/collect-results.yml
        secrets: inherit
        with:
            exp-name: "dsr1_1k1k"

    # GPTOSS 1K1K Benchmarks
    benchmark-gptoss-1k1k:
        needs: get-configs
        if: ${{ needs.get-configs.outputs.gptoss-1k1k != '[]' }}
        uses: ./.github/workflows/benchmark-tmpl.yml
        name: gptoss 1k1k /
        strategy:
            fail-fast: false
            matrix:
                config: ${{ fromJson(needs.get-configs.outputs.gptoss-1k1k) }}
        secrets: inherit
        with:
            exp-name: ${{ matrix.config.exp-name }}
            isl: ${{ matrix.config.isl }}
            osl: ${{ matrix.config.osl }}
            max-model-len: ${{ matrix.config.max-model-len }}
            runner: ${{ matrix.config.runner }}
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            tp: ${{ matrix.config.tp }}
            ep: ${{ matrix.config.ep }}
            dp-attn: ${{ matrix.config.dp-attn }}
            conc: ${{ matrix.config.conc }}

    collect-gptoss-1k1k-results:
        needs: benchmark-gptoss-1k1k
        if: ${{ always() && needs.get-configs.outputs.gptoss-1k1k != '[]' }}
        uses: ./.github/workflows/collect-results.yml
        secrets: inherit
        with:
            exp-name: "gptoss_1k1k"

    # DSR1 8K1K Benchmarks
    benchmark-dsr1-8k1k:
        needs: get-configs
        if: ${{ needs.get-configs.outputs.dsr1-8k1k != '[]' }}
        uses: ./.github/workflows/benchmark-tmpl.yml
        name: dsr1 8k1k /
        strategy:
            fail-fast: false
            matrix:
                config: ${{ fromJson(needs.get-configs.outputs.dsr1-8k1k) }}
        secrets: inherit
        with:
            exp-name: ${{ matrix.config.exp-name }}
            isl: ${{ matrix.config.isl }}
            osl: ${{ matrix.config.osl }}
            max-model-len: ${{ matrix.config.max-model-len }}
            runner: ${{ matrix.config.runner }}
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            tp: ${{ matrix.config.tp }}
            ep: ${{ matrix.config.ep }}
            dp-attn: ${{ matrix.config.dp-attn }}
            conc: ${{ matrix.config.conc }}

    collect-dsr1-8k1k-results:
        needs: benchmark-dsr1-8k1k
        if: ${{ always() && needs.get-configs.outputs.dsr1-8k1k != '[]' }}
        uses: ./.github/workflows/collect-results.yml
        secrets: inherit
        with:
            exp-name: "dsr1_8k1k"

    # GPTOSS 8K1K Benchmarks
    benchmark-gptoss-8k1k:
        needs: get-configs
        if: ${{ needs.get-configs.outputs.gptoss-8k1k != '[]' }}
        uses: ./.github/workflows/benchmark-tmpl.yml
        name: gptoss 8k1k /
        strategy:
            fail-fast: false
            matrix:
                config: ${{ fromJson(needs.get-configs.outputs.gptoss-8k1k) }}
        secrets: inherit
        with:
            exp-name: ${{ matrix.config.exp-name }}
            isl: ${{ matrix.config.isl }}
            osl: ${{ matrix.config.osl }}
            max-model-len: ${{ matrix.config.max-model-len }}
            runner: ${{ matrix.config.runner }}
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            tp: ${{ matrix.config.tp }}
            ep: ${{ matrix.config.ep }}
            dp-attn: ${{ matrix.config.dp-attn }}
            conc: ${{ matrix.config.conc }}

    collect-gptoss-8k1k-results:
        needs: benchmark-gptoss-8k1k
        if: ${{ always() && needs.get-configs.outputs.gptoss-8k1k != '[]' }}
        uses: ./.github/workflows/collect-results.yml
        secrets: inherit
        with:
            exp-name: "gptoss_8k1k"

    # DSR1 1K8K Benchmarks
    benchmark-dsr1-1k8k:
        needs: get-configs
        if: ${{ needs.get-configs.outputs.dsr1-1k8k != '[]' }}
        uses: ./.github/workflows/benchmark-tmpl.yml
        name: dsr1 1k8k /
        strategy:
            fail-fast: false
            matrix:
                config: ${{ fromJson(needs.get-configs.outputs.dsr1-1k8k) }}
        secrets: inherit
        with:
            exp-name: ${{ matrix.config.exp-name }}
            isl: ${{ matrix.config.isl }}
            osl: ${{ matrix.config.osl }}
            max-model-len: ${{ matrix.config.max-model-len }}
            runner: ${{ matrix.config.runner }}
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            tp: ${{ matrix.config.tp }}
            ep: ${{ matrix.config.ep }}
            dp-attn: ${{ matrix.config.dp-attn }}
            conc: ${{ matrix.config.conc }}

    # This is a workaround until we can integrate GB200 into master configs.
    benchmark-gb200-1k1k:
        if: ${{ inputs.use_gb200 && inputs.run_1k1k }}
        uses: ./.github/workflows/benchmark-multinode-tmpl.yml
        name: gb200 1k1k sweep
        strategy:
            fail-fast: false
            matrix:
                config: &dsr1_static_configs
                    - {
                          "image": "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.5.1-rc0.pre3",
                          "model": "deepseek-r1-fp4",
                          "model-prefix": "dsr1",
                          "precision": "fp4",
                          "framework": "dynamo-trtllm",
                          "mtp": "off",
                      }
                    - {
                          "image": "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.5.1-rc0.pre3",
                          "model": "deepseek-r1-fp4",
                          "model-prefix": "dsr1",
                          "precision": "fp4",
                          "framework": "dynamo-trtllm",
                          "mtp": "on",
                      }
                    - {
                          "image": "nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.5.1-rc0.pre1",
                          "model": "deepseek-ai/DeepSeek-R1-0528",
                          "model-prefix": "dsr1",
                          "precision": "fp8",
                          "framework": "dynamo-sglang",
                          "mtp": "off",
                      }
        secrets: inherit
        with:
            runner: gb200
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            exp-name: ${{ matrix.config.model-prefix }}_1k1k
            isl: 1024
            osl: 1024
            max-model-len: 2048
            mtp-mode: ${{ matrix.config.mtp }}

    benchmark-gb200-1k8k:
        if: ${{ inputs.use_gb200 && inputs.run_1k8k }}
        uses: ./.github/workflows/benchmark-multinode-tmpl.yml
        name: gb200 1k8k sweep
        strategy:
            fail-fast: false
            matrix:
                config: *dsr1_static_configs
        secrets: inherit
        with:
            runner: gb200
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            exp-name: ${{ matrix.config.model-prefix }}_1k8k
            isl: 1024
            osl: 8192
            max-model-len: 9216
            mtp-mode: ${{ matrix.config.mtp }}

    benchmark-gb200-8k1k:
        if: ${{ inputs.use_gb200 && inputs.run_8k1k }}
        uses: ./.github/workflows/benchmark-multinode-tmpl.yml
        name: gb200 8k1k sweep
        strategy:
            fail-fast: false
            matrix:
                config: *dsr1_static_configs
        secrets: inherit
        with:
            runner: gb200
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            exp-name: ${{ matrix.config.model-prefix }}_8k1k
            isl: 1024
            osl: 8192
            max-model-len: 9216
            mtp-mode: ${{ matrix.config.mtp }}

    collect-dsr1-1k8k-results:
        needs:
            [
                benchmark-dsr1-1k8k,
                benchmark-gb200-1k1k,
                benchmark-gb200-1k8k,
                benchmark-gb200-8k1k,
            ]
        if: ${{ always() && needs.get-configs.outputs.dsr1-1k8k != '[]' }}
        uses: ./.github/workflows/collect-results.yml
        secrets: inherit
        with:
            exp-name: "dsr1_1k8k"

    # GPTOSS 1K8K Benchmarks
    benchmark-gptoss-1k8k:
        needs: get-configs
        if: ${{ needs.get-configs.outputs.gptoss-1k8k != '[]' }}
        uses: ./.github/workflows/benchmark-tmpl.yml
        name: gptoss 1k8k /
        strategy:
            fail-fast: false
            matrix:
                config: ${{ fromJson(needs.get-configs.outputs.gptoss-1k8k) }}
        secrets: inherit
        with:
            exp-name: ${{ matrix.config.exp-name }}
            isl: ${{ matrix.config.isl }}
            osl: ${{ matrix.config.osl }}
            max-model-len: ${{ matrix.config.max-model-len }}
            runner: ${{ matrix.config.runner }}
            image: ${{ matrix.config.image }}
            model: ${{ matrix.config.model }}
            framework: ${{ matrix.config.framework }}
            precision: ${{ matrix.config.precision }}
            tp: ${{ matrix.config.tp }}
            ep: ${{ matrix.config.ep }}
            dp-attn: ${{ matrix.config.dp-attn }}
            conc: ${{ matrix.config.conc }}

    collect-gptoss-1k8k-results:
        needs: benchmark-gptoss-1k8k
        if: ${{ always() && needs.get-configs.outputs.gptoss-1k8k != '[]' }}
        uses: ./.github/workflows/collect-results.yml
        secrets: inherit
        with:
            exp-name: "gptoss_1k8k"

    calc-success-rate:
        needs:
            [
                collect-dsr1-1k1k-results,
                collect-dsr1-1k8k-results,
                collect-dsr1-8k1k-results,
                collect-gptoss-1k1k-results,
                collect-gptoss-1k8k-results,
                collect-gptoss-8k1k-results,
            ]
        if: ${{ always() }}
        runs-on: ubuntu-latest

        env:
            RESULTS_DIR: "results/"
            STATS_FILENAME: "run_stats"
            GITHUB_TOKEN: ${{ secrets.REPO_PAT }}

        steps:
            - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
              with:
                  token: ${{ secrets.REPO_PAT }}
                  fetch-depth: 0

            - name: Download results artifacts
              uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
              with:
                  path: ${{ env.RESULTS_DIR }}
                  pattern: results_*

            - name: Install python dependencies
              run: pip install PyGithub

            - name: Calculate success rate
              run: python3 utils/calc_success_rate.py $STATS_FILENAME

            - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
              with:
                  name: "run-stats"
                  path: ${{ env.STATS_FILENAME }}.json