pr-validation.yml

name: CI Validation (MUSA GPU)

on:
  pull_request:
    branches: [main, develop]
    types: [opened, synchronize, reopened]
  workflow_dispatch:
    inputs:
      target_branch:
        description: "Target branch to check rebase against (for manual run)"
        required: false
        default: "main"
        type: string
      commit_sha:
        description: "Optional commit SHA to test instead of the workflow SHA"
        required: false
        default: ""
        type: string
      baseline_sha:
        description: "Optional baseline commit SHA; leave empty to use latest upstream/main"
        required: false
        default: ""
        type: string
  schedule:
    - cron: "0 14 * * *"

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

defaults:
  run:
    shell: bash

env:
  LOG_BASE: /home/runner/ci_logs
  MODEL_ROOT: /home/runner/tf_test_model_wheel
  BASELINE_REPO_URL: https://github.com/MooreThreads/tensorflow_musa_extension.git
  EMPTY_TEST_RESULT_PATTERN: 'No tests found|No test files found|Total Tests:[[:space:]]*0|Total Tests[[:space:]]*│[[:space:]]*0|总计:[[:space:]]*0|no average_time_summary entries|report path missing|log not found'
  MIN_INTEGRATION_TESTS: 500

jobs:
  prepare_metadata:
    name: Prepare Metadata
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 10
    outputs:
      commit_id: ${{ steps.collect.outputs.commit_id }}
      requested_commit_id: ${{ steps.collect.outputs.requested_commit_id }}
      requested_baseline_id: ${{ steps.collect.outputs.requested_baseline_id }}
      run_kind: ${{ steps.collect.outputs.run_kind }}
      run_id: ${{ steps.collect.outputs.run_id }}
      workflow_run_id: ${{ steps.collect.outputs.workflow_run_id }}
      workflow_run_attempt: ${{ steps.collect.outputs.workflow_run_attempt }}
      log_root: ${{ steps.collect.outputs.log_root }}
      host_name: ${{ steps.collect.outputs.host_name }}
      host_ip: ${{ steps.collect.outputs.host_ip }}
      host_sn: ${{ steps.collect.outputs.host_sn }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Prepare metadata
        id: collect
        run: |
          set -euo pipefail

          REQUESTED_COMMIT_ID="${{ github.event.inputs.commit_sha || '' }}"
          REQUESTED_BASELINE_ID="${{ github.event.inputs.baseline_sha || '' }}"

          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            RUN_KIND="pr"
            RUN_ID="${{ github.event.pull_request.head.sha }}"
            EFFECTIVE_COMMIT_ID="${{ github.sha }}"
          elif [[ "${{ github.event_name }}" == "schedule" ]]; then
            RUN_KIND="daily"
            RUN_ID="$(date +%F)"
            EFFECTIVE_COMMIT_ID="${{ github.sha }}"
          else
            RUN_KIND="manual"
            EFFECTIVE_COMMIT_ID="${REQUESTED_COMMIT_ID:-${{ github.sha }}}"
            if [[ -n "$REQUESTED_COMMIT_ID" ]]; then
              RUN_ID="$REQUESTED_COMMIT_ID"
            else
              RUN_ID="${{ github.run_id }}"
            fi
          fi

          WORKFLOW_RUN_ID="${{ github.run_id }}"
          WORKFLOW_RUN_ATTEMPT="${{ github.run_attempt }}"
          LOG_ROOT="$LOG_BASE/$RUN_KIND/$RUN_ID/run-$WORKFLOW_RUN_ID/attempt-$WORKFLOW_RUN_ATTEMPT"
          JOB_LOG_DIR="$LOG_ROOT/prepare-metadata"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/prepare-metadata.md"
          HOST_NAME="$(hostname)"
          HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
          if [[ -z "$HOST_IP" ]]; then
            HOST_IP="$(hostname -i 2>/dev/null || true)"
          fi
          HOST_IP="${HOST_IP:-unavailable}"
          HOST_SN="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || true)"
          HOST_SN="${HOST_SN:-unavailable}"

          mkdir -p "$JOB_LOG_DIR"

          {
            echo "## Prepare Metadata"
            echo
            echo "- Commit: $EFFECTIVE_COMMIT_ID"
            echo "- Requested commit override: ${REQUESTED_COMMIT_ID:-none}"
            echo "- Requested baseline override: ${REQUESTED_BASELINE_ID:-none}"
            echo "- Run kind: $RUN_KIND"
            echo "- Run id: $RUN_ID"
            echo "- Workflow run id: $WORKFLOW_RUN_ID"
            echo "- Workflow run attempt: $WORKFLOW_RUN_ATTEMPT"
            echo "- Log root: $LOG_ROOT"
            echo "- Runner: $HOST_NAME"
            echo "- Host IP: $HOST_IP"
            echo "- Host SN: $HOST_SN"
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"

          echo "commit_id=$EFFECTIVE_COMMIT_ID" >> "$GITHUB_OUTPUT"
          echo "requested_commit_id=$REQUESTED_COMMIT_ID" >> "$GITHUB_OUTPUT"
          echo "requested_baseline_id=$REQUESTED_BASELINE_ID" >> "$GITHUB_OUTPUT"
          echo "run_kind=$RUN_KIND" >> "$GITHUB_OUTPUT"
          echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
          echo "workflow_run_id=$WORKFLOW_RUN_ID" >> "$GITHUB_OUTPUT"
          echo "workflow_run_attempt=$WORKFLOW_RUN_ATTEMPT" >> "$GITHUB_OUTPUT"
          echo "log_root=$LOG_ROOT" >> "$GITHUB_OUTPUT"
          echo "host_name=$HOST_NAME" >> "$GITHUB_OUTPUT"
          echo "host_ip=$HOST_IP" >> "$GITHUB_OUTPUT"
          echo "host_sn=$HOST_SN" >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"


  format:
    name: Format Check
    needs: prepare_metadata
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 20
    outputs:
      status: ${{ steps.collect.outputs.status }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Checkout code
        run: |
          set -euo pipefail

          REPO_URL="${{ github.server_url }}/${{ github.repository }}.git"
          COMMIT_ID="${{ needs.prepare_metadata.outputs.commit_id }}"

          cd "${RUNNER_TEMP:-/tmp}"
          rm -rf "$GITHUB_WORKSPACE"
          mkdir -p "$GITHUB_WORKSPACE"
          cd "$GITHUB_WORKSPACE"
          git init -q
          git remote add origin "$REPO_URL"

          if git fetch --no-tags --prune --depth=1 origin "$COMMIT_ID"; then
            git checkout --force FETCH_HEAD
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            git fetch --no-tags --prune --depth=1 origin "refs/pull/${{ github.event.pull_request.number }}/merge"
            git checkout --force FETCH_HEAD
          else
            git fetch --no-tags --prune origin '+refs/heads/*:refs/remotes/origin/*'
            git checkout --force "$COMMIT_ID"
          fi

          git config --global --add safe.directory "$GITHUB_WORKSPACE" || true
          git clean -fdx
          git --no-pager log -1 --oneline

      - name: Run format check
        id: run_check
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/format"
          LOG_FILE="$JOB_LOG_DIR/format_check.log"
          mkdir -p "$JOB_LOG_DIR"

          format_exit=0
          export JOB_LOG_DIR LOG_FILE
          bash -l -c '
            set -o pipefail
            cd "$GITHUB_WORKSPACE"
            FILE_LIST=$(mktemp)
            find . \( -path ./build -o -path ./.git \) -prune -o \
              -regex ".*\.\(cc\|cpp\|hpp\|c\|h\|cu\)" -print0 > "$FILE_LIST"
            if [[ ! -s "$FILE_LIST" ]]; then
              echo "No C/C++ files to format-check." | tee "$LOG_FILE"
            else
              xargs -0 clang-format --Werror --dry-run < "$FILE_LIST" 2>&1 | tee "$LOG_FILE"
            fi
          ' || format_exit=$?

          echo "format_exit=$format_exit" >> "$GITHUB_OUTPUT"

      - name: Collect format summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/format"
          LOG_FILE="$JOB_LOG_DIR/format_check.log"
          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/format.md"
          STATUS="success"
          JOB_RESULT="success (non-blocking)"

          if [[ "${{ steps.run_check.outputs.format_exit }}" != "0" ]]; then
            STATUS="failure"
            echo "::warning::Format check failed. This job remains non-blocking."
          fi

          {
            echo "## Format Check"
            echo
            echo "- Job result: $JOB_RESULT"
            echo "- Format check result: $STATUS"
            echo "- Non-blocking: yes"
            echo "- Log file: $LOG_FILE"
            echo
            echo '```text'
            if [[ -f "$LOG_FILE" ]]; then
              tail -50 "$LOG_FILE" || true
            else
              echo "format log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"


  build_current:
    name: Build Current
    needs: prepare_metadata
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 120
    outputs:
      status: ${{ steps.collect.outputs.status }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Checkout code
        run: |
          set -euo pipefail

          REPO_URL="${{ github.server_url }}/${{ github.repository }}.git"
          COMMIT_ID="${{ needs.prepare_metadata.outputs.commit_id }}"

          cd "${RUNNER_TEMP:-/tmp}"
          rm -rf "$GITHUB_WORKSPACE"
          mkdir -p "$GITHUB_WORKSPACE"
          cd "$GITHUB_WORKSPACE"
          git init -q
          git remote add origin "$REPO_URL"

          if git fetch --no-tags --prune --depth=1 origin "$COMMIT_ID"; then
            git checkout --force FETCH_HEAD
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            git fetch --no-tags --prune --depth=1 origin "refs/pull/${{ github.event.pull_request.number }}/merge"
            git checkout --force FETCH_HEAD
          else
            git fetch --no-tags --prune origin '+refs/heads/*:refs/remotes/origin/*'
            git checkout --force "$COMMIT_ID"
          fi

          git config --global --add safe.directory "$GITHUB_WORKSPACE" || true
          git clean -fdx
          git --no-pager log -1 --oneline

      - name: Build current wheel
        id: run_build
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/build-current"
          LOG_FILE="$JOB_LOG_DIR/build_current.log"
          mkdir -p "$JOB_LOG_DIR"

          build_exit=0
          export LOG_FILE
          bash -l -c '
            set -o pipefail
            export CPLUS_INCLUDE_PATH="/usr/include/c++/11:/usr/include/x86_64-linux-gnu/c++/11:${CPLUS_INCLUDE_PATH:-}"
            cd "$GITHUB_WORKSPACE"
            rm -rf ./build ./dist
            ./build.sh wheel 2>&1 | tee "$LOG_FILE"
          ' || build_exit=$?

          echo "build_exit=$build_exit" >> "$GITHUB_OUTPUT"

      - name: Collect build summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          JOB_LOG_DIR="$LOG_ROOT/build-current"
          LOG_FILE="$JOB_LOG_DIR/build_current.log"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/build-current.md"
          WHEEL_PATH="$(find "$GITHUB_WORKSPACE/dist" -maxdepth 1 -name 'tensorflow_musa-*.whl' -print -quit 2>/dev/null || true)"
          STATUS="success"

          if [[ "${{ steps.run_build.outputs.build_exit }}" != "0" || -z "$WHEEL_PATH" ]]; then
            STATUS="failure"
          else
            WHEEL_DEST="$LOG_ROOT/wheels/current"
            mkdir -p "$WHEEL_DEST"
            cp "$WHEEL_PATH" "$WHEEL_DEST/"
          fi

          WHEEL_SIZE="$(stat -c%s "$WHEEL_PATH" 2>/dev/null || echo "n/a")"

          {
            echo "## Build Current"
            echo
            echo "- Status: $STATUS"
            echo "- Wheel path: ${WHEEL_PATH:-not found}"
            echo "- Wheel size: $WHEEL_SIZE"
            echo "- Log file: $LOG_FILE"
            echo
            echo '```text'
            if [[ -f "$LOG_FILE" ]]; then
              tail -50 "$LOG_FILE" || true
            else
              echo "build log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if build failed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1


  build_baseline:
    name: Build Baseline
    needs: prepare_metadata
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 120
    outputs:
      status: ${{ steps.collect.outputs.status }}
      baseline_sha: ${{ steps.collect.outputs.baseline_sha }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Sync and build baseline
        id: run_build
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/build-baseline"
          LOG_FILE="$JOB_LOG_DIR/build_baseline.log"
          mkdir -p "$JOB_LOG_DIR"

          sync_exit=0
          build_exit=0
          baseline_sha=""

          BASELINE_WORKSPACE="$JOB_LOG_DIR/baseline-workspace"
          echo "baseline_workspace=$BASELINE_WORKSPACE" >> "$GITHUB_OUTPUT"

          export LOG_FILE BASELINE_WORKSPACE BASELINE_REPO_URL
          bash -l -c '
            BASELINE_PARENT=$(dirname "$BASELINE_WORKSPACE")
            mkdir -p "$BASELINE_PARENT"
            if [[ ! -d "$BASELINE_WORKSPACE/.git" ]]; then
              git clone "$BASELINE_REPO_URL" "$BASELINE_WORKSPACE"
            fi
            cd "$BASELINE_WORKSPACE"
            if git remote get-url upstream >/dev/null 2>&1; then
              git remote set-url upstream "$BASELINE_REPO_URL"
            else
              git remote add upstream "$BASELINE_REPO_URL"
            fi
            git fetch upstream
            if [[ -n "${{ needs.prepare_metadata.outputs.requested_baseline_id }}" ]]; then
              git reset --hard "${{ needs.prepare_metadata.outputs.requested_baseline_id }}"
            else
              git reset --hard upstream/main
            fi
            git clean -fdx
          ' || sync_exit=$?

          if [[ "$sync_exit" == "0" ]]; then
            baseline_sha="$(git -C "$BASELINE_WORKSPACE" rev-parse HEAD 2>/dev/null || true)"
            export LOG_FILE BASELINE_WORKSPACE
            bash -l -c '
              set -o pipefail
              export CPLUS_INCLUDE_PATH="/usr/include/c++/11:/usr/include/x86_64-linux-gnu/c++/11:${CPLUS_INCLUDE_PATH:-}"
              cd "$BASELINE_WORKSPACE"
              rm -rf ./build ./dist
              ./build.sh wheel 2>&1 | tee "$LOG_FILE"
            ' || build_exit=$?
          fi

          echo "sync_exit=$sync_exit" >> "$GITHUB_OUTPUT"
          echo "build_exit=$build_exit" >> "$GITHUB_OUTPUT"
          echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"

      - name: Collect build summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          JOB_LOG_DIR="$LOG_ROOT/build-baseline"
          LOG_FILE="$JOB_LOG_DIR/build_baseline.log"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/build-baseline.md"
          BASELINE_WORKSPACE="${{ steps.run_build.outputs.baseline_workspace }}"
          WHEEL_PATH="$(find "$BASELINE_WORKSPACE/dist" -maxdepth 1 -name 'tensorflow_musa-*.whl' -print -quit 2>/dev/null || true)"
          BASELINE_SHA="${{ steps.run_build.outputs.baseline_sha }}"
          if [[ -z "$BASELINE_SHA" && -d "$BASELINE_WORKSPACE/.git" ]]; then
            BASELINE_SHA="$(git -C "$BASELINE_WORKSPACE" rev-parse HEAD 2>/dev/null || true)"
          fi
          STATUS="success"

          if [[ "${{ steps.run_build.outputs.sync_exit }}" != "0" || "${{ steps.run_build.outputs.build_exit }}" != "0" || -z "$WHEEL_PATH" ]]; then
            STATUS="failure"
          else
            WHEEL_DEST="$LOG_ROOT/wheels/baseline"
            mkdir -p "$WHEEL_DEST"
            cp "$WHEEL_PATH" "$WHEEL_DEST/"
          fi

          WHEEL_SIZE="$(stat -c%s "$WHEEL_PATH" 2>/dev/null || echo "n/a")"

          {
            echo "## Build Baseline"
            echo
            echo "- Status: $STATUS"
            echo "- Baseline commit: ${BASELINE_SHA:-unavailable}"
            echo "- Wheel path: ${WHEEL_PATH:-not found}"
            echo "- Wheel size: $WHEEL_SIZE"
            echo "- Log file: $LOG_FILE"
            echo
            echo '```text'
            if [[ -f "$LOG_FILE" ]]; then
              tail -50 "$LOG_FILE" || true
            else
              echo "build log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "baseline_sha=$BASELINE_SHA" >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if build failed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  integration:
    name: Integration Test
    needs: [prepare_metadata, build_current]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 45
    outputs:
      status: ${{ steps.collect.outputs.status }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Checkout code
        run: |
          set -euo pipefail

          REPO_URL="${{ github.server_url }}/${{ github.repository }}.git"
          COMMIT_ID="${{ needs.prepare_metadata.outputs.commit_id }}"

          cd "${RUNNER_TEMP:-/tmp}"
          rm -rf "$GITHUB_WORKSPACE"
          mkdir -p "$GITHUB_WORKSPACE"
          cd "$GITHUB_WORKSPACE"
          git init -q
          git remote add origin "$REPO_URL"

          if git fetch --no-tags --prune --depth=1 origin "$COMMIT_ID"; then
            git checkout --force FETCH_HEAD
          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
            git fetch --no-tags --prune --depth=1 origin "refs/pull/${{ github.event.pull_request.number }}/merge"
            git checkout --force FETCH_HEAD
          else
            git fetch --no-tags --prune origin '+refs/heads/*:refs/remotes/origin/*'
            git checkout --force "$COMMIT_ID"
          fi

          git config --global --add safe.directory "$GITHUB_WORKSPACE" || true
          git clean -fdx
          git --no-pager log -1 --oneline

      - name: Run integration tests
        id: run_test
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/integration"
          LOG_FILE="$JOB_LOG_DIR/integration_fusion_tests.log"
          mkdir -p "$JOB_LOG_DIR"

          test_exit=0
          export LOG_FILE WHEEL_DIR="${{ needs.prepare_metadata.outputs.log_root }}/wheels/current"
          bash -l <<'BASH' || test_exit=$?
          set -e -o pipefail
          cd "$GITHUB_WORKSPACE"
          pip install "$WHEEL_DIR"/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee "$LOG_FILE"
          cd "$GITHUB_WORKSPACE/test"
          python - <<'PY' 2>&1 | tee -a "$LOG_FILE"
          import importlib
          import pathlib
          import sys
          import unittest

          test_dir = pathlib.Path("ops")
          test_files = sorted(test_dir.glob("*_op_test.py"))
          if not test_files:
              print("No test files found matching pattern: *_op_test.py in ops/")
              sys.exit(1)

          sys.path.insert(0, str(test_dir.resolve()))
          loader = unittest.TestLoader()
          load_failures = []
          total_tests = 0

          for test_file in test_files:
              module_name = test_file.stem
              try:
                  module = importlib.import_module(module_name)
                  total_tests += loader.loadTestsFromModule(module).countTestCases()
              except Exception as exc:
                  load_failures.append((module_name, exc))

          if load_failures:
              print(f"Failed to load {len(load_failures)} test module(s):")
              for module_name, error in load_failures:
                  print(f"  ops/{module_name}: {error}")
              sys.exit(1)

          print(f"Discovered test cases: {total_tests}")
          if total_tests == 0:
              print("No tests found")
              sys.exit(1)
          PY
          timeout 60m python test_runner.py --quiet 2>&1 | tee -a "$LOG_FILE"
          BASH

          echo "test_exit=$test_exit" >> "$GITHUB_OUTPUT"

      - name: Collect integration summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          JOB_LOG_DIR="$LOG_ROOT/integration"
          LOG_FILE="$JOB_LOG_DIR/integration_fusion_tests.log"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/integration.md"
          STATUS="success"

          if [[ "${{ steps.run_test.outputs.test_exit }}" != "0" ]]; then
            STATUS="failure"
          fi
          if [[ ! -f "$LOG_FILE" ]]; then
            STATUS="failure"
          elif grep -Eq "$EMPTY_TEST_RESULT_PATTERN" "$LOG_FILE"; then
            STATUS="failure"
          else
            total_tests="$(grep -E "Total Tests[[:space:]]*(\||│|:)[[:space:]]*[0-9]+" "$LOG_FILE" | tail -1 | grep -oE "[0-9]+" | tail -1 || true)"
            if [[ -z "$total_tests" || "$total_tests" -lt "$MIN_INTEGRATION_TESTS" ]]; then
              STATUS="failure"
            fi
          fi

          {
            echo "## Integration Test"
            echo
            echo "- Status: $STATUS"
            echo "- Log file: $LOG_FILE"
            echo
            echo '```text'
            if [[ -f "$LOG_FILE" ]]; then
              grep -E "Total Tests|Passed|Failed|Errors|Skipped|Pass Rate|Execution Time" "$LOG_FILE" || tail -80 "$LOG_FILE" || true
            else
              echo "integration log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if integration failed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  t_perf:
    name: T Performance
    needs: [prepare_metadata, build_current, build_baseline]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success' && needs.build_baseline.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 60
    outputs:
      status: ${{ steps.collect.outputs.status }}
      baseline_ms: ${{ steps.collect.outputs.baseline_ms }}
      current_ms: ${{ steps.collect.outputs.current_ms }}
      threshold_ms: ${{ steps.collect.outputs.threshold_ms }}
      metrics_summary: ${{ steps.collect.outputs.metrics_summary }}
      metrics_json: ${{ steps.collect.outputs.metrics_json }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Run T performance baseline/current
        id: run_perf
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/t-perf"
          BASELINE_LOG="$JOB_LOG_DIR/t_perf_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/t_perf_current.log"
          mkdir -p "$JOB_LOG_DIR"
          rm -f "$BASELINE_LOG" "$CURRENT_LOG"

          baseline_exit=0
          current_exit=0
          guard_failed=0
          if [[ -z "${GPU_LOCK_FILE:-}" ]]; then
            echo "GPU_LOCK_FILE is not set; refusing to use a global GPU lock." | tee -a "$BASELINE_LOG"
            guard_failed=1
            lock_file=""
          else
            lock_file="$GPU_LOCK_FILE"
          fi

          export MODEL_ROOT JOB_LOG_DIR BASELINE_LOG CURRENT_LOG

          if [[ "$guard_failed" == "0" ]]; then
            exec 9>"$lock_file"
            flock 9
            echo "Acquired GPU test lock: $lock_file" | tee -a "$BASELINE_LOG"

            pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/baseline/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$BASELINE_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/prunedGraph"
              timeout 20m python run_inference.py \
                --device musa \
                --batch-size 1024 \
                --infer-iters 1000 2>&1 | tee "$BASELINE_LOG"
            ' || baseline_exit=$?

            pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/current/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$CURRENT_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/prunedGraph"
              timeout 20m python run_inference.py \
                --device musa \
                --batch-size 100 \
                --infer-iters 1000 2>&1 | tee "$CURRENT_LOG"
            ' || current_exit=$?
          else
            baseline_exit=1
            current_exit=1
          fi

          echo "baseline_exit=$baseline_exit" >> "$GITHUB_OUTPUT"
          echo "current_exit=$current_exit" >> "$GITHUB_OUTPUT"

      - name: Collect T performance summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          JOB_LOG_DIR="$LOG_ROOT/t-perf"
          BASELINE_LOG="$JOB_LOG_DIR/t_perf_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/t_perf_current.log"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/t-perf.md"
          STATUS="success"
          BASELINE_MS=""
          CURRENT_MS=""
          THRESHOLD_MS=""
          METRICS_SUMMARY=""
          METRICS_JSON=""
          RUNNER_NAME="${RUNNER_NAME:-unknown}"
          HOST_NAME="$(hostname)"
          HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
          HOST_SN="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || true)"

          HOST_IP="${HOST_IP:-unavailable}"
          HOST_SN="${HOST_SN:-unavailable}"

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" == "0" && -f "$BASELINE_LOG" ]]; then
            BASELINE_MS="$(python3 -c 'import pathlib,re,sys; text=pathlib.Path(sys.argv[1]).read_text(encoding="utf-8", errors="ignore"); matches=re.findall(r"平均:\s*([0-9]+(?:\.[0-9]+)?)\s*ms", text); print(matches[-1] if matches else "")' "$BASELINE_LOG" || true)"
          fi
          if [[ "${{ steps.run_perf.outputs.current_exit }}" == "0" && -f "$CURRENT_LOG" ]]; then
            CURRENT_MS="$(python3 -c 'import pathlib,re,sys; text=pathlib.Path(sys.argv[1]).read_text(encoding="utf-8", errors="ignore"); matches=re.findall(r"平均:\s*([0-9]+(?:\.[0-9]+)?)\s*ms", text); print(matches[-1] if matches else "")' "$CURRENT_LOG" || true)"
          fi

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" != "0" || "${{ steps.run_perf.outputs.current_exit }}" != "0" || -z "$BASELINE_MS" || -z "$CURRENT_MS" ]]; then
            STATUS="failure"
          else
            THRESHOLD_MS="$(awk -v base="$BASELINE_MS" 'BEGIN {printf "%.4f", base * 1.05}')"
            if awk -v cur="$CURRENT_MS" -v thr="$THRESHOLD_MS" 'BEGIN {exit !(cur > thr)}'; then
              STATUS="failure"
            fi
          fi

          METRICS_SUMMARY="baseline=${BASELINE_MS:-n/a} ms, current=${CURRENT_MS:-n/a} ms, threshold=${THRESHOLD_MS:-n/a} ms, result=${STATUS}"
          METRICS_JSON="$(python3 - "$STATUS" "$BASELINE_MS" "$CURRENT_MS" "$THRESHOLD_MS" <<'PY'
          import json
          import sys

          status, baseline_ms, current_ms, threshold_ms = sys.argv[1:5]
          payload = {
              "name": "T Performance",
              "kind": "single",
              "status": status,
              "baseline_ms": baseline_ms or "n/a",
              "current_ms": current_ms or "n/a",
              "threshold_ms": threshold_ms or "n/a",
          }
          print(json.dumps(payload, ensure_ascii=False, separators=(",", ":")))
          PY
          )"

          {
            echo "## T Performance"
            echo
            echo "- Status: $STATUS"
            echo "- Runner name: $RUNNER_NAME"
            echo "- Host name: $HOST_NAME"
            echo "- Host IP: $HOST_IP"
            echo "- Host SN: $HOST_SN"
            echo "- Log artifact: logs-t-perf"
            echo "- Baseline: ${BASELINE_MS:-n/a} ms"
            echo "- Current: ${CURRENT_MS:-n/a} ms"
            echo "- Threshold: ${THRESHOLD_MS:-n/a} ms"
            echo
            echo "### Baseline log tail"
            echo '```text'
            if [[ -f "$BASELINE_LOG" ]]; then
              tail -20 "$BASELINE_LOG" || true
            else
              echo "baseline log not found"
            fi
            echo '```'
            echo
            echo "### Current log tail"
            echo '```text'
            if [[ -f "$CURRENT_LOG" ]]; then
              tail -20 "$CURRENT_LOG" || true
            else
              echo "current log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "baseline_ms=$BASELINE_MS" >> "$GITHUB_OUTPUT"
          echo "current_ms=$CURRENT_MS" >> "$GITHUB_OUTPUT"
          echo "threshold_ms=$THRESHOLD_MS" >> "$GITHUB_OUTPUT"
          echo "metrics_summary=$METRICS_SUMMARY" >> "$GITHUB_OUTPUT"
          {
            echo "metrics_json<<EOF"
            echo "$METRICS_JSON"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if T performance regressed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1


  t_accuracy:
    name: T Accuracy
    needs: [prepare_metadata, build_current]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 30
    outputs:
      status: ${{ steps.collect.outputs.status }}
      current_acc: ${{ steps.collect.outputs.current_acc }}
      metrics_json: ${{ steps.collect.outputs.metrics_json }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Install current wheel
        run: pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/current/tensorflow_musa-*.whl --no-deps --force-reinstall

      - name: Run T accuracy current only
        id: run_acc
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/t-accuracy"
          CURRENT_LOG="$JOB_LOG_DIR/t_accuracy_current.log"
          mkdir -p "$JOB_LOG_DIR"
          rm -f "$CURRENT_LOG"

          current_exit=0
          guard_failed=0
          if [[ -z "${GPU_LOCK_FILE:-}" ]]; then
            echo "GPU_LOCK_FILE is not set; refusing to use a global GPU lock." | tee -a "$CURRENT_LOG"
            guard_failed=1
            lock_file=""
          else
            lock_file="$GPU_LOCK_FILE"
          fi

          export MODEL_ROOT JOB_LOG_DIR CURRENT_LOG

          if [[ "$guard_failed" == "0" ]]; then
            exec 9>"$lock_file"
            flock 9
            echo "Acquired GPU test lock: $lock_file" | tee -a "$CURRENT_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/prunedGraph"
              TF_ENABLE_ONEDNN_OPTS=0 \
              MUSA_ENABLE_TF32=0 \
              timeout 20m python run_inference.py \
                --device musa \
                --batch-size 100 \
                --check-acc \
                --rtol 1e-2 \
                --atol 1e-2 2>&1 | tee "$CURRENT_LOG"
            ' || current_exit=$?
          else
            current_exit=1
          fi

          echo "current_exit=$current_exit" >> "$GITHUB_OUTPUT"

      - name: Collect T accuracy summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          JOB_LOG_DIR="$LOG_ROOT/t-accuracy"
          CURRENT_LOG="$JOB_LOG_DIR/t_accuracy_current.log"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/t-accuracy.md"
          STATUS="success"
          CURRENT_ACC="$(grep -E "整体状态:.*(PASSED|FAILED)" "$CURRENT_LOG" | tail -1 | grep -oE "PASSED|FAILED" || echo "n/a")"
          METRICS_JSON=""
          RUNNER_NAME="${RUNNER_NAME:-unknown}"
          HOST_NAME="$(hostname)"
          HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
          HOST_SN="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || true)"

          HOST_IP="${HOST_IP:-unavailable}"
          HOST_SN="${HOST_SN:-unavailable}"

          if [[ "${{ steps.run_acc.outputs.current_exit }}" != "0" ]]; then
            STATUS="failure"
          fi
          if [[ "$CURRENT_ACC" != *"PASSED"* ]]; then
            STATUS="failure"
          fi

          METRICS_JSON="$(python3 - "$STATUS" "$CURRENT_ACC" <<'PY'
          import json
          import sys

          status, current_acc = sys.argv[1:3]
          payload = {
              "name": "T Accuracy",
              "kind": "accuracy",
              "status": status,
              "current": current_acc or "n/a",
          }
          print(json.dumps(payload, ensure_ascii=False, separators=(",", ":")))
          PY
          )"

          {
            echo "## T Accuracy"
            echo
            echo "- Status: $STATUS"
            echo "- Runner name: $RUNNER_NAME"
            echo "- Host name: $HOST_NAME"
            echo "- Host IP: $HOST_IP"
            echo "- Host SN: $HOST_SN"
            echo "- Log artifact: logs-t-accuracy"
            echo "- Current: ${CURRENT_ACC:-unavailable}"
            echo
            echo "### Current highlights"
            echo '```text'
            if [[ -f "$CURRENT_LOG" ]]; then
              grep -E '整体状态:|PASSED|FAILED|rtol|atol|误差|差异' "$CURRENT_LOG" | tail -20 || true
            else
              echo "current log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "current_acc=$CURRENT_ACC" >> "$GITHUB_OUTPUT"
          {
            echo "metrics_json<<EOF"
            echo "$METRICS_JSON"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if T accuracy failed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  bd_model1:
    name: BD Model 1
    needs: [prepare_metadata, build_current, build_baseline]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success' && needs.build_baseline.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 75
    outputs:
      status: ${{ steps.collect.outputs.status }}
      baseline_ms: ${{ steps.collect.outputs.baseline_ms }}
      current_ms: ${{ steps.collect.outputs.current_ms }}
      threshold_ms: ${{ steps.collect.outputs.threshold_ms }}
      metrics_summary: ${{ steps.collect.outputs.metrics_summary }}
      metrics_json: ${{ steps.collect.outputs.metrics_json }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Run model 1 baseline/current
        id: run_perf
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/bd-model1"
          BASELINE_LOG="$JOB_LOG_DIR/bd_model1_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/bd_model1_current.log"
          SPEC_PATH="$MODEL_ROOT/inference/metaGraph/meta_graph/meta_graph_1.spec"
          mkdir -p "$JOB_LOG_DIR"
          rm -f "$BASELINE_LOG" "$CURRENT_LOG"
          rm -rf "$JOB_LOG_DIR/baseline-out" "$JOB_LOG_DIR/current-out"

          baseline_exit=0
          current_exit=0
          guard_failed=0
          if [[ -z "${GPU_LOCK_FILE:-}" ]]; then
            echo "GPU_LOCK_FILE is not set; refusing to use a global GPU lock." | tee -a "$BASELINE_LOG"
            guard_failed=1
            lock_file=""
          else
            lock_file="$GPU_LOCK_FILE"
          fi

          export MODEL_ROOT JOB_LOG_DIR BASELINE_LOG CURRENT_LOG SPEC_PATH

          if [[ "$guard_failed" == "0" ]]; then
            exec 9>"$lock_file"
            flock 9
            echo "Acquired GPU test lock: $lock_file" | tee -a "$BASELINE_LOG"

            pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/baseline/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$BASELINE_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/metaGraph"
              MUSA_PINNED_FEED=1 \
              MUSA_PINNED_H2D_ON_COMPUTE_STREAM=1 \
              timeout 30m python musa_run_pb_graph.py \
                --spec "$SPEC_PATH" \
                --bs 32,128,256,1024 \
                --run_iters 30 \
                --out_root "$JOB_LOG_DIR/baseline-out" 2>&1 | tee "$BASELINE_LOG"
            ' || baseline_exit=$?

            pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/current/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$CURRENT_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/metaGraph"
              MUSA_PINNED_FEED=1 \
              MUSA_PINNED_H2D_ON_COMPUTE_STREAM=1 \
              timeout 30m python musa_run_pb_graph.py \
                --spec "$SPEC_PATH" \
                --bs 32,128,256,1024 \
                --run_iters 30 \
                --out_root "$JOB_LOG_DIR/current-out" 2>&1 | tee "$CURRENT_LOG"
            ' || current_exit=$?
          else
            baseline_exit=1
            current_exit=1
          fi

          echo "baseline_exit=$baseline_exit" >> "$GITHUB_OUTPUT"
          echo "current_exit=$current_exit" >> "$GITHUB_OUTPUT"

      - name: Collect model 1 summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/bd-model1"
          BASELINE_LOG="$JOB_LOG_DIR/bd_model1_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/bd_model1_current.log"
          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/bd-model1.md"
          COMPARE_JSON="$RUNNER_TEMP/bd-model1-compare.json"
          DETAIL_FILE="$RUNNER_TEMP/bd-model1-details.md"
          STATUS="success"
          BASELINE_MS=""
          CURRENT_MS=""
          THRESHOLD_MS=""
          METRICS_SUMMARY=""
          METRICS_JSON=""
          BASELINE_REPORT=""
          CURRENT_REPORT=""
          RUNNER_NAME="${RUNNER_NAME:-unknown}"
          HOST_NAME="$(hostname)"
          HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
          HOST_SN="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || true)"

          HOST_IP="${HOST_IP:-unavailable}"
          HOST_SN="${HOST_SN:-unavailable}"

          if [[ -f "$BASELINE_LOG" ]]; then
            REPORT_PATH="$(grep '^\[OK\] report=' "$BASELINE_LOG" | tail -1 | sed 's/^\[OK\] report=//' || true)"
            if [[ -n "$REPORT_PATH" && -f "$REPORT_PATH" ]]; then
              BASELINE_REPORT="$REPORT_PATH"
            fi
          fi

          if [[ -f "$CURRENT_LOG" ]]; then
            REPORT_PATH="$(grep '^\[OK\] report=' "$CURRENT_LOG" | tail -1 | sed 's/^\[OK\] report=//' || true)"
            if [[ -n "$REPORT_PATH" && -f "$REPORT_PATH" ]]; then
              CURRENT_REPORT="$REPORT_PATH"
            fi
          fi

          python3 - "$BASELINE_REPORT" "$CURRENT_REPORT" "$COMPARE_JSON" "$DETAIL_FILE" <<'PY'
          import json
          import os
          import sys

          baseline_report, current_report, compare_json, detail_file = sys.argv[1:5]
          THRESHOLD_RATIO = 1.05

          def load_report(path, label):
              if not path:
                  return None, [f"{label} report path missing"]
              if not os.path.isfile(path):
                  return None, [f"{label} report not found: {path}"]
              try:
                  with open(path, "r", encoding="utf-8") as f:
                      return json.load(f), []
              except Exception as exc:
                  return None, [f"failed to load {label} report {path}: {exc}"]

          def collect_rows(report, label):
              rows = {}
              errors = []
              if report is None:
                  return rows, errors
              for item in report.get("average_time_summary", []):
                  bs = str(item.get("batch_size"))
                  if not bs or bs == "None":
                      errors.append(f"{label} report contains empty batch_size entry")
                      continue
                  if bs in rows:
                      errors.append(f"{label} report contains duplicate batch_size={bs}")
                      continue
                  rows[bs] = {
                      "status": item.get("status"),
                      "average_time_ms": item.get("trimmed_avg_ms")
                      if item.get("trimmed_avg_ms") is not None
                      else item.get("average_time_ms"),
                      "error_core": item.get("error_core"),
                  }
              if not rows:
                  errors.append(f"{label} report has no average_time_summary entries")
              return rows, errors

          baseline_data, errors = load_report(baseline_report, "baseline")
          current_data, cur_errors = load_report(current_report, "current")
          errors.extend(cur_errors)

          baseline_rows, cur_errors = collect_rows(baseline_data, "baseline")
          errors.extend(cur_errors)
          current_rows, cur_errors = collect_rows(current_data, "current")
          errors.extend(cur_errors)

          def sort_key(bs):
              try:
                  return (0, int(bs))
              except Exception:
                  return (1, str(bs))

          all_bs = sorted(set(baseline_rows) | set(current_rows), key=sort_key)
          result_rows = []
          baseline_parts = []
          current_parts = []
          threshold_parts = []
          metrics_parts = []
          overall_status = "success"

          for bs in all_bs:
              base = baseline_rows.get(bs)
              cur = current_rows.get(bs)
              row = {
                  "batch_size": bs,
                  "baseline_ms": None,
                  "current_ms": None,
                  "threshold_ms": None,
                  "result": "ok",
                  "detail": "",
              }

              if base is None:
                  row["result"] = "missing-baseline"
                  row["detail"] = "baseline report missing this batch size"
              elif cur is None:
                  row["result"] = "missing-current"
                  row["detail"] = "current report missing this batch size"
              elif base.get("status") != "ok" or base.get("average_time_ms") is None:
                  row["result"] = "baseline-failed"
                  row["detail"] = base.get("error_core") or f"baseline status={base.get('status')}"
              elif cur.get("status") != "ok" or cur.get("average_time_ms") is None:
                  row["result"] = "current-failed"
                  row["detail"] = cur.get("error_core") or f"current status={cur.get('status')}"
              else:
                  row["baseline_ms"] = float(base["average_time_ms"])
                  row["current_ms"] = float(cur["average_time_ms"])
                  row["threshold_ms"] = float(base["average_time_ms"]) * THRESHOLD_RATIO
                  if row["current_ms"] > row["threshold_ms"]:
                      row["result"] = "regressed"
                      row["detail"] = "current > baseline * 1.05"

              if row["result"] != "ok":
                  overall_status = "failure"

              result_rows.append(row)
              baseline_parts.append(f"bs{bs}={'%.4f' % row['baseline_ms'] if row['baseline_ms'] is not None else 'n/a'}")
              current_parts.append(f"bs{bs}={'%.4f' % row['current_ms'] if row['current_ms'] is not None else 'n/a'}")
              threshold_parts.append(f"bs{bs}={'%.4f' % row['threshold_ms'] if row['threshold_ms'] is not None else 'n/a'}")
              metrics_parts.append(
                  "bs{bs}: baseline={base}, current={cur}, threshold={thr}, result={result}".format(
                      bs=bs,
                      base="%.4f" % row["baseline_ms"] if row["baseline_ms"] is not None else "n/a",
                      cur="%.4f" % row["current_ms"] if row["current_ms"] is not None else "n/a",
                      thr="%.4f" % row["threshold_ms"] if row["threshold_ms"] is not None else "n/a",
                      result=row["result"],
                  )
              )

          if errors:
              overall_status = "failure"

          with open(detail_file, "w", encoding="utf-8") as f:
              f.write("| BS | Baseline (ms) | Current (ms) | Threshold (ms) | Result | Detail |\n")
              f.write("|----|---------------|--------------|----------------|--------|--------|\n")
              for row in result_rows:
                  f.write(
                      "| {bs} | {base} | {cur} | {thr} | {result} | {detail} |\n".format(
                          bs=row["batch_size"],
                          base="%.4f" % row["baseline_ms"] if row["baseline_ms"] is not None else "n/a",
                          cur="%.4f" % row["current_ms"] if row["current_ms"] is not None else "n/a",
                          thr="%.4f" % row["threshold_ms"] if row["threshold_ms"] is not None else "n/a",
                          result=row["result"],
                          detail=(row["detail"] or "").replace("|", "/"),
                      )
                  )
              if errors:
                  f.write("\nAdditional errors:\n")
                  for err in errors:
                      f.write(f"- {err}\n")

          payload = {
              "name": "BD Model 1",
              "kind": "batch_compare",
              "status": overall_status,
              "baseline_ms": ", ".join(baseline_parts) if baseline_parts else "",
              "current_ms": ", ".join(current_parts) if current_parts else "",
              "threshold_ms": ", ".join(threshold_parts) if threshold_parts else "",
              "metrics_summary": " ; ".join(metrics_parts) if metrics_parts else "",
              "rows": result_rows,
          }
          with open(compare_json, "w", encoding="utf-8") as f:
              json.dump(payload, f, ensure_ascii=False, indent=2)
          PY

          STATUS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("status","failure"))' "$COMPARE_JSON")"
          BASELINE_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("baseline_ms",""))' "$COMPARE_JSON")"
          CURRENT_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("current_ms",""))' "$COMPARE_JSON")"
          THRESHOLD_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("threshold_ms",""))' "$COMPARE_JSON")"
          METRICS_SUMMARY="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("metrics_summary",""))' "$COMPARE_JSON")"
          METRICS_JSON="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(json.dumps(data, ensure_ascii=False, separators=(",", ":")))' "$COMPARE_JSON")"

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" != "0" || "${{ steps.run_perf.outputs.current_exit }}" != "0" ]]; then
            STATUS="failure"
          fi

          {
            echo "## BD Model 1"
            echo
            echo "- Status: $STATUS"
            echo "- Runner name: $RUNNER_NAME"
            echo "- Host name: $HOST_NAME"
            echo "- Host IP: $HOST_IP"
            echo "- Host SN: $HOST_SN"
            echo "- Log artifact: logs-bd-model1"
            echo "- Baseline: ${BASELINE_MS:-n/a}"
            echo "- Current: ${CURRENT_MS:-n/a}"
            echo "- Threshold: ${THRESHOLD_MS:-n/a}"
            echo
            echo "### Batch Size Comparison"
            cat "$DETAIL_FILE"
            echo
            echo "### Baseline log tail"
            echo '```text'
            if [[ -f "$BASELINE_LOG" ]]; then
              tail -20 "$BASELINE_LOG" || true
            else
              echo "baseline log not found"
            fi
            echo '```'
            echo
            echo "### Current log tail"
            echo '```text'
            if [[ -f "$CURRENT_LOG" ]]; then
              tail -20 "$CURRENT_LOG" || true
            else
              echo "current log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "baseline_ms=$BASELINE_MS" >> "$GITHUB_OUTPUT"
          echo "current_ms=$CURRENT_MS" >> "$GITHUB_OUTPUT"
          echo "threshold_ms=$THRESHOLD_MS" >> "$GITHUB_OUTPUT"
          echo "metrics_summary=$METRICS_SUMMARY" >> "$GITHUB_OUTPUT"
          {
            echo "metrics_json<<EOF"
            echo "$METRICS_JSON"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if model 1 regressed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  bd_model2:
    name: BD Model 2
    needs: [prepare_metadata, build_current, build_baseline]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success' && needs.build_baseline.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 75
    outputs:
      status: ${{ steps.collect.outputs.status }}
      baseline_ms: ${{ steps.collect.outputs.baseline_ms }}
      current_ms: ${{ steps.collect.outputs.current_ms }}
      threshold_ms: ${{ steps.collect.outputs.threshold_ms }}
      metrics_summary: ${{ steps.collect.outputs.metrics_summary }}
      metrics_json: ${{ steps.collect.outputs.metrics_json }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Run model 2 baseline/current
        id: run_perf
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/bd-model2"
          BASELINE_LOG="$JOB_LOG_DIR/bd_model2_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/bd_model2_current.log"
          SPEC_PATH="$MODEL_ROOT/inference/metaGraph/meta_graph/meta_graph_2.spec"
          mkdir -p "$JOB_LOG_DIR"
          rm -f "$BASELINE_LOG" "$CURRENT_LOG"
          rm -rf "$JOB_LOG_DIR/baseline-out" "$JOB_LOG_DIR/current-out"

          baseline_exit=0
          current_exit=0
          guard_failed=0
          if [[ -z "${GPU_LOCK_FILE:-}" ]]; then
            echo "GPU_LOCK_FILE is not set; refusing to use a global GPU lock." | tee -a "$BASELINE_LOG"
            guard_failed=1
            lock_file=""
          else
            lock_file="$GPU_LOCK_FILE"
          fi

          export MODEL_ROOT JOB_LOG_DIR BASELINE_LOG CURRENT_LOG SPEC_PATH

          if [[ "$guard_failed" == "0" ]]; then
            exec 9>"$lock_file"
            flock 9
            echo "Acquired GPU test lock: $lock_file" | tee -a "$BASELINE_LOG"

            pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/baseline/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$BASELINE_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/metaGraph"
              MUSA_PINNED_FEED=1 \
              MUSA_PINNED_H2D_ON_COMPUTE_STREAM=1 \
              timeout 30m python musa_run_pb_graph.py \
                --spec "$SPEC_PATH" \
                --bs 32,128,256,1024 \
                --run_iters 30 \
                --out_root "$JOB_LOG_DIR/baseline-out" 2>&1 | tee "$BASELINE_LOG"
            ' || baseline_exit=$?

            pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/current/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$CURRENT_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/metaGraph"
              MUSA_PINNED_FEED=1 \
              MUSA_PINNED_H2D_ON_COMPUTE_STREAM=1 \
              timeout 30m python musa_run_pb_graph.py \
                --spec "$SPEC_PATH" \
                --bs 32,128,256,1024 \
                --run_iters 30 \
                --out_root "$JOB_LOG_DIR/current-out" 2>&1 | tee "$CURRENT_LOG"
            ' || current_exit=$?
          else
            baseline_exit=1
            current_exit=1
          fi

          echo "baseline_exit=$baseline_exit" >> "$GITHUB_OUTPUT"
          echo "current_exit=$current_exit" >> "$GITHUB_OUTPUT"

      - name: Collect model 2 summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/bd-model2"
          BASELINE_LOG="$JOB_LOG_DIR/bd_model2_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/bd_model2_current.log"
          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/bd-model2.md"
          COMPARE_JSON="$RUNNER_TEMP/bd-model2-compare.json"
          DETAIL_FILE="$RUNNER_TEMP/bd-model2-details.md"
          STATUS="success"
          BASELINE_MS=""
          CURRENT_MS=""
          THRESHOLD_MS=""
          METRICS_SUMMARY=""
          METRICS_JSON=""
          BASELINE_REPORT=""
          CURRENT_REPORT=""
          RUNNER_NAME="${RUNNER_NAME:-unknown}"
          HOST_NAME="$(hostname)"
          HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
          HOST_SN="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || true)"

          HOST_IP="${HOST_IP:-unavailable}"
          HOST_SN="${HOST_SN:-unavailable}"

          if [[ -f "$BASELINE_LOG" ]]; then
            REPORT_PATH="$(grep '^\[OK\] report=' "$BASELINE_LOG" | tail -1 | sed 's/^\[OK\] report=//' || true)"
            if [[ -n "$REPORT_PATH" && -f "$REPORT_PATH" ]]; then
              BASELINE_REPORT="$REPORT_PATH"
            fi
          fi

          if [[ -f "$CURRENT_LOG" ]]; then
            REPORT_PATH="$(grep '^\[OK\] report=' "$CURRENT_LOG" | tail -1 | sed 's/^\[OK\] report=//' || true)"
            if [[ -n "$REPORT_PATH" && -f "$REPORT_PATH" ]]; then
              CURRENT_REPORT="$REPORT_PATH"
            fi
          fi

          python3 - "$BASELINE_REPORT" "$CURRENT_REPORT" "$COMPARE_JSON" "$DETAIL_FILE" <<'PY'
          import json
          import os
          import sys

          baseline_report, current_report, compare_json, detail_file = sys.argv[1:5]
          THRESHOLD_RATIO = 1.05

          def load_report(path, label):
              if not path:
                  return None, [f"{label} report path missing"]
              if not os.path.isfile(path):
                  return None, [f"{label} report not found: {path}"]
              try:
                  with open(path, "r", encoding="utf-8") as f:
                      return json.load(f), []
              except Exception as exc:
                  return None, [f"failed to load {label} report {path}: {exc}"]

          def collect_rows(report, label):
              rows = {}
              errors = []
              if report is None:
                  return rows, errors
              for item in report.get("average_time_summary", []):
                  bs = str(item.get("batch_size"))
                  if not bs or bs == "None":
                      errors.append(f"{label} report contains empty batch_size entry")
                      continue
                  if bs in rows:
                      errors.append(f"{label} report contains duplicate batch_size={bs}")
                      continue
                  rows[bs] = {
                      "status": item.get("status"),
                      "average_time_ms": item.get("trimmed_avg_ms")
                      if item.get("trimmed_avg_ms") is not None
                      else item.get("average_time_ms"),
                      "error_core": item.get("error_core"),
                  }
              if not rows:
                  errors.append(f"{label} report has no average_time_summary entries")
              return rows, errors

          baseline_data, errors = load_report(baseline_report, "baseline")
          current_data, cur_errors = load_report(current_report, "current")
          errors.extend(cur_errors)

          baseline_rows, cur_errors = collect_rows(baseline_data, "baseline")
          errors.extend(cur_errors)
          current_rows, cur_errors = collect_rows(current_data, "current")
          errors.extend(cur_errors)

          def sort_key(bs):
              try:
                  return (0, int(bs))
              except Exception:
                  return (1, str(bs))

          all_bs = sorted(set(baseline_rows) | set(current_rows), key=sort_key)
          result_rows = []
          baseline_parts = []
          current_parts = []
          threshold_parts = []
          metrics_parts = []
          overall_status = "success"

          for bs in all_bs:
              base = baseline_rows.get(bs)
              cur = current_rows.get(bs)
              row = {
                  "batch_size": bs,
                  "baseline_ms": None,
                  "current_ms": None,
                  "threshold_ms": None,
                  "result": "ok",
                  "detail": "",
              }

              if base is None:
                  row["result"] = "missing-baseline"
                  row["detail"] = "baseline report missing this batch size"
              elif cur is None:
                  row["result"] = "missing-current"
                  row["detail"] = "current report missing this batch size"
              elif base.get("status") != "ok" or base.get("average_time_ms") is None:
                  row["result"] = "baseline-failed"
                  row["detail"] = base.get("error_core") or f"baseline status={base.get('status')}"
              elif cur.get("status") != "ok" or cur.get("average_time_ms") is None:
                  row["result"] = "current-failed"
                  row["detail"] = cur.get("error_core") or f"current status={cur.get('status')}"
              else:
                  row["baseline_ms"] = float(base["average_time_ms"])
                  row["current_ms"] = float(cur["average_time_ms"])
                  row["threshold_ms"] = float(base["average_time_ms"]) * THRESHOLD_RATIO
                  if row["current_ms"] > row["threshold_ms"]:
                      row["result"] = "regressed"
                      row["detail"] = "current > baseline * 1.05"

              if row["result"] != "ok":
                  overall_status = "failure"

              result_rows.append(row)
              baseline_parts.append(f"bs{bs}={'%.4f' % row['baseline_ms'] if row['baseline_ms'] is not None else 'n/a'}")
              current_parts.append(f"bs{bs}={'%.4f' % row['current_ms'] if row['current_ms'] is not None else 'n/a'}")
              threshold_parts.append(f"bs{bs}={'%.4f' % row['threshold_ms'] if row['threshold_ms'] is not None else 'n/a'}")
              metrics_parts.append(
                  "bs{bs}: baseline={base}, current={cur}, threshold={thr}, result={result}".format(
                      bs=bs,
                      base="%.4f" % row["baseline_ms"] if row["baseline_ms"] is not None else "n/a",
                      cur="%.4f" % row["current_ms"] if row["current_ms"] is not None else "n/a",
                      thr="%.4f" % row["threshold_ms"] if row["threshold_ms"] is not None else "n/a",
                      result=row["result"],
                  )
              )

          if errors:
              overall_status = "failure"

          with open(detail_file, "w", encoding="utf-8") as f:
              f.write("| BS | Baseline (ms) | Current (ms) | Threshold (ms) | Result | Detail |\n")
              f.write("|----|---------------|--------------|----------------|--------|--------|\n")
              for row in result_rows:
                  f.write(
                      "| {bs} | {base} | {cur} | {thr} | {result} | {detail} |\n".format(
                          bs=row["batch_size"],
                          base="%.4f" % row["baseline_ms"] if row["baseline_ms"] is not None else "n/a",
                          cur="%.4f" % row["current_ms"] if row["current_ms"] is not None else "n/a",
                          thr="%.4f" % row["threshold_ms"] if row["threshold_ms"] is not None else "n/a",
                          result=row["result"],
                          detail=(row["detail"] or "").replace("|", "/"),
                      )
                  )
              if errors:
                  f.write("\nAdditional errors:\n")
                  for err in errors:
                      f.write(f"- {err}\n")

          payload = {
              "name": "BD Model 2",
              "kind": "batch_compare",
              "status": overall_status,
              "baseline_ms": ", ".join(baseline_parts) if baseline_parts else "",
              "current_ms": ", ".join(current_parts) if current_parts else "",
              "threshold_ms": ", ".join(threshold_parts) if threshold_parts else "",
              "metrics_summary": " ; ".join(metrics_parts) if metrics_parts else "",
              "rows": result_rows,
          }
          with open(compare_json, "w", encoding="utf-8") as f:
              json.dump(payload, f, ensure_ascii=False, indent=2)
          PY

          STATUS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("status","failure"))' "$COMPARE_JSON")"
          BASELINE_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("baseline_ms",""))' "$COMPARE_JSON")"
          CURRENT_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("current_ms",""))' "$COMPARE_JSON")"
          THRESHOLD_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("threshold_ms",""))' "$COMPARE_JSON")"
          METRICS_SUMMARY="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("metrics_summary",""))' "$COMPARE_JSON")"
          METRICS_JSON="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(json.dumps(data, ensure_ascii=False, separators=(",", ":")))' "$COMPARE_JSON")"

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" != "0" || "${{ steps.run_perf.outputs.current_exit }}" != "0" ]]; then
            STATUS="failure"
          fi

          {
            echo "## BD Model 2"
            echo
            echo "- Status: $STATUS"
            echo "- Runner name: $RUNNER_NAME"
            echo "- Host name: $HOST_NAME"
            echo "- Host IP: $HOST_IP"
            echo "- Host SN: $HOST_SN"
            echo "- Log artifact: logs-bd-model2"
            echo "- Baseline: ${BASELINE_MS:-n/a}"
            echo "- Current: ${CURRENT_MS:-n/a}"
            echo "- Threshold: ${THRESHOLD_MS:-n/a}"
            echo
            echo "### Batch Size Comparison"
            cat "$DETAIL_FILE"
            echo
            echo "### Baseline log tail"
            echo '```text'
            if [[ -f "$BASELINE_LOG" ]]; then
              tail -20 "$BASELINE_LOG" || true
            else
              echo "baseline log not found"
            fi
            echo '```'
            echo
            echo "### Current log tail"
            echo '```text'
            if [[ -f "$CURRENT_LOG" ]]; then
              tail -20 "$CURRENT_LOG" || true
            else
              echo "current log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "baseline_ms=$BASELINE_MS" >> "$GITHUB_OUTPUT"
          echo "current_ms=$CURRENT_MS" >> "$GITHUB_OUTPUT"
          echo "threshold_ms=$THRESHOLD_MS" >> "$GITHUB_OUTPUT"
          echo "metrics_summary=$METRICS_SUMMARY" >> "$GITHUB_OUTPUT"
          {
            echo "metrics_json<<EOF"
            echo "$METRICS_JSON"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if model 2 regressed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  bd_model3:
    name: BD Model 3
    needs: [prepare_metadata, build_current, build_baseline]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success' && needs.build_baseline.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 75
    outputs:
      status: ${{ steps.collect.outputs.status }}
      baseline_ms: ${{ steps.collect.outputs.baseline_ms }}
      current_ms: ${{ steps.collect.outputs.current_ms }}
      threshold_ms: ${{ steps.collect.outputs.threshold_ms }}
      metrics_summary: ${{ steps.collect.outputs.metrics_summary }}
      metrics_json: ${{ steps.collect.outputs.metrics_json }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Run model 3 baseline/current
        id: run_perf
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/bd-model3"
          BASELINE_LOG="$JOB_LOG_DIR/bd_model3_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/bd_model3_current.log"
          SPEC_PATH="$MODEL_ROOT/inference/metaGraph/meta_graph/meta_graph_3.spec"
          mkdir -p "$JOB_LOG_DIR"
          rm -f "$BASELINE_LOG" "$CURRENT_LOG"
          rm -rf "$JOB_LOG_DIR/baseline-out" "$JOB_LOG_DIR/current-out"

          baseline_exit=0
          current_exit=0
          guard_failed=0
          if [[ -z "${GPU_LOCK_FILE:-}" ]]; then
            echo "GPU_LOCK_FILE is not set; refusing to use a global GPU lock." | tee -a "$BASELINE_LOG"
            guard_failed=1
            lock_file=""
          else
            lock_file="$GPU_LOCK_FILE"
          fi

          export MODEL_ROOT JOB_LOG_DIR BASELINE_LOG CURRENT_LOG SPEC_PATH

          if [[ "$guard_failed" == "0" ]]; then
            exec 9>"$lock_file"
            flock 9
            echo "Acquired GPU test lock: $lock_file" | tee -a "$BASELINE_LOG"

            pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/baseline/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$BASELINE_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/metaGraph"
              MUSA_PINNED_FEED=1 \
              MUSA_PINNED_H2D_ON_COMPUTE_STREAM=1 \
              timeout 30m python musa_run_pb_graph.py \
                --spec "$SPEC_PATH" \
                --bs 32,128,256,1024 \
                --run_iters 30 \
                --out_root "$JOB_LOG_DIR/baseline-out" 2>&1 | tee "$BASELINE_LOG"
            ' || baseline_exit=$?

            pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/current/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$CURRENT_LOG"

            bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/inference/metaGraph"
              MUSA_PINNED_FEED=1 \
              MUSA_PINNED_H2D_ON_COMPUTE_STREAM=1 \
              timeout 30m python musa_run_pb_graph.py \
                --spec "$SPEC_PATH" \
                --bs 32,128,256,1024 \
                --run_iters 30 \
                --out_root "$JOB_LOG_DIR/current-out" 2>&1 | tee "$CURRENT_LOG"
            ' || current_exit=$?
          else
            baseline_exit=1
            current_exit=1
          fi

          echo "baseline_exit=$baseline_exit" >> "$GITHUB_OUTPUT"
          echo "current_exit=$current_exit" >> "$GITHUB_OUTPUT"

      - name: Collect model 3 summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/bd-model3"
          BASELINE_LOG="$JOB_LOG_DIR/bd_model3_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/bd_model3_current.log"
          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/bd-model3.md"
          COMPARE_JSON="$RUNNER_TEMP/bd-model3-compare.json"
          DETAIL_FILE="$RUNNER_TEMP/bd-model3-details.md"
          STATUS="success"
          BASELINE_MS=""
          CURRENT_MS=""
          THRESHOLD_MS=""
          METRICS_SUMMARY=""
          METRICS_JSON=""
          BASELINE_REPORT=""
          CURRENT_REPORT=""
          RUNNER_NAME="${RUNNER_NAME:-unknown}"
          HOST_NAME="$(hostname)"
          HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
          HOST_SN="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || true)"

          HOST_IP="${HOST_IP:-unavailable}"
          HOST_SN="${HOST_SN:-unavailable}"

          if [[ -f "$BASELINE_LOG" ]]; then
            REPORT_PATH="$(grep '^\[OK\] report=' "$BASELINE_LOG" | tail -1 | sed 's/^\[OK\] report=//' || true)"
            if [[ -n "$REPORT_PATH" && -f "$REPORT_PATH" ]]; then
              BASELINE_REPORT="$REPORT_PATH"
            fi
          fi

          if [[ -f "$CURRENT_LOG" ]]; then
            REPORT_PATH="$(grep '^\[OK\] report=' "$CURRENT_LOG" | tail -1 | sed 's/^\[OK\] report=//' || true)"
            if [[ -n "$REPORT_PATH" && -f "$REPORT_PATH" ]]; then
              CURRENT_REPORT="$REPORT_PATH"
            fi
          fi

          python3 - "$BASELINE_REPORT" "$CURRENT_REPORT" "$COMPARE_JSON" "$DETAIL_FILE" <<'PY'
          import json
          import os
          import sys

          baseline_report, current_report, compare_json, detail_file = sys.argv[1:5]
          THRESHOLD_RATIO = 1.05

          def load_report(path, label):
              if not path:
                  return None, [f"{label} report path missing"]
              if not os.path.isfile(path):
                  return None, [f"{label} report not found: {path}"]
              try:
                  with open(path, "r", encoding="utf-8") as f:
                      return json.load(f), []
              except Exception as exc:
                  return None, [f"failed to load {label} report {path}: {exc}"]

          def collect_rows(report, label):
              rows = {}
              errors = []
              if report is None:
                  return rows, errors
              for item in report.get("average_time_summary", []):
                  bs = str(item.get("batch_size"))
                  if not bs or bs == "None":
                      errors.append(f"{label} report contains empty batch_size entry")
                      continue
                  if bs in rows:
                      errors.append(f"{label} report contains duplicate batch_size={bs}")
                      continue
                  rows[bs] = {
                      "status": item.get("status"),
                      "average_time_ms": item.get("trimmed_avg_ms")
                      if item.get("trimmed_avg_ms") is not None
                      else item.get("average_time_ms"),
                      "error_core": item.get("error_core"),
                  }
              if not rows:
                  errors.append(f"{label} report has no average_time_summary entries")
              return rows, errors

          baseline_data, errors = load_report(baseline_report, "baseline")
          current_data, cur_errors = load_report(current_report, "current")
          errors.extend(cur_errors)

          baseline_rows, cur_errors = collect_rows(baseline_data, "baseline")
          errors.extend(cur_errors)
          current_rows, cur_errors = collect_rows(current_data, "current")
          errors.extend(cur_errors)

          def sort_key(bs):
              try:
                  return (0, int(bs))
              except Exception:
                  return (1, str(bs))

          all_bs = sorted(set(baseline_rows) | set(current_rows), key=sort_key)
          result_rows = []
          baseline_parts = []
          current_parts = []
          threshold_parts = []
          metrics_parts = []
          overall_status = "success"

          for bs in all_bs:
              base = baseline_rows.get(bs)
              cur = current_rows.get(bs)
              row = {
                  "batch_size": bs,
                  "baseline_ms": None,
                  "current_ms": None,
                  "threshold_ms": None,
                  "result": "ok",
                  "detail": "",
              }

              if base is None:
                  row["result"] = "missing-baseline"
                  row["detail"] = "baseline report missing this batch size"
              elif cur is None:
                  row["result"] = "missing-current"
                  row["detail"] = "current report missing this batch size"
              elif base.get("status") != "ok" or base.get("average_time_ms") is None:
                  row["result"] = "baseline-failed"
                  row["detail"] = base.get("error_core") or f"baseline status={base.get('status')}"
              elif cur.get("status") != "ok" or cur.get("average_time_ms") is None:
                  row["result"] = "current-failed"
                  row["detail"] = cur.get("error_core") or f"current status={cur.get('status')}"
              else:
                  row["baseline_ms"] = float(base["average_time_ms"])
                  row["current_ms"] = float(cur["average_time_ms"])
                  row["threshold_ms"] = float(base["average_time_ms"]) * THRESHOLD_RATIO
                  if row["current_ms"] > row["threshold_ms"]:
                      row["result"] = "regressed"
                      row["detail"] = "current > baseline * 1.05"

              if row["result"] != "ok":
                  overall_status = "failure"

              result_rows.append(row)
              baseline_parts.append(f"bs{bs}={'%.4f' % row['baseline_ms'] if row['baseline_ms'] is not None else 'n/a'}")
              current_parts.append(f"bs{bs}={'%.4f' % row['current_ms'] if row['current_ms'] is not None else 'n/a'}")
              threshold_parts.append(f"bs{bs}={'%.4f' % row['threshold_ms'] if row['threshold_ms'] is not None else 'n/a'}")
              metrics_parts.append(
                  "bs{bs}: baseline={base}, current={cur}, threshold={thr}, result={result}".format(
                      bs=bs,
                      base="%.4f" % row["baseline_ms"] if row["baseline_ms"] is not None else "n/a",
                      cur="%.4f" % row["current_ms"] if row["current_ms"] is not None else "n/a",
                      thr="%.4f" % row["threshold_ms"] if row["threshold_ms"] is not None else "n/a",
                      result=row["result"],
                  )
              )

          if errors:
              overall_status = "failure"

          with open(detail_file, "w", encoding="utf-8") as f:
              f.write("| BS | Baseline (ms) | Current (ms) | Threshold (ms) | Result | Detail |\n")
              f.write("|----|---------------|--------------|----------------|--------|--------|\n")
              for row in result_rows:
                  f.write(
                      "| {bs} | {base} | {cur} | {thr} | {result} | {detail} |\n".format(
                          bs=row["batch_size"],
                          base="%.4f" % row["baseline_ms"] if row["baseline_ms"] is not None else "n/a",
                          cur="%.4f" % row["current_ms"] if row["current_ms"] is not None else "n/a",
                          thr="%.4f" % row["threshold_ms"] if row["threshold_ms"] is not None else "n/a",
                          result=row["result"],
                          detail=(row["detail"] or "").replace("|", "/"),
                      )
                  )
              if errors:
                  f.write("\nAdditional errors:\n")
                  for err in errors:
                      f.write(f"- {err}\n")

          payload = {
              "name": "BD Model 3",
              "kind": "batch_compare",
              "status": overall_status,
              "baseline_ms": ", ".join(baseline_parts) if baseline_parts else "",
              "current_ms": ", ".join(current_parts) if current_parts else "",
              "threshold_ms": ", ".join(threshold_parts) if threshold_parts else "",
              "metrics_summary": " ; ".join(metrics_parts) if metrics_parts else "",
              "rows": result_rows,
          }
          with open(compare_json, "w", encoding="utf-8") as f:
              json.dump(payload, f, ensure_ascii=False, indent=2)
          PY

          STATUS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("status","failure"))' "$COMPARE_JSON")"
          BASELINE_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("baseline_ms",""))' "$COMPARE_JSON")"
          CURRENT_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("current_ms",""))' "$COMPARE_JSON")"
          THRESHOLD_MS="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("threshold_ms",""))' "$COMPARE_JSON")"
          METRICS_SUMMARY="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(data.get("metrics_summary",""))' "$COMPARE_JSON")"
          METRICS_JSON="$(python3 -c 'import json,sys; data=json.load(open(sys.argv[1], encoding="utf-8")); print(json.dumps(data, ensure_ascii=False, separators=(",", ":")))' "$COMPARE_JSON")"

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" != "0" || "${{ steps.run_perf.outputs.current_exit }}" != "0" ]]; then
            STATUS="failure"
          fi

          {
            echo "## BD Model 3"
            echo
            echo "- Status: $STATUS"
            echo "- Runner name: $RUNNER_NAME"
            echo "- Host name: $HOST_NAME"
            echo "- Host IP: $HOST_IP"
            echo "- Host SN: $HOST_SN"
            echo "- Log artifact: logs-bd-model3"
            echo "- Baseline: ${BASELINE_MS:-n/a}"
            echo "- Current: ${CURRENT_MS:-n/a}"
            echo "- Threshold: ${THRESHOLD_MS:-n/a}"
            echo
            echo "### Batch Size Comparison"
            cat "$DETAIL_FILE"
            echo
            echo "### Baseline log tail"
            echo '```text'
            if [[ -f "$BASELINE_LOG" ]]; then
              tail -20 "$BASELINE_LOG" || true
            else
              echo "baseline log not found"
            fi
            echo '```'
            echo
            echo "### Current log tail"
            echo '```text'
            if [[ -f "$CURRENT_LOG" ]]; then
              tail -20 "$CURRENT_LOG" || true
            else
              echo "current log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "baseline_ms=$BASELINE_MS" >> "$GITHUB_OUTPUT"
          echo "current_ms=$CURRENT_MS" >> "$GITHUB_OUTPUT"
          echo "threshold_ms=$THRESHOLD_MS" >> "$GITHUB_OUTPUT"
          echo "metrics_summary=$METRICS_SUMMARY" >> "$GITHUB_OUTPUT"
          {
            echo "metrics_json<<EOF"
            echo "$METRICS_JSON"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if model 3 regressed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  onetrans_perf:
    name: OneTrans Performance
    needs: [prepare_metadata, build_current, build_baseline]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success' && needs.build_baseline.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 120
    outputs:
      status: ${{ steps.collect.outputs.status }}
      baseline_ms: ${{ steps.collect.outputs.baseline_ms }}
      current_ms: ${{ steps.collect.outputs.current_ms }}
      threshold_ms: ${{ steps.collect.outputs.threshold_ms }}
      metrics_summary: ${{ steps.collect.outputs.metrics_summary }}
      metrics_json: ${{ steps.collect.outputs.metrics_json }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Run OneTrans performance baseline/current
        id: run_perf
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/onetrans-perf"
          BASELINE_LOG="$JOB_LOG_DIR/onetrans_perf_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/onetrans_perf_current.log"
          ONETRANS_PERF_ROOT="${ONETRANS_PERF_ROOT:-$MODEL_ROOT/training/onetrans}"
          ONETRANS_PERF_SCRIPT_NAME="${ONETRANS_PERF_SCRIPT_NAME:-train_perf_bf16.py}"
          ONETRANS_PERF_SCRIPT="$ONETRANS_PERF_ROOT/$ONETRANS_PERF_SCRIPT_NAME"
          ONETRANS_PERF_DATA_DIR="${ONETRANS_PERF_DATA_DIR:-$ONETRANS_PERF_ROOT/data}"
          ONETRANS_PERF_WARMUP_STEPS="${ONETRANS_PERF_WARMUP_STEPS:-20}"
          ONETRANS_PERF_MEASURE_STEPS="${ONETRANS_PERF_MEASURE_STEPS:-50}"
          ONETRANS_PERF_TIMEOUT="${ONETRANS_PERF_TIMEOUT:-60m}"
          mkdir -p "$JOB_LOG_DIR" "$ONETRANS_PERF_ROOT"
          rm -f "$BASELINE_LOG" "$CURRENT_LOG"

          baseline_exit=0
          current_exit=0
          guard_failed=0
          if [[ -z "${GPU_LOCK_FILE:-}" ]]; then
            echo "GPU_LOCK_FILE is not set; refusing to use a global GPU lock." | tee -a "$BASELINE_LOG"
            guard_failed=1
            lock_file=""
          else
            lock_file="$GPU_LOCK_FILE"
          fi

          if [[ ! -d "$ONETRANS_PERF_ROOT" ]]; then
            echo "OneTrans training root not found: $ONETRANS_PERF_ROOT" | tee -a "$BASELINE_LOG"
            guard_failed=1
          fi
          if [[ ! -d "$ONETRANS_PERF_ROOT/model" || ! -d "$ONETRANS_PERF_ROOT/data" ]]; then
            echo "OneTrans model/data directories not found under: $ONETRANS_PERF_ROOT" | tee -a "$BASELINE_LOG"
            guard_failed=1
          fi
          if [[ ! -f "$ONETRANS_PERF_SCRIPT" ]]; then
            echo "OneTrans performance script missing: $ONETRANS_PERF_SCRIPT" | tee -a "$BASELINE_LOG"
            guard_failed=1
          fi

          ONETRANS_PERF_DATA_FILE="${ONETRANS_PERF_DATA_FILE:-}"
          if [[ -z "$ONETRANS_PERF_DATA_FILE" && -f "$ONETRANS_PERF_ROOT/data/kaggleAdDisplayChallenge_processed_sub.npz" ]]; then
            ONETRANS_PERF_DATA_FILE="$ONETRANS_PERF_ROOT/data/kaggleAdDisplayChallenge_processed_sub.npz"
          fi
          if [[ -z "$ONETRANS_PERF_DATA_FILE" && -d "$ONETRANS_PERF_DATA_DIR" ]]; then
            ONETRANS_PERF_DATA_FILE="$(find "$ONETRANS_PERF_DATA_DIR" -maxdepth 2 -type f -name '*.npz' | sort | head -n 1)"
          fi
          if [[ -z "$ONETRANS_PERF_DATA_FILE" || ! -f "$ONETRANS_PERF_DATA_FILE" ]]; then
            echo "OneTrans performance data npz not found under: $ONETRANS_PERF_DATA_DIR" | tee -a "$BASELINE_LOG"
            guard_failed=1
          fi

          run_onetrans_perf() {
            local wheel_dir="$1"
            local log_file="$2"
            local label="$3"

            set -o pipefail
            pip install "$wheel_dir"/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$log_file" || return $?

            python3 - <<'PY' 2>&1 | tee -a "$log_file" || return $?
          import pathlib
          import tensorflow_musa

          package_root = pathlib.Path(tensorflow_musa.__file__).resolve().parent
          print(f"tensorflow_musa wheel: {package_root}")
          PY

            export ONETRANS_PERF_LABEL="$label"

            bash -l -c '
              set -o pipefail
              cd "$ONETRANS_PERF_ROOT"
              echo "Running OneTrans performance: $ONETRANS_PERF_LABEL"
              echo "Script: $ONETRANS_PERF_SCRIPT"
              echo "Data file: $ONETRANS_PERF_DATA_FILE"
              timeout "$ONETRANS_PERF_TIMEOUT" python -u "$ONETRANS_PERF_SCRIPT" \
                --data_path "$ONETRANS_PERF_DATA_FILE" \
                --warmup_steps "$ONETRANS_PERF_WARMUP_STEPS" \
                --measure_steps "$ONETRANS_PERF_MEASURE_STEPS" \
                --max_steps "$((ONETRANS_PERF_WARMUP_STEPS + ONETRANS_PERF_MEASURE_STEPS))"
            ' 2>&1 | tee -a "$log_file"
          }

          export ONETRANS_PERF_ROOT ONETRANS_PERF_SCRIPT ONETRANS_PERF_DATA_FILE ONETRANS_PERF_WARMUP_STEPS ONETRANS_PERF_MEASURE_STEPS ONETRANS_PERF_TIMEOUT

          if [[ "$guard_failed" == "0" ]]; then
            exec 9>"$lock_file"
            flock 9
            echo "Acquired GPU test lock: $lock_file" | tee -a "$BASELINE_LOG"

            run_onetrans_perf "${{ needs.prepare_metadata.outputs.log_root }}/wheels/baseline" "$BASELINE_LOG" "baseline" || baseline_exit=$?
            run_onetrans_perf "${{ needs.prepare_metadata.outputs.log_root }}/wheels/current" "$CURRENT_LOG" "current" || current_exit=$?
          else
            baseline_exit=1
            current_exit=1
          fi

          echo "baseline_exit=$baseline_exit" >> "$GITHUB_OUTPUT"
          echo "current_exit=$current_exit" >> "$GITHUB_OUTPUT"

      - name: Collect OneTrans performance summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          JOB_LOG_DIR="$LOG_ROOT/onetrans-perf"
          BASELINE_LOG="$JOB_LOG_DIR/onetrans_perf_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/onetrans_perf_current.log"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/onetrans-perf.md"
          STATUS="success"
          BASELINE_MS=""
          CURRENT_MS=""
          THRESHOLD_MS=""
          METRICS_SUMMARY=""
          METRICS_JSON=""
          RUNNER_NAME="${RUNNER_NAME:-unknown}"
          HOST_NAME="$(hostname)"
          HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
          HOST_SN="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || true)"

          HOST_IP="${HOST_IP:-unavailable}"
          HOST_SN="${HOST_SN:-unavailable}"

          parse_avg_ms() {
            python3 -c 'import pathlib,re,sys; text=pathlib.Path(sys.argv[1]).read_text(encoding="utf-8", errors="ignore"); matches=re.findall(r"Average ms/full step:\s*([0-9]+(?:\.[0-9]+)?)", text); print(matches[-1] if matches else "")' "$1" || true
          }

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" == "0" && -f "$BASELINE_LOG" ]]; then
            BASELINE_MS="$(parse_avg_ms "$BASELINE_LOG")"
          fi
          if [[ "${{ steps.run_perf.outputs.current_exit }}" == "0" && -f "$CURRENT_LOG" ]]; then
            CURRENT_MS="$(parse_avg_ms "$CURRENT_LOG")"
          fi

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" != "0" || "${{ steps.run_perf.outputs.current_exit }}" != "0" || -z "$BASELINE_MS" || -z "$CURRENT_MS" ]]; then
            STATUS="failure"
          else
            THRESHOLD_MS="$(awk -v base="$BASELINE_MS" 'BEGIN {printf "%.4f", base * 1.05}')"
            if awk -v cur="$CURRENT_MS" -v thr="$THRESHOLD_MS" 'BEGIN {exit !(cur > thr)}'; then
              STATUS="failure"
            fi
          fi

          METRICS_SUMMARY="baseline=${BASELINE_MS:-n/a} ms, current=${CURRENT_MS:-n/a} ms, threshold=${THRESHOLD_MS:-n/a} ms, result=${STATUS}"
          METRICS_JSON="$(python3 - "$STATUS" "$BASELINE_MS" "$CURRENT_MS" "$THRESHOLD_MS" <<'PY'
          import json
          import sys

          status, baseline_ms, current_ms, threshold_ms = sys.argv[1:5]
          payload = {
              "name": "OneTrans Performance",
              "kind": "single",
              "status": status,
              "baseline_ms": baseline_ms or "n/a",
              "current_ms": current_ms or "n/a",
              "threshold_ms": threshold_ms or "n/a",
          }
          print(json.dumps(payload, ensure_ascii=False, separators=(",", ":")))
          PY
          )"

          {
            echo "## OneTrans Performance"
            echo
            echo "- Status: $STATUS"
            echo "- Runner name: $RUNNER_NAME"
            echo "- Host name: $HOST_NAME"
            echo "- Host IP: $HOST_IP"
            echo "- Host SN: $HOST_SN"
            echo "- Log artifact: logs-onetrans-perf"
            echo "- Baseline: ${BASELINE_MS:-n/a} ms"
            echo "- Current: ${CURRENT_MS:-n/a} ms"
            echo "- Threshold: ${THRESHOLD_MS:-n/a} ms"
            echo
            echo "### Baseline log tail"
            echo '```text'
            if [[ -f "$BASELINE_LOG" ]]; then
              tail -20 "$BASELINE_LOG" || true
            else
              echo "baseline log not found"
            fi
            echo '```'
            echo
            echo "### Current log tail"
            echo '```text'
            if [[ -f "$CURRENT_LOG" ]]; then
              tail -20 "$CURRENT_LOG" || true
            else
              echo "current log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "baseline_ms=$BASELINE_MS" >> "$GITHUB_OUTPUT"
          echo "current_ms=$CURRENT_MS" >> "$GITHUB_OUTPUT"
          echo "threshold_ms=$THRESHOLD_MS" >> "$GITHUB_OUTPUT"
          echo "metrics_summary=$METRICS_SUMMARY" >> "$GITHUB_OUTPUT"
          {
            echo "metrics_json<<EOF"
            echo "$METRICS_JSON"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if OneTrans performance regressed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  tokenlarge_perf:
    name: TokenMixer Large Performance
    needs: [prepare_metadata, build_current, build_baseline]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success' && needs.build_baseline.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 120
    outputs:
      status: ${{ steps.collect.outputs.status }}
      baseline_ms: ${{ steps.collect.outputs.baseline_ms }}
      current_ms: ${{ steps.collect.outputs.current_ms }}
      threshold_ms: ${{ steps.collect.outputs.threshold_ms }}
      metrics_summary: ${{ steps.collect.outputs.metrics_summary }}
      metrics_json: ${{ steps.collect.outputs.metrics_json }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Run TokenMixer Large performance baseline/current
        id: run_perf
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/tokenlarge-perf"
          BASELINE_LOG="$JOB_LOG_DIR/tokenlarge_perf_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/tokenlarge_perf_current.log"
          TOKENLARGE_PERF_ROOT="${TOKENLARGE_PERF_ROOT:-$MODEL_ROOT/training/tokenmixer-large}"
          TOKENLARGE_PERF_SCRIPT_NAME="${TOKENLARGE_PERF_SCRIPT_NAME:-train_perf_bf16.py}"
          TOKENLARGE_PERF_SCRIPT="$TOKENLARGE_PERF_ROOT/$TOKENLARGE_PERF_SCRIPT_NAME"
          TOKENLARGE_PERF_DATA_DIR="${TOKENLARGE_PERF_DATA_DIR:-$TOKENLARGE_PERF_ROOT/data}"
          TOKENLARGE_PERF_WARMUP_STEPS="${TOKENLARGE_PERF_WARMUP_STEPS:-20}"
          TOKENLARGE_PERF_MEASURE_STEPS="${TOKENLARGE_PERF_MEASURE_STEPS:-50}"
          TOKENLARGE_PERF_TIMEOUT="${TOKENLARGE_PERF_TIMEOUT:-60m}"
          mkdir -p "$JOB_LOG_DIR" "$TOKENLARGE_PERF_ROOT"
          rm -f "$BASELINE_LOG" "$CURRENT_LOG"

          baseline_exit=0
          current_exit=0
          guard_failed=0
          if [[ -z "${GPU_LOCK_FILE:-}" ]]; then
            echo "GPU_LOCK_FILE is not set; refusing to use a global GPU lock." | tee -a "$BASELINE_LOG"
            guard_failed=1
            lock_file=""
          else
            lock_file="$GPU_LOCK_FILE"
          fi

          if [[ ! -d "$TOKENLARGE_PERF_ROOT" ]]; then
            echo "TokenMixer Large training root not found: $TOKENLARGE_PERF_ROOT" | tee -a "$BASELINE_LOG"
            guard_failed=1
          fi
          if [[ ! -d "$TOKENLARGE_PERF_ROOT/model" || ! -d "$TOKENLARGE_PERF_ROOT/data" ]]; then
            echo "TokenMixer Large model/data directories not found under: $TOKENLARGE_PERF_ROOT" | tee -a "$BASELINE_LOG"
            guard_failed=1
          fi
          if [[ ! -f "$TOKENLARGE_PERF_SCRIPT" ]]; then
            echo "TokenMixer Large performance script missing: $TOKENLARGE_PERF_SCRIPT" | tee -a "$BASELINE_LOG"
            guard_failed=1
          fi

          TOKENLARGE_PERF_DATA_FILE="${TOKENLARGE_PERF_DATA_FILE:-}"
          if [[ -z "$TOKENLARGE_PERF_DATA_FILE" && -f "$TOKENLARGE_PERF_ROOT/data/kaggleAdDisplayChallenge_processed_sub.npz" ]]; then
            TOKENLARGE_PERF_DATA_FILE="$TOKENLARGE_PERF_ROOT/data/kaggleAdDisplayChallenge_processed_sub.npz"
          fi
          if [[ -z "$TOKENLARGE_PERF_DATA_FILE" && -d "$TOKENLARGE_PERF_DATA_DIR" ]]; then
            TOKENLARGE_PERF_DATA_FILE="$(find "$TOKENLARGE_PERF_DATA_DIR" -maxdepth 2 -type f -name '*.npz' | sort | head -n 1)"
          fi
          if [[ -z "$TOKENLARGE_PERF_DATA_FILE" || ! -f "$TOKENLARGE_PERF_DATA_FILE" ]]; then
            echo "TokenMixer Large performance data npz not found under: $TOKENLARGE_PERF_DATA_DIR" | tee -a "$BASELINE_LOG"
            guard_failed=1
          fi

          run_tokenlarge_perf() {
            local wheel_dir="$1"
            local log_file="$2"
            local label="$3"

            set -o pipefail
            pip install "$wheel_dir"/tensorflow_musa-*.whl --no-deps --force-reinstall 2>&1 | tee -a "$log_file" || return $?

            python3 - <<'PY' 2>&1 | tee -a "$log_file" || return $?
          import pathlib
          import tensorflow_musa

          package_root = pathlib.Path(tensorflow_musa.__file__).resolve().parent
          print(f"tensorflow_musa wheel: {package_root}")
          PY

            export TOKENLARGE_PERF_LABEL="$label"

            bash -l -c '
              set -o pipefail
              cd "$TOKENLARGE_PERF_ROOT"
              echo "Running TokenMixer Large performance: $TOKENLARGE_PERF_LABEL"
              echo "Script: $TOKENLARGE_PERF_SCRIPT"
              echo "Data file: $TOKENLARGE_PERF_DATA_FILE"
              timeout "$TOKENLARGE_PERF_TIMEOUT" python -u "$TOKENLARGE_PERF_SCRIPT" \
                --data_path "$TOKENLARGE_PERF_DATA_FILE" \
                --warmup_steps "$TOKENLARGE_PERF_WARMUP_STEPS" \
                --measure_steps "$TOKENLARGE_PERF_MEASURE_STEPS" \
                --max_steps "$((TOKENLARGE_PERF_WARMUP_STEPS + TOKENLARGE_PERF_MEASURE_STEPS))"
            ' 2>&1 | tee -a "$log_file"
          }

          export TOKENLARGE_PERF_ROOT TOKENLARGE_PERF_SCRIPT TOKENLARGE_PERF_DATA_FILE TOKENLARGE_PERF_WARMUP_STEPS TOKENLARGE_PERF_MEASURE_STEPS TOKENLARGE_PERF_TIMEOUT

          if [[ "$guard_failed" == "0" ]]; then
            exec 9>"$lock_file"
            flock 9
            echo "Acquired GPU test lock: $lock_file" | tee -a "$BASELINE_LOG"

            run_tokenlarge_perf "${{ needs.prepare_metadata.outputs.log_root }}/wheels/baseline" "$BASELINE_LOG" "baseline" || baseline_exit=$?
            run_tokenlarge_perf "${{ needs.prepare_metadata.outputs.log_root }}/wheels/current" "$CURRENT_LOG" "current" || current_exit=$?
          else
            baseline_exit=1
            current_exit=1
          fi

          echo "baseline_exit=$baseline_exit" >> "$GITHUB_OUTPUT"
          echo "current_exit=$current_exit" >> "$GITHUB_OUTPUT"

      - name: Collect TokenMixer Large performance summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          JOB_LOG_DIR="$LOG_ROOT/tokenlarge-perf"
          BASELINE_LOG="$JOB_LOG_DIR/tokenlarge_perf_baseline.log"
          CURRENT_LOG="$JOB_LOG_DIR/tokenlarge_perf_current.log"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/tokenlarge-perf.md"
          STATUS="success"
          BASELINE_MS=""
          CURRENT_MS=""
          THRESHOLD_MS=""
          METRICS_SUMMARY=""
          METRICS_JSON=""
          RUNNER_NAME="${RUNNER_NAME:-unknown}"
          HOST_NAME="$(hostname)"
          HOST_IP="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
          HOST_SN="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || true)"

          HOST_IP="${HOST_IP:-unavailable}"
          HOST_SN="${HOST_SN:-unavailable}"

          parse_avg_ms() {
            python3 -c 'import pathlib,re,sys; text=pathlib.Path(sys.argv[1]).read_text(encoding="utf-8", errors="ignore"); matches=re.findall(r"Average ms/full step:\s*([0-9]+(?:\.[0-9]+)?)", text); print(matches[-1] if matches else "")' "$1" || true
          }

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" == "0" && -f "$BASELINE_LOG" ]]; then
            BASELINE_MS="$(parse_avg_ms "$BASELINE_LOG")"
          fi
          if [[ "${{ steps.run_perf.outputs.current_exit }}" == "0" && -f "$CURRENT_LOG" ]]; then
            CURRENT_MS="$(parse_avg_ms "$CURRENT_LOG")"
          fi

          if [[ "${{ steps.run_perf.outputs.baseline_exit }}" != "0" || "${{ steps.run_perf.outputs.current_exit }}" != "0" || -z "$BASELINE_MS" || -z "$CURRENT_MS" ]]; then
            STATUS="failure"
          else
            THRESHOLD_MS="$(awk -v base="$BASELINE_MS" 'BEGIN {printf "%.4f", base * 1.05}')"
            if awk -v cur="$CURRENT_MS" -v thr="$THRESHOLD_MS" 'BEGIN {exit !(cur > thr)}'; then
              STATUS="failure"
            fi
          fi

          METRICS_SUMMARY="baseline=${BASELINE_MS:-n/a} ms, current=${CURRENT_MS:-n/a} ms, threshold=${THRESHOLD_MS:-n/a} ms, result=${STATUS}"
          METRICS_JSON="$(python3 - "$STATUS" "$BASELINE_MS" "$CURRENT_MS" "$THRESHOLD_MS" <<'PY'
          import json
          import sys

          status, baseline_ms, current_ms, threshold_ms = sys.argv[1:5]
          payload = {
              "name": "TokenMixer Large Performance",
              "kind": "single",
              "status": status,
              "baseline_ms": baseline_ms or "n/a",
              "current_ms": current_ms or "n/a",
              "threshold_ms": threshold_ms or "n/a",
          }
          print(json.dumps(payload, ensure_ascii=False, separators=(",", ":")))
          PY
          )"

          {
            echo "## TokenMixer Large Performance"
            echo
            echo "- Status: $STATUS"
            echo "- Runner name: $RUNNER_NAME"
            echo "- Host name: $HOST_NAME"
            echo "- Host IP: $HOST_IP"
            echo "- Host SN: $HOST_SN"
            echo "- Log artifact: logs-tokenlarge-perf"
            echo "- Baseline: ${BASELINE_MS:-n/a} ms"
            echo "- Current: ${CURRENT_MS:-n/a} ms"
            echo "- Threshold: ${THRESHOLD_MS:-n/a} ms"
            echo
            echo "### Baseline log tail"
            echo '```text'
            if [[ -f "$BASELINE_LOG" ]]; then
              tail -20 "$BASELINE_LOG" || true
            else
              echo "baseline log not found"
            fi
            echo '```'
            echo
            echo "### Current log tail"
            echo '```text'
            if [[ -f "$CURRENT_LOG" ]]; then
              tail -20 "$CURRENT_LOG" || true
            else
              echo "current log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "baseline_ms=$BASELINE_MS" >> "$GITHUB_OUTPUT"
          echo "current_ms=$CURRENT_MS" >> "$GITHUB_OUTPUT"
          echo "threshold_ms=$THRESHOLD_MS" >> "$GITHUB_OUTPUT"
          echo "metrics_summary=$METRICS_SUMMARY" >> "$GITHUB_OUTPUT"
          {
            echo "metrics_json<<EOF"
            echo "$METRICS_JSON"
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if TokenMixer Large performance regressed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  training:
    name: Training
    needs: [prepare_metadata, build_current]
    if: needs.prepare_metadata.result == 'success' && needs.build_current.result == 'success'
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 210
    outputs:
      status: ${{ steps.collect.outputs.status }}
      summary_file: ${{ steps.collect.outputs.summary_file }}
    steps:
      - name: Install current wheel
        run: pip install "${{ needs.prepare_metadata.outputs.log_root }}"/wheels/current/tensorflow_musa-*.whl --no-deps --force-reinstall

      - name: Run training tests
        id: run_training
        run: |
          set +e

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/training"
          LOG_FILE="$JOB_LOG_DIR/training_tests.log"
          ERROR_LOG_DIR="$JOB_LOG_DIR/error_logs"
          pr_training_models=(rankmixer)
          daily_training_models=(rankmixer onetrans tokenmixer-large afm autoint bst ccpm dcn dcnmix deepfefm deepfm dien difm din dsin edcn esmm fgcnn fibinet flen fnn fwfm ifm mlr mmoe nfm onn ple pnn wdl wukong xdeepfm)
          mkdir -p "$JOB_LOG_DIR"
          rm -f "$LOG_FILE"
          rm -rf "$ERROR_LOG_DIR"

          training_exit=0
          guard_failed=0
          if [[ -z "${GPU_LOCK_FILE:-}" ]]; then
            echo "GPU_LOCK_FILE is not set; refusing to use a global GPU lock." | tee -a "$LOG_FILE"
            guard_failed=1
            lock_file=""
          else
            lock_file="$GPU_LOCK_FILE"
          fi

          if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            selected_training_models=("${daily_training_models[@]}")
            training_mode="daily"
          else
            selected_training_models=("${pr_training_models[@]}")
            training_mode="pr-smoke"
          fi
          TRAINING_MODELS="${selected_training_models[*]}"
          printf 'Training mode: %s\n' "$training_mode" | tee -a "$LOG_FILE"
          printf 'Selected models (%d): %s\n' "${#selected_training_models[@]}" "$TRAINING_MODELS" | tee -a "$LOG_FILE"

          LIVE_TRAINING_WRAPPER="$RUNNER_TEMP/run_training_live.py"
          cat > "$LIVE_TRAINING_WRAPPER" <<'PY'
          import os
          import subprocess
          import selectors
          import sys
          import time

          sys.path.insert(0, os.getcwd())
          import run_all_training_tests as runner

          _real_run = subprocess.run


          def run_live(cmd, capture_output=False, text=False, cwd=None, env=None, timeout=None, **kwargs):
              if not capture_output:
                  return _real_run(cmd, capture_output=capture_output, text=text, cwd=cwd, env=env, timeout=timeout, **kwargs)

              effective_timeout = int(env.get("TRAINING_MODEL_TIMEOUT_SECONDS", "3600")) if env else 3600
              proc = subprocess.Popen(
                  cmd,
                  stdout=subprocess.PIPE,
                  stderr=subprocess.STDOUT,
                  text=True,
                  bufsize=1,
                  cwd=cwd,
                  env=env,
              )

              lines = []
              start = time.monotonic()
              selector = selectors.DefaultSelector()
              selector.register(proc.stdout, selectors.EVENT_READ)

              try:
                  while True:
                      for key, _ in selector.select(timeout=0.2):
                          line = key.fileobj.readline()
                          if line:
                              lines.append(line)
                              sys.stdout.write(line)
                              sys.stdout.flush()

                      if proc.poll() is not None:
                          rest = proc.stdout.read()
                          if rest:
                              lines.append(rest)
                              sys.stdout.write(rest)
                              sys.stdout.flush()
                          break

                      if effective_timeout > 0 and time.monotonic() - start > effective_timeout:
                          proc.kill()
                          proc.wait()
                          timeout_msg = f"\n[ERROR] Model command timeout (>{effective_timeout}s)\n"
                          lines.append(timeout_msg)
                          sys.stdout.write(timeout_msg)
                          sys.stdout.flush()
                          return subprocess.CompletedProcess(cmd, 124, stdout="".join(lines), stderr="")
              finally:
                  selector.close()
                  if proc.stdout:
                      proc.stdout.close()

              return subprocess.CompletedProcess(cmd, proc.returncode, stdout="".join(lines), stderr="")


          runner.subprocess.run = run_live
          sys.exit(runner.main())
          PY

          export MODEL_ROOT LOG_FILE ERROR_LOG_DIR LIVE_TRAINING_WRAPPER TRAINING_MODELS
          if [[ "$guard_failed" == "0" ]]; then
            exec 9>"$lock_file"
            flock 9
            echo "Acquired GPU test lock: $lock_file" | tee -a "$LOG_FILE"

            timeout 180m bash -l -c '
              set -o pipefail
              cd "$MODEL_ROOT/training"
              mkdir -p "$ERROR_LOG_DIR"
              read -r -a training_models <<< "$TRAINING_MODELS"

              TRAINING_MODEL_TIMEOUT_SECONDS="${TRAINING_MODEL_TIMEOUT_SECONDS:-3600}" \
                PYTHONUNBUFFERED=1 \
                python -u "$LIVE_TRAINING_WRAPPER" \
                  --epochs 10 \
                  --log-dir "$ERROR_LOG_DIR" \
                  --models "${training_models[@]}" 2>&1 | tee "$LOG_FILE"
            ' || training_exit=$?
          else
            training_exit=1
          fi

          echo "training_exit=$training_exit" >> "$GITHUB_OUTPUT"

      - name: Collect training summary
        id: collect
        if: always()
        run: |
          set -euo pipefail

          JOB_LOG_DIR="${{ needs.prepare_metadata.outputs.log_root }}/training"
          LOG_FILE="$JOB_LOG_DIR/training_tests.log"
          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          mkdir -p "$LOG_ROOT/summaries"
          SUMMARY_FILE="$LOG_ROOT/summaries/training.md"
          STATUS="success"

          if [[ "${{ steps.run_training.outputs.training_exit }}" != "0" ]]; then
            STATUS="failure"
          fi
          if [[ ! -f "$LOG_FILE" ]]; then
            STATUS="failure"
          elif grep -Eq "$EMPTY_TEST_RESULT_PATTERN" "$LOG_FILE" || grep -q '^\[FAIL\]' "$LOG_FILE" || grep -Eq '^失败:[[:space:]]*[1-9][0-9]*' "$LOG_FILE"; then
            STATUS="failure"
          fi

          {
            echo "## Training"
            echo
            echo "- Status: $STATUS"
            echo "- Log file: $LOG_FILE"
            echo
            echo '```text'
            if [[ -f "$LOG_FILE" ]]; then
              grep -E '^\[OK\]|^\[FAIL\]|^总计:|^成功:|^失败:|^  - ' "$LOG_FILE" || tail -80 "$LOG_FILE" || true
            else
              echo "training log not found"
            fi
            echo '```'
          } | tee "$SUMMARY_FILE"

          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"

      - name: Fail job if training failed
        if: always() && steps.collect.outputs.status == 'failure'
        run: exit 1

  summary:
    name: Final Summary
    needs:
      - prepare_metadata
      - format
      - build_current
      - build_baseline
      - integration
      - t_perf
      - t_accuracy
      - bd_model1
      - bd_model2
      - bd_model3
      - onetrans_perf
      - tokenlarge_perf
      - training
    if: ${{ always() && !cancelled() }}
    runs-on: [self-hosted, musa-gpu]
    timeout-minutes: 10
    env:
      T_PERF_JSON: ${{ needs.t_perf.outputs.metrics_json || '' }}
      T_ACC_JSON: ${{ needs.t_accuracy.outputs.metrics_json || '' }}
      BD1_JSON: ${{ needs.bd_model1.outputs.metrics_json || '' }}
      BD2_JSON: ${{ needs.bd_model2.outputs.metrics_json || '' }}
      BD3_JSON: ${{ needs.bd_model3.outputs.metrics_json || '' }}
      ONETRANS_PERF_JSON: ${{ needs.onetrans_perf.outputs.metrics_json || '' }}
      TOKENLARGE_PERF_JSON: ${{ needs.tokenlarge_perf.outputs.metrics_json || '' }}
    steps:

      - name: Generate final summary
        id: final_summary
        env:
          COMMIT_ID: ${{ needs.prepare_metadata.outputs.commit_id }}
          RUN_KIND: ${{ needs.prepare_metadata.outputs.run_kind }}
          RUN_ID: ${{ needs.prepare_metadata.outputs.run_id }}
          WORKFLOW_RUN_ID: ${{ needs.prepare_metadata.outputs.workflow_run_id || github.run_id }}
          WORKFLOW_RUN_ATTEMPT: ${{ needs.prepare_metadata.outputs.workflow_run_attempt || github.run_attempt }}
          HOST_NAME: ${{ needs.prepare_metadata.outputs.host_name || 'unavailable' }}
          HOST_IP: ${{ needs.prepare_metadata.outputs.host_ip || 'unavailable' }}
          HOST_SN: ${{ needs.prepare_metadata.outputs.host_sn || 'unavailable' }}
          FORMAT_DISPLAY: ${{ needs.format.outputs.status == 'failure' && 'failure (non-blocking)' || needs.format.outputs.status || 'skipped' }}
          BUILD_CURRENT_STATUS: ${{ needs.build_current.outputs.status || 'skipped' }}
          BUILD_BASELINE_STATUS: ${{ needs.build_baseline.outputs.status || 'skipped' }}
          INTEGRATION_STATUS: ${{ needs.integration.outputs.status || 'skipped' }}
          T_PERF_STATUS: ${{ needs.t_perf.outputs.status || 'skipped' }}
          T_ACC_STATUS: ${{ needs.t_accuracy.outputs.status || 'skipped' }}
          BD1_STATUS: ${{ needs.bd_model1.outputs.status || 'skipped' }}
          BD2_STATUS: ${{ needs.bd_model2.outputs.status || 'skipped' }}
          BD3_STATUS: ${{ needs.bd_model3.outputs.status || 'skipped' }}
          ONETRANS_PERF_STATUS: ${{ needs.onetrans_perf.outputs.status || 'skipped' }}
          TOKENLARGE_PERF_STATUS: ${{ needs.tokenlarge_perf.outputs.status || 'skipped' }}
          TRAINING_STATUS: ${{ needs.training.outputs.status || 'skipped' }}
        run: |
          set -euo pipefail

          LOG_ROOT="${{ needs.prepare_metadata.outputs.log_root }}"
          SUMMARY_DIR="$RUNNER_TEMP/final-summary"
          SUMMARY_FILE="$SUMMARY_DIR/final-summary.md"
          SUMMARY_JSON="$SUMMARY_DIR/final-summary.json"
          SUMMARY_ERROR_DIR="$SUMMARY_DIR/error_logs"
          TRAINING_ERROR_DIR="$LOG_ROOT/training/error_logs"
          mkdir -p "$SUMMARY_DIR"
          FORMAT_STATUS="${{ needs.format.outputs.status || 'skipped' }}"
          FORMAT_RESULT="${{ needs.format.result }}"
          FORMAT_DISPLAY="$FORMAT_STATUS"
          BUILD_CURRENT_STATUS="${{ needs.build_current.outputs.status || 'skipped' }}"
          BUILD_CURRENT_RESULT="${{ needs.build_current.result }}"
          BUILD_BASELINE_STATUS="${{ needs.build_baseline.outputs.status || 'skipped' }}"
          BUILD_BASELINE_RESULT="${{ needs.build_baseline.result }}"
          INTEGRATION_STATUS="${{ needs.integration.outputs.status || 'skipped' }}"
          INTEGRATION_RESULT="${{ needs.integration.result }}"
          T_PERF_STATUS="${{ needs.t_perf.outputs.status || 'skipped' }}"
          T_PERF_RESULT="${{ needs.t_perf.result }}"
          T_ACC_STATUS="${{ needs.t_accuracy.outputs.status || 'skipped' }}"
          T_ACC_RESULT="${{ needs.t_accuracy.result }}"
          BD1_STATUS="${{ needs.bd_model1.outputs.status || 'skipped' }}"
          BD1_RESULT="${{ needs.bd_model1.result }}"
          BD2_STATUS="${{ needs.bd_model2.outputs.status || 'skipped' }}"
          BD2_RESULT="${{ needs.bd_model2.result }}"
          BD3_STATUS="${{ needs.bd_model3.outputs.status || 'skipped' }}"
          BD3_RESULT="${{ needs.bd_model3.result }}"
          ONETRANS_PERF_STATUS="${{ needs.onetrans_perf.outputs.status || 'skipped' }}"
          ONETRANS_PERF_RESULT="${{ needs.onetrans_perf.result }}"
          TOKENLARGE_PERF_STATUS="${{ needs.tokenlarge_perf.outputs.status || 'skipped' }}"
          TOKENLARGE_PERF_RESULT="${{ needs.tokenlarge_perf.result }}"
          TRAINING_STATUS="${{ needs.training.outputs.status || 'skipped' }}"
          TRAINING_RESULT="${{ needs.training.result }}"
          PREPARE_METADATA_RESULT="${{ needs.prepare_metadata.result }}"

          if [[ "$FORMAT_STATUS" == "failure" ]]; then
            FORMAT_DISPLAY="failure (non-blocking)"
          fi

          python3 - "$SUMMARY_FILE" "$SUMMARY_JSON" <<'PY'
          import json
          import os
          import sys

          summary_file, summary_json = sys.argv[1:3]

          def load_json_env(name):
              raw = (os.environ.get(name) or "").strip()
              if not raw:
                  return None
              try:
                  return json.loads(raw)
              except json.JSONDecodeError:
                  return None

          def format_value(value):
              if value is None:
                  return "n/a"
              if isinstance(value, float):
                  return f"{value:.4f}"
              return str(value)

          commit_id = os.environ["COMMIT_ID"]
          run_kind = os.environ["RUN_KIND"]
          run_id = os.environ["RUN_ID"]
          workflow_run_id = os.environ["WORKFLOW_RUN_ID"]
          workflow_run_attempt = os.environ["WORKFLOW_RUN_ATTEMPT"]
          host_name = os.environ["HOST_NAME"]
          host_ip = os.environ["HOST_IP"]
          host_sn = os.environ["HOST_SN"]

          jobs = [
              {"name": "Format", "result": os.environ["FORMAT_DISPLAY"], "blocking": "no"},
              {"name": "Build Current", "result": os.environ["BUILD_CURRENT_STATUS"], "blocking": "yes"},
              {"name": "Build Baseline", "result": os.environ["BUILD_BASELINE_STATUS"], "blocking": "yes"},
              {"name": "Integration", "result": os.environ["INTEGRATION_STATUS"], "blocking": "yes"},
              {"name": "T Performance", "result": os.environ["T_PERF_STATUS"], "blocking": "yes"},
              {"name": "T Accuracy", "result": os.environ["T_ACC_STATUS"], "blocking": "yes"},
              {"name": "BD Model 1", "result": os.environ["BD1_STATUS"], "blocking": "yes"},
              {"name": "BD Model 2", "result": os.environ["BD2_STATUS"], "blocking": "yes"},
              {"name": "BD Model 3", "result": os.environ["BD3_STATUS"], "blocking": "yes"},
              {"name": "OneTrans Performance", "result": os.environ["ONETRANS_PERF_STATUS"], "blocking": "yes"},
              {"name": "TokenMixer Large Performance", "result": os.environ["TOKENLARGE_PERF_STATUS"], "blocking": "yes"},
              {"name": "Training", "result": os.environ["TRAINING_STATUS"], "blocking": "yes"},
          ]

          metrics = [
              load_json_env("T_PERF_JSON"),
              load_json_env("T_ACC_JSON"),
              load_json_env("BD1_JSON"),
              load_json_env("BD2_JSON"),
              load_json_env("BD3_JSON"),
              load_json_env("ONETRANS_PERF_JSON"),
              load_json_env("TOKENLARGE_PERF_JSON"),
          ]
          metrics = [item for item in metrics if item]

          payload = {
              "commit": commit_id,
              "run_kind": run_kind,
              "run_id": run_id,
              "workflow_run_id": workflow_run_id,
              "workflow_run_attempt": workflow_run_attempt,
              "host": {
                  "name": host_name,
                  "ip": host_ip,
                  "sn": host_sn,
              },
              "jobs": jobs,
              "metrics": metrics,
          }

          lines = [
              "# CI Report",
              "",
              f"- Commit: {commit_id}",
              f"- Run kind: {run_kind}",
              f"- Run id: {run_id}",
              f"- Workflow run id: {workflow_run_id}",
              f"- Workflow run attempt: {workflow_run_attempt}",
              f"- Host name: {host_name}",
              f"- Host IP: {host_ip}",
              f"- Host SN: {host_sn}",
              "",
              "## Job Status",
              "| Job | Result | Blocking |",
              "|-----|--------|----------|",
          ]

          for job in jobs:
              lines.append(f"| {job['name']} | {job['result']} | {job['blocking']} |")

          lines.extend(["", "## Key Summary"])

          for metric in metrics:
              name = metric.get("name", "Unknown Metric")
              kind = metric.get("kind", "unknown")
              lines.extend(["", f"### {name}"])

              if kind == "single":
                  lines.append(f"- Status: {metric.get('status', 'unknown')}")
                  lines.append(f"- Baseline: {metric.get('baseline_ms', 'n/a')} ms")
                  lines.append(f"- Current: {metric.get('current_ms', 'n/a')} ms")
                  lines.append(f"- Threshold: {metric.get('threshold_ms', 'n/a')} ms")
              elif kind == "batch_compare":
                  lines.append(f"- Status: {metric.get('status', 'unknown')}")
                  lines.append("| BS | Baseline (ms) | Current (ms) | Threshold (ms) | Result | Detail |")
                  lines.append("|----|---------------|--------------|----------------|--------|--------|")
                  for row in metric.get("rows", []):
                      lines.append(
                          "| {bs} | {base} | {cur} | {thr} | {result} | {detail} |".format(
                              bs=row.get("batch_size", "n/a"),
                              base=format_value(row.get("baseline_ms")),
                              cur=format_value(row.get("current_ms")),
                              thr=format_value(row.get("threshold_ms")),
                              result=row.get("result", "unknown"),
                              detail=str(row.get("detail", "")).replace("|", "/"),
                          )
                      )
              elif kind == "accuracy":
                  lines.append(f"- Status: {metric.get('status', 'unknown')}")
                  lines.append(f"- Current: {metric.get('current', 'n/a')}")
              else:
                  lines.append(f"- Raw metric: {json.dumps(metric, ensure_ascii=False)}")

          with open(summary_file, "w", encoding="utf-8") as f:
              f.write("\n".join(lines) + "\n")
          with open(summary_json, "w", encoding="utf-8") as f:
              json.dump(payload, f, ensure_ascii=False, indent=2)
          PY

          for file in \
            "$LOG_ROOT/summaries/prepare-metadata.md" \
            "$LOG_ROOT/summaries/format.md" \
            "$LOG_ROOT/summaries/build-current.md" \
            "$LOG_ROOT/summaries/build-baseline.md" \
            "$LOG_ROOT/summaries/integration.md" \
            "$LOG_ROOT/summaries/t-perf.md" \
            "$LOG_ROOT/summaries/t-accuracy.md" \
            "$LOG_ROOT/summaries/bd-model1.md" \
            "$LOG_ROOT/summaries/bd-model2.md" \
            "$LOG_ROOT/summaries/bd-model3.md" \
            "$LOG_ROOT/summaries/onetrans-perf.md" \
            "$LOG_ROOT/summaries/tokenlarge-perf.md" \
            "$LOG_ROOT/summaries/training.md"; do
            if [[ -f "$file" ]]; then
              echo >> "$SUMMARY_FILE"
              cat "$file" >> "$SUMMARY_FILE"
            fi
          done

          mkdir -p "$SUMMARY_ERROR_DIR"
          if [[ -d "$TRAINING_ERROR_DIR" ]]; then
            find "$TRAINING_ERROR_DIR" -maxdepth 1 -type f -name '*_error.log' -exec cp {} "$SUMMARY_ERROR_DIR/" \;
            copied_error_logs_count="$(find "$SUMMARY_ERROR_DIR" -maxdepth 1 -type f -name '*_error.log' | wc -l | tr -d '[:space:]')"
          else
            copied_error_logs_count="0"
          fi

          {
            echo
            echo "## Training Error Logs"
            echo "- Source dir: $TRAINING_ERROR_DIR"
            echo "- Copied logs: $copied_error_logs_count"
          } >> "$SUMMARY_FILE"

          echo "===== Final CI Summary ====="
          cat "$SUMMARY_FILE"
          cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"

          BLOCKING_FAILURE=0
          if [[ "$BUILD_CURRENT_STATUS" == "failure" || "$BUILD_CURRENT_RESULT" == "failure" || "$BUILD_CURRENT_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$BUILD_BASELINE_STATUS" == "failure" || "$BUILD_BASELINE_RESULT" == "failure" || "$BUILD_BASELINE_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$INTEGRATION_STATUS" == "failure" || "$INTEGRATION_RESULT" == "failure" || "$INTEGRATION_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$T_PERF_STATUS" == "failure" || "$T_PERF_RESULT" == "failure" || "$T_PERF_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$T_ACC_STATUS" == "failure" || "$T_ACC_RESULT" == "failure" || "$T_ACC_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$BD1_STATUS" == "failure" || "$BD1_RESULT" == "failure" || "$BD1_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$BD2_STATUS" == "failure" || "$BD2_RESULT" == "failure" || "$BD2_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$BD3_STATUS" == "failure" || "$BD3_RESULT" == "failure" || "$BD3_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$ONETRANS_PERF_STATUS" == "failure" || "$ONETRANS_PERF_RESULT" == "failure" || "$ONETRANS_PERF_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$TOKENLARGE_PERF_STATUS" == "failure" || "$TOKENLARGE_PERF_RESULT" == "failure" || "$TOKENLARGE_PERF_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$TRAINING_STATUS" == "failure" || "$TRAINING_RESULT" == "failure" || "$TRAINING_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi
          if [[ "$PREPARE_METADATA_RESULT" == "failure" || "$PREPARE_METADATA_RESULT" == "skipped" ]]; then BLOCKING_FAILURE=1; fi

          if [[ "$BLOCKING_FAILURE" -ne 0 ]]; then
            FINAL_STATUS="failure"
          else
            FINAL_STATUS="success"
          fi

          echo "final_status=$FINAL_STATUS" >> "$GITHUB_OUTPUT"
          echo "summary_file=$SUMMARY_FILE" >> "$GITHUB_OUTPUT"
          echo "summary_json=$SUMMARY_JSON" >> "$GITHUB_OUTPUT"

      - name: Upload final summary artifact
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: summary-final
          path: ${{ runner.temp }}/final-summary
          if-no-files-found: error
          retention-days: 7

      - name: Fail job if blocking checks failed
        if: always() && steps.final_summary.outputs.final_status == 'failure'
        run: exit 1