fix(profile): enable vllm trace output for GB200

Author: Oseltamivir

.github/workflows/profile.yml

@@ -241,17 +241,19 @@ jobs:
           fi
 
           trace_path="profile_${res_name}.trace.json.gz"
-          if [ ! -f "$trace_path" ] && [ -d LOGS/profiles ]; then
+          if [ ! -f "$trace_path" ] && [ -d LOGS ]; then
             trace_candidate="$(python3 - <<'PY'
           from pathlib import Path
 
-          root = Path("LOGS/profiles")
+          root = Path("LOGS")
           candidates = [
               p for p in root.rglob("*")
               if p.is_file() and (
                   p.name.endswith(".trace.json")
                   or p.name.endswith(".trace.json.gz")
                   or p.name.endswith(".pt.trace.json")
+                  or p.name.endswith(".pt.trace.json.gz")
+                  or p.name.endswith(".json.gz")
                   or p.name.endswith(".json")
               )
           ]
@@ -293,6 +295,11 @@ jobs:
             fi
           else
             echo "Profile trace not found: $trace_path" >&2
+            if [ -d LOGS ]; then
+              echo "LOGS profile candidates:" >&2
+              find LOGS -maxdepth 8 -type f \( -path "*/profiles/*" -o -name "*trace*" -o -name "*profile*" \) -printf "%p %s bytes\n" 2>/dev/null | sort >&2 || true
+            fi
+            exit 1
           fi
 
       - name: Process result (json -> agg)

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-gb200-profile-16gpu-conc256-mtp3.yaml

@@ -34,6 +34,8 @@ backend:
   connector: null
   aggregated_environment:
     VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "1800000"
+    VLLM_TORCH_PROFILER_DIR: "/logs/profiles/agg"
     TILELANG_CLEANUP_TEMP_FILES: "1"
     VLLM_USE_NCCL_SYMM_MEM: "1"
     TORCH_SYMMMEM: "NVSHMEM"
@@ -67,6 +69,7 @@ backend:
       max-num-seqs: 256
       max-num-batched-tokens: 256
       max-cudagraph-capture-size: 256
+      profiler-config: '{"profiler":"torch","torch_profiler_dir":"/logs/profiles/agg","ignore_frontend":true,"delay_iterations":3,"max_iterations":1,"active_iterations":1,"torch_profiler_with_stack":false}'
       trust-remote-code: true
       no-enable-prefix-caching: true
       no-enable-flashinfer-autotune: true