feat(power): capture AMD temp/util/mem (gfx_activity, used_vram, hotspot)

The amd-smi monitor only ran `metric -p -c -t -u`, so no VRAM column was
emitted and avg_mem_used_mb never populated on AMD. It also used util/mem
column matchers tuned for NVIDIA/srt-slurm names, which miss amd-smi's
conventions — so avg_util_pct and avg_temp_c silently dropped too.

- benchmark_lib.sh: add `-m` (mem-usage) to the amd-smi command so a
  used_vram column is captured.
- aggregate_power.py column detection:
  - util: also match amd-smi `gfx_activity` (umc/mm_activity excluded).
  - mem: match positively on memory/vram + "used" instead of broad "mem"
    minus a growing exclude list — picks memory.used / mem_used_mb /
    used_vram while rejecting mem_temperature, mem_voltage, total/free_vram,
    the memory clock, and utilization.memory.
  - temp: prefer hotspot/junction over the first temp column, since edge
    temperature reads N/A on data-center AMD parts (MI300/MI355).

NVIDIA and srt-slurm detection is unchanged (verified by existing tests).
Adds AMD-header detection tests; full suite 111 passed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: arygupt

benchmarks/benchmark_lib.sh

@@ -41,17 +41,19 @@ start_gpu_monitor() {
         GPU_MONITOR_PID=$!
         echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)"
     elif command -v amd-smi &>/dev/null; then
-        # amd-smi metric flags: -p power, -c clocks, -t temperature, -u usage,
-        # -w <interval> native watch mode (emits a timestamp column per sample),
-        # --csv. The awk filter keeps the first CSV header line and drops
+        # amd-smi metric flags: -p power, -c clocks, -t temperature, -u usage
+        # (gfx_activity), -m mem-usage (used_vram), -w <interval> native watch
+        # mode (emits a timestamp column per sample), --csv. Without -m there is
+        # no VRAM column, so avg_mem_used_mb would never populate on AMD.
+        # The awk filter keeps the first CSV header line and drops
         # amd-smi's preamble / repeated headers. Header match is case-insensitive
         # (tolower) so a capitalized "Timestamp," header — should amd-smi ever
         # emit one — still passes through; aggregate_power's column detection is
         # case-insensitive too. NOTE: amd-smi timestamps are node-local wall
         # clock, so multinode aggregation assumes cluster clocks are NTP-synced
         # (same assumption as nvidia-smi; aggregate_power windows by absolute
         # epoch from benchmark_serving.py).
-        amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \
+        amd-smi metric -p -c -t -u -m -w "$interval" --csv 2>/dev/null \
             | awk 'tolower($0) ~ /^timestamp,/{if(!h){print;h=1};next} h{print}' > "$output" &
         GPU_MONITOR_PID=$!
         echo "[GPU Monitor] Started AMD (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)"

utils/aggregate_power.py

@@ -57,14 +57,18 @@
   - Power: timestamp + column whose name contains "power" (excluding
     "limit"/"cap"/"max"/"min"). NVIDIA: "power.draw [W]". AMD: "socket_power".
     srt-slurm: "power_w".
-  - Temperature: column name contains "temp". NVIDIA: "temperature.gpu". AMD:
-    "temperature". srt-slurm: "temp_c". Unit: Celsius.
-  - Utilization: column name starts with "utilization" or contains "util".
+  - Temperature: column name contains "temp"; hotspot/junction columns are
+    preferred over the first match because data-center AMD parts report edge
+    temperature as N/A. NVIDIA: "temperature.gpu". AMD amd-smi: "edge_temperature"
+    / "hotspot_temperature" (junction picked). srt-slurm: "temp_c". Unit: Celsius.
+  - Utilization: column starts with "utilization" or contains "util", or is
+    amd-smi's "gfx_activity" (umc_activity / mm_activity are not matched).
     NVIDIA: "utilization.gpu". srt-slurm: "util_pct". Unit: percent.
-  - Memory: column name contains "mem" but not "total"/"clock"/"util" — so
-    "memory.total", "clocks.current.memory" (a frequency), and
-    "utilization.memory" (a percent) are all rejected; only memory *used* is
-    picked. NVIDIA: "memory.used [MiB]". srt-slurm: "mem_used_mb". Unit: MiB/MB.
+  - Memory used: column mentions memory/vram AND "used" — picks NVIDIA
+    "memory.used [MiB]", srt-slurm "mem_used_mb", amd-smi "used_vram"; rejects
+    decoys lacking "used" (memory.total / total_vram / free_vram, the memory
+    *clock* "clocks.current.memory", utilization.memory, mem_temperature,
+    mem_voltage). Unit: MiB/MB.
 
 Power is required for aggregation to fire; the other metrics degrade gracefully
 when their columns are absent (those fields are simply omitted from the output).
@@ -90,14 +94,24 @@
 _POWER_COL_RE = re.compile(r"power", re.IGNORECASE)
 _POWER_EXCLUDE_RE = re.compile(r"limit|cap|max|min", re.IGNORECASE)
 _TEMP_COL_RE = re.compile(r"temp", re.IGNORECASE)
-_UTIL_COL_RE = re.compile(r"^utilization|util", re.IGNORECASE)
-_MEM_COL_RE = re.compile(r"mem", re.IGNORECASE)
-# Exclude "total" (memory.total), "clock" (clocks.current.memory — a frequency,
-# not memory used), and "util" (utilization.memory — a percent). nvidia-smi's
-# query emits clocks.current.memory BEFORE any used-memory column, so without
-# these excludes _MEM_COL_RE would grab the memory *clock* (~2500 MHz) as
-# avg_mem_used_mb.
-_MEM_EXCLUDE_RE = re.compile(r"total|clock|util", re.IGNORECASE)
+# Data-center AMD parts (MI300/MI355) report edge temperature as N/A and expose
+# the real die temperature as hotspot/junction; prefer those when present so
+# avg_temp_c isn't computed over an all-N/A edge column. NVIDIA's single
+# "temperature.gpu" and srt-slurm's "temp_c" have neither token and fall through
+# to the first temperature column unchanged.
+_TEMP_PREFER_RE = re.compile(r"hotspot|junction", re.IGNORECASE)
+# Utilization: NVIDIA "utilization.gpu", srt-slurm "util_pct", AMD amd-smi
+# "gfx_activity" (the GPU/graphics-engine busy percent). amd-smi's other usage
+# columns — umc_activity (memory controller), mm_activity (multimedia) — are
+# intentionally NOT matched so gfx_activity is the one picked.
+_UTIL_COL_RE = re.compile(r"^utilization|util|gfx_activity", re.IGNORECASE)
+# Memory *used*: match positively on a column that mentions both memory/vram and
+# "used" rather than broad "mem" + a growing exclude list. This naturally picks
+# NVIDIA "memory.used [MiB]", srt-slurm "mem_used_mb", and amd-smi "used_vram"
+# while rejecting same-prefix decoys that lack "used": memory.total / total_vram /
+# free_vram, clocks.current.memory (a frequency), utilization.memory (a percent),
+# and amd-smi's mem_temperature / mem_voltage.
+_MEM_COL_RE = re.compile(r"(?:mem|vram).*used|used.*(?:mem|vram)", re.IGNORECASE)
 _TIMESTAMP_COL_RE = re.compile(r"time", re.IGNORECASE)
 _GPU_INDEX_COL_RE = re.compile(r"^(index|gpu|gpu_id|gpu_index|card|device)$", re.IGNORECASE)
 _NUMBER_RE = re.compile(r"-?\d+(?:\.\d+)?")
@@ -208,12 +222,14 @@ def _detect_all_columns(header: list[str]) -> dict[str, str | None]:
         (c for c in header if _POWER_COL_RE.search(c) and not _POWER_EXCLUDE_RE.search(c)),
         None,
     )
-    temp_col = next((c for c in header if _TEMP_COL_RE.search(c)), None)
-    util_col = next((c for c in header if _UTIL_COL_RE.search(c)), None)
-    mem_col = next(
-        (c for c in header if _MEM_COL_RE.search(c) and not _MEM_EXCLUDE_RE.search(c)),
-        None,
+    temp_cols = [c for c in header if _TEMP_COL_RE.search(c)]
+    # Prefer hotspot/junction (the real die temp on data-center AMD parts) over
+    # the first temperature column (edge on AMD, temperature.gpu on NVIDIA).
+    temp_col = next((c for c in temp_cols if _TEMP_PREFER_RE.search(c)), None) or (
+        temp_cols[0] if temp_cols else None
     )
+    util_col = next((c for c in header if _UTIL_COL_RE.search(c)), None)
+    mem_col = next((c for c in header if _MEM_COL_RE.search(c)), None)
     gpu_col = next((c for c in header if _GPU_INDEX_COL_RE.match(c.strip())), None)
     return {
         "timestamp": timestamp_col,

utils/test_aggregate_power.py

@@ -1196,6 +1196,46 @@ def test_detect_all_columns_amd_style():
     assert cols["mem"] is None
 
 
+def test_detect_all_columns_amd_smi_full():
+    """Real amd-smi `metric -p -c -t -u -m --csv` header on a data-center part.
+
+    Exercises: gfx_activity as util (not umc/mm_activity), used_vram as mem (not
+    total/free_vram, mem_clock, mem_voltage, or mem_temperature), and
+    hotspot_temperature preferred over the N/A edge_temperature.
+    """
+    header = [
+        "timestamp", "gpu",
+        "socket_power", "mem_voltage",            # -p
+        "gfx_clock", "mem_clock",                 # -c
+        "edge_temperature", "hotspot_temperature", "mem_temperature",  # -t
+        "gfx_activity", "umc_activity", "mm_activity",                 # -u
+        "total_vram", "used_vram", "free_vram",   # -m
+    ]
+    cols = _detect_all_columns(header)
+    assert cols["power"] == "socket_power"
+    assert cols["util"] == "gfx_activity"
+    assert cols["mem"] == "used_vram"
+    # Hotspot/junction preferred over edge (edge reads N/A on MI300/MI355).
+    assert cols["temp"] == "hotspot_temperature"
+
+
+def test_detect_all_columns_temp_prefers_junction():
+    """junction_temperature wins over a leading edge_temperature column."""
+    header = ["timestamp", "gpu", "socket_power",
+              "edge_temperature", "junction_temperature"]
+    assert _detect_all_columns(header)["temp"] == "junction_temperature"
+
+
+def test_detect_all_columns_mem_vram_used_variants():
+    """Both used_vram and vram_used resolve; total/free_vram never do."""
+    assert _detect_all_columns(
+        ["timestamp", "power_w", "total_vram", "vram_used", "free_vram"]
+    )["mem"] == "vram_used"
+    assert _detect_all_columns(
+        ["timestamp", "power_w", "total_vram", "free_vram"]
+    )["mem"] is None
+
+
 def test_detect_all_columns_excludes_memory_total():
     """memory.total must not be picked as the memory column (we want USED memory)."""
     header = ["timestamp", "index", "power.draw [W]", "memory.total [MiB]", "memory.used [MiB]"]