Touch up table

Author: Oseltamivir

utils/collect_eval_results.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import re
 import sys
 import json
 from pathlib import Path
@@ -60,18 +61,44 @@ def detect_eval_jsons(d: Path) -> Tuple[Optional[Path], Optional[Path]]:
 
 def parse_pretty_env(pretty_env: str) -> str:
     try:
-        lines = [l for l in pretty_env.splitlines() if l.startswith('GPU ')]
+        # Only match lines like "GPU 0: <name>", avoid headers like "GPU models and configuration:"
+        lines = [l for l in pretty_env.splitlines() if re.match(r'^GPU\s+\d+\s*:', l)]
         names = [l.split(':', 1)[1].strip() for l in lines]
         if not names:
             return 'Unknown GPU'
-        # Compress identical names (roughly)
-        from collections import Counter
-        c = Counter(names)
-        return ' + '.join([f"{n}× {name}" for name, n in c.items()])
+        # Unique GPU names (no counts), order preserved
+        seen = []
+        for n in names:
+            if n not in seen:
+                seen.append(n)
+        return ' + '.join(seen) if seen else 'Unknown GPU'
     except Exception:
         return 'Unknown GPU'
 
 
+def derive_lens_and_conc(dir_name: str, exp_name_hint: str) -> Tuple[Optional[int], Optional[int], Optional[int]]:
+    """Extract (isl, osl, conc) from artifact directory name or exp_name hint.
+
+    Expected patterns inside `dir_name` (or `exp_name_hint`):
+      - "<num>k<num>k" for (ISL, OSL) where k -> 1024
+      - "_conc<num>_" for concurrency
+    """
+    isl = osl = conc = None
+    try:
+        m = re.search(r'(\d+)k(\d+)k', dir_name)
+        if not m and exp_name_hint:
+            m = re.search(r'(\d+)k(\d+)k', exp_name_hint)
+        if m:
+            isl = int(m.group(1)) * 1024
+            osl = int(m.group(2)) * 1024
+        c = re.search(r'_conc(\d+)(?:_|$)', dir_name)
+        if c:
+            conc = int(c.group(1))
+    except Exception:
+        pass
+    return isl, osl, conc
+
+
 def extract_lm_metrics(json_path: Path, task: Optional[str] = None) -> Dict[str, Any]:
     data = load_json(json_path) or {}
     results = data.get('results') or {}
@@ -179,6 +206,7 @@ def main():
     rows: List[Dict[str, Any]] = []
     for d in find_eval_sets(root):
         meta = load_json(d / 'meta_env.json') or {}
+        isl, osl, conc = derive_lens_and_conc(d.name, exp_name)
         lm_path, le_path = detect_eval_jsons(d)
         # Prefer lm-eval when available, else lighteval
         if lm_path:
@@ -196,6 +224,9 @@ def main():
             'precision': (meta.get('precision') or 'unknown').lower(),
             'tp': int(meta.get('tp') or 1),
             'ep': int(meta.get('ep') or 1),
+            'conc': int(conc) if conc is not None else None,
+            'isl': int(isl) if isl is not None else None,
+            'osl': int(osl) if osl is not None else None,
             'dp_attention': str(meta.get('dp_attention') or 'false'),
             'task': m.get('task') or 'unknown',
             'em_strict': m.get('strict'),
@@ -214,8 +245,8 @@ def main():
         print('> No eval results found to summarize.')
     else:
         # Print Markdown summary table
-        print('| Model | Hardware | Framework | Precision | TP | EP | DPA | Task | EM Strict | EM Flexible | N (eff) |')
-        print('| :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: |')
+        print('| Model | Hardware | Framework | Precision | TP | EP | Conc | ISL | OSL | DPA | Task | EM Strict | EM Flexible | N (eff) |')
+        print('| :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: |')
         for r in rows:
             print(
                 f"| {r['model']} "
@@ -224,6 +255,9 @@ def main():
                 f"| {r['precision'].upper()} "
                 f"| {r['tp']} "
                 f"| {r['ep']} "
+                f"| {r.get('conc','')} "
+                f"| {r.get('isl','')} "
+                f"| {r.get('osl','')} "
                 f"| {r['dp_attention']} "
                 f"| {r['task']} "
                 f"| {pct(r['em_strict'])}{se(r['em_strict_se'])} "