KempnerInstitute · mmshad · Apr 8, 2026
diff --git a/.gitignore b/.gitignore
@@ -1 +1,4 @@
 .DS_Store
+build/
+*.o
+__pycache__
diff --git a/README.md b/README.md
@@ -33,6 +33,19 @@ For this benchmarking effort, we use the official NVIDIA HPC Benchmark container
 - [NVIDIA HPC Benchmark](https://docs.nvidia.com/nvidia-hpc-benchmarks/overview.html)
 - [Convert NVIDIA HPC Benchmark Container to Singularity](p0_ngc_to_singularity/README.md)
 
+
+# Common Infrastructure
+
+All parameterized launchers share a common configuration layer in the [`common/`](common/) directory:
+
+| File | Purpose |
+|------|---------|
+| [`common/config.sh`](common/config.sh) | Shared SLURM/container defaults (partition, account, SIF path, modules) — all overridable via environment variables |
+| [`common/detect_arch.sh`](common/detect_arch.sh) | GPU architecture detection (Hopper/Blackwell) and precision gating (FP16/FP8/FP4) |
+| [`common/parse_results.py`](common/parse_results.py) | Auto-detect benchmark type from output logs, extract metrics, produce Markdown summary tables |
+
+Each benchmark directory (`p1`–`p5`) now contains a parameterized launcher (`run_*.sh`) alongside the original per-configuration scripts in `runs/`. The launchers accept environment variables for GPU count, precision, problem size, etc.
+
 # NVIDIA HPL Benchmark
 
 HPL (High-Performance Linpack) is a widely used benchmark for measuring the floating-point compute performance of supercomputers. It solves a dense system of linear equations and is the basis for the TOP500 list of supercomputers. In this section, we will run the HPL benchmark using the NVIDIA HPC container on the Kempner AI Cluster.
@@ -46,7 +59,7 @@ Here is the summary of the HPL benchmark runs on the Kempner AI Cluster:
 | 1       | 1      | 92160  | 1024 | 1 | 1 | 12.40    | 4.208e+04 ( 4.208e+04)   |
 | 1       | 2      | 136192 | 1024 | 2 | 1 | 20.28    | 8.304e+04 ( 4.152e+04)   |
 | 1       | 4      | 190464 | 1024 | 2 | 2 | 27.35    | 1.684e+05 ( 4.210e+04)   |
-| 2       | 4      | 264192 | 1024 | 4 | 2 | 37.49    | 3.279e+05 ( 4.099e+04)   |  
+| 2       | 8      | 264192 | 1024 | 4 | 2 | 37.49    | 3.279e+05 ( 4.099e+04)   |  
 
 
 # NVIDIA HPL-MxP Benchmark
@@ -57,7 +70,7 @@ HPL-MxP (High-Performance Linpack - Mixed Precision) is an enhanced version of t
 
 Here is a summary of MxP benchmark (using `FP16` for LU factorization) runs on the Kempner AI Cluster using NVIDIA Hopper GPUs (H100):
 
-| Metric               | 1 N 1 GPU  | 1 N 2 GPUs | 1 N 4 GPUs | 2 N 4 GPUs |
+| Metric               | 1 N 1 GPU  | 1 N 2 GPUs | 1 N 4 GPUs | 2 N 8 GPUs |
 |----------------------|------------|------------|------------|------------|
 | Problem Size (N)     | 92160      | 136192     | 190464     | 264192     |
 | Block Size (NB)      | 1024       | 1024       | 1024       | 1024       |
@@ -101,3 +114,35 @@ Here is the summary of the STREAM benchmark runs on the Kempner AI Cluster using
 
 
 Done!
+
+
+# Tensor-Core GEMM Saturation Benchmark
+
+The tensor-core GEMM saturation benchmark (`p5_tensor_gemm`) is a custom cuBLASLt-based microbenchmark designed to measure the **raw tensor-core throughput** of NVIDIA GPUs in FP16 and FP8 precisions. Unlike HPL/HPL-MxP (which solve a Linpack problem and use tensor cores as part of a larger workflow), this benchmark isolates the GEMM operation to determine how close the hardware gets to its theoretical peak.
+
+- [Tensor-Core GEMM Saturation Benchmark](p5_tensor_gemm/README.md)
+
+| Precision | Theoretical Peak (H100/H200) | What It Measures                     |
+|-----------|------------------------------|--------------------------------------|
+| FP16      | 989.4 TFLOPS                 | Half-precision tensor-core throughput |
+| FP8       | 1978.9 TFLOPS                | FP8 (E4M3) tensor-core throughput     |
+
+### Measured Results (Kempner AI Cluster)
+
+| GPU | Clock (MHz) | FP16 TFLOPS | % of Peak | FP8 TFLOPS | % of Peak |
+|-----|-------------|-------------|-----------|------------|-----------|
+| H100 SXM (holygpu8a17604) | 1980 | 928.4 | 93.8% | 1654.8 | 83.6% |
+| H200 SXM (holygpu8a10302) | 1980 | 885.5 | 89.5% | 1595.7 | 80.6% |
+
+
+# Benchmark Taxonomy
+
+This repository contains two categories of GPU benchmarks:
+
+| Category                      | Benchmarks           | What It Measures                                        |
+|-------------------------------|----------------------|---------------------------------------------------------|
+| **Linpack-style (end-to-end)**| HPL, HPL-MxP         | Dense linear solve performance (FP64, mixed-precision)  |
+| **Tensor-core saturation**    | Tensor GEMM (p5)     | Raw GEMM throughput at specific precisions (FP16, FP8)  |
+| **Memory / system**           | HPCG, STREAM         | Memory bandwidth, sparse solver, interconnect           |
+
+> **Key distinction:** HPL-MxP uses tensor cores for LU factorization but includes iterative refinement, communication, and other overhead — so its TFLOPS numbers reflect *application-level* throughput. The tensor GEMM benchmark isolates the compute kernel to measure *hardware-level* tensor-core saturation.
diff --git a/common/config.sh b/common/config.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# common/config.sh — Shared configuration for all benchmark launchers.
+# Source this file from any launcher script to pick up defaults.
+# All values are env-var-overridable: set them before sourcing this file
+# or export them in your shell / SLURM preamble.
+#
+# Usage:
+#   source "$(dirname "$0")/../common/config.sh"
+
+set -euo pipefail
+
+# ── Cluster / SLURM defaults ────────────────────────────────────────────────
+BENCH_PARTITION="${BENCH_PARTITION:-kempner_h100}"
+BENCH_ACCOUNT="${BENCH_ACCOUNT:-kempner_dev}"
+BENCH_CONSTRAINT="${BENCH_CONSTRAINT:-h100}"
+BENCH_CPUS_PER_TASK="${BENCH_CPUS_PER_TASK:-24}"
+BENCH_MEM="${BENCH_MEM:-375G}"
+BENCH_WALLTIME="${BENCH_WALLTIME:-2:00:00}"
+BENCH_MPI="${BENCH_MPI:-pmix}"
+
+# ── Container ────────────────────────────────────────────────────────────────
+BENCH_SIF_DIR="${BENCH_SIF_DIR:-/n/holylfs06/LABS/kempner_dev/Everyone/nvidia-hpl}"
+BENCH_SIF_FILE="${BENCH_SIF_FILE:-nvidia-hpc-benchmarks-25-04.sif}"
+BENCH_CONTAINER_TAG="${BENCH_CONTAINER_TAG:-25.04}"
+
+# ── Modules (space-separated) ───────────────────────────────────────────────
+BENCH_MODULES="${BENCH_MODULES:-intel/24.0.1-fasrc01 intelmpi/2021.11-fasrc01}"
+
+# ── Derived ──────────────────────────────────────────────────────────────────
+BENCH_CONT="${BENCH_SIF_DIR}/${BENCH_SIF_FILE}"
+
+# ── Helper functions ─────────────────────────────────────────────────────────
+
+bench_load_modules() {
+    for mod in ${BENCH_MODULES}; do
+        module load "$mod"
+    done
+}
+
+bench_container_path() {
+    if [[ ! -f "${BENCH_CONT}" ]]; then
+        echo "ERROR: Container not found at ${BENCH_CONT}" >&2
+        echo "Set BENCH_SIF_DIR and BENCH_SIF_FILE to point to a valid .sif image." >&2
+        return 1
+    fi
+    echo "${BENCH_CONT}"
+}
+
+bench_print_config() {
+    echo "════════════════════════════════════════════════════════════════════"
+    echo "  Benchmark Configuration"
+    echo "════════════════════════════════════════════════════════════════════"
+    echo "  Partition        : ${BENCH_PARTITION}"
+    echo "  Account          : ${BENCH_ACCOUNT}"
+    echo "  Constraint       : ${BENCH_CONSTRAINT}"
+    echo "  CPUs/task        : ${BENCH_CPUS_PER_TASK}"
+    echo "  Memory           : ${BENCH_MEM}"
+    echo "  Wall time        : ${BENCH_WALLTIME}"
+    echo "  Container        : ${BENCH_CONT}"
+    echo "  Container tag    : ${BENCH_CONTAINER_TAG}"
+    echo "  Modules          : ${BENCH_MODULES}"
+    echo "  MPI launcher     : ${BENCH_MPI}"
+    echo "════════════════════════════════════════════════════════════════════"
+}
+
+bench_timestamp() {
+    date "+%Y-%m-%dT%H:%M:%S"
+}
+
+bench_job_header() {
+    echo "Job started at: $(bench_timestamp)"
+    echo "Running on hosts: $(scontrol show hostname 2>/dev/null || hostname)"
+    bench_print_config
+}
+
+# Compare semver strings: returns 0 if $1 >= $2
+bench_version_ge() {
+    local v1="$1" v2="$2"
+    # Replace dots with spaces, compare major.minor
+    local v1_major v1_minor v2_major v2_minor
+    v1_major="${v1%%.*}"; v1_minor="${v1#*.}"; v1_minor="${v1_minor%%.*}"
+    v2_major="${v2%%.*}"; v2_minor="${v2#*.}"; v2_minor="${v2_minor%%.*}"
+    if (( v1_major > v2_major )); then return 0; fi
+    if (( v1_major == v2_major && v1_minor >= v2_minor )); then return 0; fi
+    return 1
+}
diff --git a/common/detect_arch.sh b/common/detect_arch.sh
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+# common/detect_arch.sh — GPU architecture detection and precision gating.
+# Source this file from any launcher that needs arch-aware behaviour.
+#
+# Exports:
+#   GPU_SM        — numeric SM version (e.g. 90, 100)
+#   GPU_ARCH      — human-readable arch name (hopper, blackwell, rtx_blackwell, unknown)
+#   GPU_NAME      — GPU product name from nvidia-smi
+#
+# Functions:
+#   detect_gpu_arch           — populate the variables above
+#   arch_supports_precision   — check if a precision is supported on the detected arch
+#
+# Usage:
+#   source "$(dirname "$0")/../common/detect_arch.sh"
+#   detect_gpu_arch
+#   arch_supports_precision fp8  || { echo "FP8 not supported"; exit 1; }
+
+# ── Detection ────────────────────────────────────────────────────────────────
+
+detect_gpu_arch() {
+    # Try nvidia-smi first (works on compute nodes)
+    if command -v nvidia-smi &>/dev/null; then
+        # Get compute capability (e.g. "9.0" for H100)
+        local cc
+        cc=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '[:space:]')
+        # Convert "9.0" → 90, "10.0" → 100
+        GPU_SM=$(echo "$cc" | awk -F. '{printf "%d", $1*10 + $2}')
+        GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1 | xargs)
+    else
+        echo "WARNING: nvidia-smi not found. Falling back to manual GPU_SM=${GPU_SM:-0}." >&2
+        GPU_SM="${GPU_SM:-0}"
+        GPU_NAME="${GPU_NAME:-unknown}"
+    fi
+
+    # Map SM version to architecture name
+    case "${GPU_SM}" in
+        90)  GPU_ARCH="hopper"        ;;   # H100, H200
+        100) GPU_ARCH="blackwell"     ;;   # B100, B200, GB200
+        120) GPU_ARCH="rtx_blackwell" ;;   # RTX 5090, RTX 5080, etc.
+        *)   GPU_ARCH="unknown"       ;;
+    esac
+
+    export GPU_SM GPU_ARCH GPU_NAME
+}
+
+# ── Precision gating ─────────────────────────────────────────────────────────
+#
+# Precision support matrix:
+#   Arch            FP16    FP8     FP4
+#   hopper (SM90)   ✓       ✓       ✗
+#   blackwell       ✓       ✓       ✓
+#   rtx_blackwell   ✓       ✓       ✓
+#
+# Returns 0 (true) if the precision is supported, 1 otherwise.
+
+arch_supports_precision() {
+    local precision="${1,,}"  # lowercase
+
+    case "${GPU_ARCH}" in
+        hopper)
+            case "${precision}" in
+                fp16|fp8) return 0 ;;
+                fp4|nvfp4) return 1 ;;
+                *) return 1 ;;
+            esac
+            ;;
+        blackwell|rtx_blackwell)
+            case "${precision}" in
+                fp16|fp8|fp4|nvfp4) return 0 ;;
+                *) return 1 ;;
+            esac
+            ;;
+        *)
+            echo "WARNING: Unknown GPU architecture '${GPU_ARCH}' (SM ${GPU_SM}). Cannot verify precision support." >&2
+            return 1
+            ;;
+    esac
+}
+
+# ── Pretty-print ─────────────────────────────────────────────────────────────
+
+print_gpu_info() {
+    echo "────────────────────────────────────────────────────────"
+    echo "  GPU Architecture Detection"
+    echo "────────────────────────────────────────────────────────"
+    echo "  GPU Name   : ${GPU_NAME}"
+    echo "  SM Version : ${GPU_SM}"
+    echo "  Arch Family: ${GPU_ARCH}"
+    echo "  FP16 TC    : $(arch_supports_precision fp16 && echo '✓' || echo '✗')"
+    echo "  FP8 TC     : $(arch_supports_precision fp8  && echo '✓' || echo '✗')"
+    echo "  FP4 TC     : $(arch_supports_precision fp4  && echo '✓' || echo '✗')"
+    echo "────────────────────────────────────────────────────────"
+}