run_benchmark.sh

#!/usr/bin/env bash
# =============================================================================
# run_benchmark.sh — Head-to-Head: quant.cpp vs llama.cpp KV cache benchmark
# =============================================================================
#
# Runs identical workloads on both engines and produces a side-by-side
# comparison table. All measurements are automated and reproducible.
#
# Usage:
#   bash bench/head_to_head/run_benchmark.sh <model.gguf> [threads]
#
# Example:
#   bash bench/head_to_head/run_benchmark.sh models/SmolLM2-1.7B-Q8_0.gguf 8
#
# Prerequisites:
#   1. Build quant.cpp:
#      cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j$(nproc)
#   2. Build llama.cpp:
#      bash bench/head_to_head/setup_llamacpp.sh
#   3. PPL test data:
#      bench/data/ppl_4k.txt (generated by bench/generate_long_text.py)
#
# Output:
#   - Summary table on stdout
#   - CSV at bench/head_to_head/results/results_<date>.csv
#
# Measurements per config:
#   - Perplexity (PPL) on 4K-token text
#   - Generation speed (tok/s) on 128-token generation
#   - Peak RSS memory usage
#
# =============================================================================

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
DATA_DIR="$PROJECT_DIR/bench/data"
RESULTS_DIR="$SCRIPT_DIR/results"

MODEL="${1:?Usage: bash bench/head_to_head/run_benchmark.sh <model.gguf> [threads]}"
THREADS="${2:-$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)}"

# Resolve model to absolute path
if [[ "$MODEL" != /* ]]; then
    MODEL="$PROJECT_DIR/$MODEL"
fi

# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
TQ_RUN="$PROJECT_DIR/build/quant"
LLAMA_BIN_DIR="$PROJECT_DIR/deps/llama.cpp/build/bin"
LLAMA_CLI="$LLAMA_BIN_DIR/llama-cli"
LLAMA_PPL="$LLAMA_BIN_DIR/llama-perplexity"
PPL_TEXT="$DATA_DIR/ppl_4k.txt"

GEN_TOKENS=128
GEN_PROMPT="The meaning of life is"
DATE_STR=$(date +%Y-%m-%d_%H%M%S)
HOSTNAME_STR=$(hostname -s 2>/dev/null || echo "unknown")
MODEL_NAME=$(basename "$MODEL")

mkdir -p "$RESULTS_DIR"
CSV_OUT="$RESULTS_DIR/results_${DATE_STR}.csv"

# ---------------------------------------------------------------------------
# Validate prerequisites
# ---------------------------------------------------------------------------
ERRORS=0

if [ ! -f "$TQ_RUN" ]; then
    echo "ERROR: quant.cpp binary not found: $TQ_RUN"
    echo "  Build: cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j\$(nproc)"
    ERRORS=1
fi
if [ ! -f "$LLAMA_CLI" ] || [ ! -f "$LLAMA_PPL" ]; then
    echo "ERROR: llama.cpp binaries not found in $LLAMA_BIN_DIR"
    echo "  Run: bash bench/head_to_head/setup_llamacpp.sh"
    ERRORS=1
fi
if [ ! -f "$MODEL" ]; then
    echo "ERROR: Model not found: $MODEL"
    ERRORS=1
fi
if [ ! -f "$PPL_TEXT" ]; then
    echo "WARNING: $PPL_TEXT not found. Attempting to generate..."
    if command -v python3 &>/dev/null && [ -f "$PROJECT_DIR/bench/generate_long_text.py" ]; then
        python3 "$PROJECT_DIR/bench/generate_long_text.py"
    fi
    if [ ! -f "$PPL_TEXT" ]; then
        echo "ERROR: Could not create $PPL_TEXT"
        echo "  Run: python3 bench/generate_long_text.py"
        ERRORS=1
    fi
fi

if [ "$ERRORS" -ne 0 ]; then
    exit 1
fi

# ---------------------------------------------------------------------------
# Header
# ---------------------------------------------------------------------------
echo ""
echo "================================================================"
echo "  Head-to-Head: quant.cpp vs llama.cpp"
echo "================================================================"
echo ""
echo "  Model:       $MODEL_NAME"
echo "  Threads:     $THREADS"
echo "  Host:        $HOSTNAME_STR ($(uname -m))"
echo "  Date:        $DATE_STR"
echo "  PPL text:    ppl_4k.txt ($(wc -w < "$PPL_TEXT" | tr -d ' ') words)"
echo "  Gen tokens:  $GEN_TOKENS"
echo ""
echo "================================================================"
echo ""

# ---------------------------------------------------------------------------
# CSV header
# ---------------------------------------------------------------------------
echo "date,host,model,engine,config,ppl,tok_s,peak_rss_mb" > "$CSV_OUT"

# ---------------------------------------------------------------------------
# Helper: measure peak RSS (macOS vs Linux)
# ---------------------------------------------------------------------------
measure_rss() {
    # Runs a command and returns peak RSS in MB on stdout (last line)
    # All other output goes to a temp file
    local out_file="$1"
    shift
    local rss_kb=0

    if [ "$(uname -s)" = "Darwin" ]; then
        # macOS: /usr/bin/time -l reports "maximum resident set size" in bytes
        local time_out
        time_out=$(/usr/bin/time -l "$@" >"$out_file" 2>&1 && echo "---TIME_OK---" || true)

        # If the command output contains time stats mixed in, separate them
        # Re-run with proper stderr capture
        local time_stderr
        time_stderr=$(mktemp)
        /usr/bin/time -l /bin/sh -c '"$@"' _ "$@" >"$out_file" 2>"$time_stderr" || true
        local rss_bytes
        rss_bytes=$(grep "maximum resident set size" "$time_stderr" | awk '{print $1}' | head -1)
        rm -f "$time_stderr"
        if [ -n "$rss_bytes" ] && [ "$rss_bytes" -gt 0 ] 2>/dev/null; then
            rss_kb=$((rss_bytes / 1024))
        fi
    else
        # Linux: /usr/bin/time -v reports "Maximum resident set size (kbytes)"
        local time_stderr
        time_stderr=$(mktemp)
        /usr/bin/time -v "$@" >"$out_file" 2>"$time_stderr" || true
        rss_kb=$(grep "Maximum resident set size" "$time_stderr" | awk '{print $NF}' | head -1)
        rm -f "$time_stderr"
    fi

    local rss_mb=0
    if [ -n "$rss_kb" ] && [ "$rss_kb" -gt 0 ] 2>/dev/null; then
        rss_mb=$((rss_kb / 1024))
    fi
    echo "$rss_mb"
}

# ---------------------------------------------------------------------------
# Test matrix
# ---------------------------------------------------------------------------
# Format: "engine|config_label|ppl_extra_args|gen_extra_args"
#
# llama.cpp uses --cache-type-k / --cache-type-v (long flags)
# quant.cpp uses -k / -v (short flags)

CONFIGS=(
    # --- llama.cpp configs ---
    "llama|FP16 KV|--cache-type-k f16 --cache-type-v f16|--cache-type-k f16 --cache-type-v f16"
    "llama|Q8_0 K + Q5_0 V|--cache-type-k q8_0 --cache-type-v q5_0|--cache-type-k q8_0 --cache-type-v q5_0"
    "llama|Q4_0 KV|--cache-type-k q4_0 --cache-type-v q4_0|--cache-type-k q4_0 --cache-type-v q4_0"
    # --- quant.cpp configs ---
    "tq|FP16 KV|fp32 fp16|fp32 fp16"
    "tq|uniform_4b K + Q4 V|uniform_4b q4|uniform_4b q4"
    "tq|delta_3b K + Q4 V|turbo_kv_3b q4|turbo_kv_3b q4"
)

# Storage for results: indexed by config order
declare -a R_LABEL R_ENGINE R_PPL R_TOKS R_RSS

# ---------------------------------------------------------------------------
# Run llama.cpp PPL
# ---------------------------------------------------------------------------
run_llama_ppl() {
    local extra_args="$1"
    local out_file
    out_file=$(mktemp)

    $LLAMA_PPL \
        -m "$MODEL" \
        -f "$PPL_TEXT" \
        $extra_args \
        -t "$THREADS" \
        >"$out_file" 2>&1 || true

    # llama-perplexity outputs: "Final estimate: PPL = <value>"
    # or "perplexity: <value>"
    local ppl
    ppl=$(grep -i "final estimate.*PPL" "$out_file" | grep -o '[0-9]*\.[0-9]*' | tail -1)
    if [ -z "$ppl" ]; then
        ppl=$(grep -i "^perplexity" "$out_file" | grep -o '[0-9]*\.[0-9]*' | tail -1)
    fi
    if [ -z "$ppl" ]; then
        # Try broader pattern
        ppl=$(grep -oE "PPL\s*=?\s*[0-9]+\.[0-9]+" "$out_file" | grep -o '[0-9]*\.[0-9]*' | tail -1)
    fi

    rm -f "$out_file"
    echo "${ppl:-N/A}"
}

# ---------------------------------------------------------------------------
# Run llama.cpp generation speed
# ---------------------------------------------------------------------------
run_llama_gen() {
    local extra_args="$1"
    local out_file
    out_file=$(mktemp)

    local rss_mb
    rss_mb=$(measure_rss "$out_file" \
        "$LLAMA_CLI" \
        -m "$MODEL" \
        -p "$GEN_PROMPT" \
        -n "$GEN_TOKENS" \
        $extra_args \
        -t "$THREADS" \
        --temp 0 \
        --no-display-prompt)

    # Parse tok/s from llama-cli output
    # llama_perf_sampler_print:    sampling time =   ... ms / ... runs   (... ms per token, ... tokens per second)
    # llama_perf_context_print:        eval time =   ... ms / ... tokens (... ms per token, ... tokens per second)
    local tok_s
    tok_s=$(grep -oE "[0-9]+\.[0-9]+ tokens per second" "$out_file" | tail -1 | grep -o '[0-9]*\.[0-9]*')
    if [ -z "$tok_s" ]; then
        tok_s=$(grep -oE "[0-9]+\.[0-9]+ tok/s" "$out_file" | tail -1 | grep -o '[0-9]*\.[0-9]*')
    fi

    rm -f "$out_file"
    echo "${tok_s:-N/A}|${rss_mb}"
}

# ---------------------------------------------------------------------------
# Run quant.cpp PPL
# ---------------------------------------------------------------------------
run_tq_ppl() {
    local kv_type="$1"
    local v_quant="$2"
    local out_file
    out_file=$(mktemp)

    local cmd="$TQ_RUN $MODEL --ppl $PPL_TEXT -j $THREADS -k $kv_type"
    if [ "$v_quant" != "fp16" ]; then
        cmd="$cmd -v $v_quant"
    fi

    $cmd >"$out_file" 2>&1 || true

    local ppl
    ppl=$(grep "^PPL_CSV:" "$out_file" | cut -d, -f3)
    if [ -z "$ppl" ]; then
        ppl=$(grep "Perplexity:" "$out_file" | grep -o '[0-9]*\.[0-9]*' | head -1)
    fi

    rm -f "$out_file"
    echo "${ppl:-N/A}"
}

# ---------------------------------------------------------------------------
# Run quant.cpp generation speed
# ---------------------------------------------------------------------------
run_tq_gen() {
    local kv_type="$1"
    local v_quant="$2"
    local out_file
    out_file=$(mktemp)

    local cmd="$TQ_RUN $MODEL -p '$GEN_PROMPT' -n $GEN_TOKENS -T 0.0 -j $THREADS -k $kv_type"
    if [ "$v_quant" != "fp16" ]; then
        cmd="$cmd -v $v_quant"
    fi

    local rss_mb
    rss_mb=$(measure_rss "$out_file" /bin/sh -c "$cmd")

    local tok_s
    tok_s=$(grep -oE "[0-9]+\.[0-9]+ tok/s" "$out_file" | tail -1 | grep -o '[0-9]*\.[0-9]*')

    rm -f "$out_file"
    echo "${tok_s:-N/A}|${rss_mb}"
}

# ---------------------------------------------------------------------------
# Main benchmark loop
# ---------------------------------------------------------------------------
echo "Running ${#CONFIGS[@]} configurations..."
echo "(Each config: PPL + generation speed + memory)"
echo ""

for i in "${!CONFIGS[@]}"; do
    IFS='|' read -r engine label ppl_args gen_args <<< "${CONFIGS[$i]}"

    printf "  [%d/%d] %-12s %-24s " "$((i+1))" "${#CONFIGS[@]}" "($engine)" "$label"

    if [ "$engine" = "llama" ]; then
        # llama.cpp
        ppl=$(run_llama_ppl "$ppl_args")
        gen_result=$(run_llama_gen "$gen_args")
        tok_s=$(echo "$gen_result" | cut -d'|' -f1)
        rss=$(echo "$gen_result" | cut -d'|' -f2)
    else
        # quant.cpp
        kv_type=$(echo "$ppl_args" | awk '{print $1}')
        v_quant=$(echo "$ppl_args" | awk '{print $2}')
        ppl=$(run_tq_ppl "$kv_type" "$v_quant")
        gen_result=$(run_tq_gen "$kv_type" "$v_quant")
        tok_s=$(echo "$gen_result" | cut -d'|' -f1)
        rss=$(echo "$gen_result" | cut -d'|' -f2)
    fi

    R_ENGINE[$i]="$engine"
    R_LABEL[$i]="$label"
    R_PPL[$i]="$ppl"
    R_TOKS[$i]="$tok_s"
    R_RSS[$i]="${rss:-0}"

    printf "PPL=%-8s %s tok/s  RSS=%s MB\n" "$ppl" "$tok_s" "${rss:-N/A}"

    # CSV row
    echo "$DATE_STR,$HOSTNAME_STR,$MODEL_NAME,$engine,$label,$ppl,$tok_s,${rss:-0}" >> "$CSV_OUT"
done

# ---------------------------------------------------------------------------
# Summary table
# ---------------------------------------------------------------------------
echo ""
echo "================================================================"
echo "  RESULTS: Head-to-Head Comparison"
echo "================================================================"
echo ""
printf "  %-12s %-24s %10s %10s %10s\n" "Engine" "Config" "PPL" "tok/s" "RSS (MB)"
printf "  %-12s %-24s %10s %10s %10s\n" "------" "------" "---" "-----" "--------"

for i in "${!CONFIGS[@]}"; do
    printf "  %-12s %-24s %10s %10s %10s\n" \
        "${R_ENGINE[$i]}" "${R_LABEL[$i]}" \
        "${R_PPL[$i]}" "${R_TOKS[$i]}" "${R_RSS[$i]}"
done

# ---------------------------------------------------------------------------
# PPL degradation vs respective FP16 baselines
# ---------------------------------------------------------------------------
echo ""
echo "================================================================"
echo "  PPL DEGRADATION vs FP16 Baseline"
echo "================================================================"
echo ""

# Find baselines
LLAMA_BASE_PPL=""
TQ_BASE_PPL=""
for i in "${!CONFIGS[@]}"; do
    if [ "${R_LABEL[$i]}" = "FP16 KV" ]; then
        if [ "${R_ENGINE[$i]}" = "llama" ]; then
            LLAMA_BASE_PPL="${R_PPL[$i]}"
        else
            TQ_BASE_PPL="${R_PPL[$i]}"
        fi
    fi
done

printf "  %-12s %-24s %10s %12s\n" "Engine" "Config" "PPL" "vs FP16"
printf "  %-12s %-24s %10s %12s\n" "------" "------" "---" "-------"

for i in "${!CONFIGS[@]}"; do
    local_engine="${R_ENGINE[$i]}"
    local_ppl="${R_PPL[$i]}"
    pct_str="(baseline)"

    if [ "${R_LABEL[$i]}" != "FP16 KV" ]; then
        base=""
        if [ "$local_engine" = "llama" ]; then
            base="$LLAMA_BASE_PPL"
        else
            base="$TQ_BASE_PPL"
        fi

        if [ -n "$base" ] && [ "$base" != "N/A" ] && [ "$local_ppl" != "N/A" ]; then
            pct=$(echo "scale=2; ($local_ppl - $base) / $base * 100" | bc 2>/dev/null || echo "N/A")
            if [ "$pct" != "N/A" ]; then
                if echo "$pct" | grep -q '^-'; then
                    pct_str="${pct}%"
                else
                    pct_str="+${pct}%"
                fi
            else
                pct_str="--"
            fi
        else
            pct_str="--"
        fi
    fi

    printf "  %-12s %-24s %10s %12s\n" \
        "$local_engine" "${R_LABEL[$i]}" "$local_ppl" "$pct_str"
done

# ---------------------------------------------------------------------------
# Cross-engine comparison at similar bit rates
# ---------------------------------------------------------------------------
echo ""
echo "================================================================"
echo "  KEY COMPARISON: Similar Bit Rates"
echo "================================================================"
echo ""
echo "  The critical question: at comparable KV compression levels,"
echo "  how does quant.cpp's quality compare to llama.cpp?"
echo ""

# Find specific configs for comparison
LLAMA_Q4_PPL=""
TQ_U4_PPL=""
TQ_D3_PPL=""
for i in "${!CONFIGS[@]}"; do
    case "${R_ENGINE[$i]}|${R_LABEL[$i]}" in
        "llama|Q4_0 KV")          LLAMA_Q4_PPL="${R_PPL[$i]}" ;;
        "tq|uniform_4b K + Q4 V") TQ_U4_PPL="${R_PPL[$i]}" ;;
        "tq|delta_3b K + Q4 V")   TQ_D3_PPL="${R_PPL[$i]}" ;;
    esac
done

echo "  4-bit comparison:"
echo "    llama.cpp Q4_0 KV:          PPL = ${LLAMA_Q4_PPL:-N/A}"
echo "    quant.cpp uniform_4b + Q4V: PPL = ${TQ_U4_PPL:-N/A}"
echo ""
echo "  3-bit vs 4-bit (quant.cpp advantage test):"
echo "    llama.cpp Q4_0 KV:          PPL = ${LLAMA_Q4_PPL:-N/A}  (4.5 bits K + 4.5 bits V)"
echo "    quant.cpp delta_3b + Q4V:   PPL = ${TQ_D3_PPL:-N/A}  (3 bits K + 4 bits V)"
echo ""
echo "  If quant.cpp delta_3b matches llama.cpp Q4_0 in PPL, that is"
echo "  ~30% more KV compression at equal quality."
echo ""

# ---------------------------------------------------------------------------
# Footer
# ---------------------------------------------------------------------------
echo "================================================================"
echo "  Results saved to: $CSV_OUT"
echo "================================================================"
echo ""
echo "  To re-run a single llama.cpp config:"
echo "    $LLAMA_PPL -m $MODEL -f $PPL_TEXT --cache-type-k q4_0 --cache-type-v q4_0 -t $THREADS"
echo ""
echo "  To re-run a single quant.cpp config:"
echo "    $TQ_RUN $MODEL --ppl $PPL_TEXT -k turbo_kv_3b -v q4 -j $THREADS"
echo ""