Post nightly bench-vs results to Slack (#530)

* Post nightly Lambda VM vs SP1 benchmark results to Slack

* Skip Slack notification when webhook is unset

Author: nicole-graus

.github/scripts/publish_bench_vs.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+set -euo pipefail
+
+WEBHOOK_URL="$1"
+
+if [ -z "$WEBHOOK_URL" ]; then
+    echo "SLACK_WEBHOOK not configured, skipping notification."
+    exit 0
+fi
+
+METRICS_FILE="bench_vs_artifacts/metrics.txt"
+
+if [ ! -f "$METRICS_FILE" ]; then
+    curl -X POST "$WEBHOOK_URL" \
+        -H 'Content-Type: application/json; charset=utf-8' \
+        --data '{"blocks":[{"type":"header","text":{"type":"plain_text","text":"Lambda VM vs SP1 v6 - Nightly Benchmark"}},{"type":"section","text":{"type":"mrkdwn","text":":x: Benchmark failed - no metrics found. Check the workflow logs."}}]}'
+    exit 0
+fi
+
+parse_metric() {
+    { grep "^${1}=" "$METRICS_FILE" || true; } | cut -d= -f2-
+}
+
+TARGET_STEPS_SERIES=$(parse_metric "target_steps_series")
+LAMBDA_TIMES=$(parse_metric "lambda_times")
+SP1_TIMES=$(parse_metric "sp1_times")
+RATIOS=$(parse_metric "ratios")
+TARGET_CYCLES=$(parse_metric "target_cycles")
+TARGET_CYCLES="${TARGET_CYCLES:-500000000}"
+LAMBDA_PROJECTED_H=$(parse_metric "lambda_projected_time_h")
+SP1_PROJECTED_H=$(parse_metric "sp1_projected_time_h")
+LAMBDA_R2=$(parse_metric "lambda_r2")
+SP1_R2=$(parse_metric "sp1_r2")
+
+IFS='/' read -ra STEPS_ARR <<< "${TARGET_STEPS_SERIES:-}"
+IFS='/' read -ra LAMBDA_ARR <<< "${LAMBDA_TIMES:-}"
+IFS='/' read -ra SP1_ARR <<< "${SP1_TIMES:-}"
+IFS='/' read -ra RATIO_ARR <<< "${RATIOS:-}"
+
+RESULTS_MRKDWN=""
+for i in "${!STEPS_ARR[@]}"; do
+    steps="${STEPS_ARR[$i]}"
+    [ -z "$steps" ] && continue
+    lambda_t="${LAMBDA_ARR[$i]:-n/a}"
+    sp1_t="${SP1_ARR[$i]:-n/a}"
+    ratio="${RATIO_ARR[$i]:-n/a}"
+    steps_m=$(LC_NUMERIC=C awk -v s="$steps" 'BEGIN { printf "%dM", s/1000000 }')
+    if [ "$ratio" != "n/a" ]; then
+        ratio_fmt=$(LC_NUMERIC=C awk -v r="$ratio" 'BEGIN { printf "%.2fx", r }')
+        line="*${steps_m} steps:* Lambda ${lambda_t}s / SP1 ${sp1_t}s - ratio ${ratio_fmt}"
+    else
+        line="*${steps_m} steps:* Lambda ${lambda_t}s / SP1 ${sp1_t}s"
+    fi
+    if [ -n "$RESULTS_MRKDWN" ]; then
+        RESULTS_MRKDWN="${RESULTS_MRKDWN}\\n${line}"
+    else
+        RESULTS_MRKDWN="$line"
+    fi
+done
+
+if [ -z "$RESULTS_MRKDWN" ]; then
+    RESULTS_MRKDWN="(no data)"
+fi
+
+PROJ_SECTION=""
+if [ -n "$LAMBDA_PROJECTED_H" ] || [ -n "$SP1_PROJECTED_H" ]; then
+    TARGET_M=$(LC_NUMERIC=C awk -v c="$TARGET_CYCLES" 'BEGIN { printf "%dM", c/1000000 }')
+    PROJ_MRKDWN="Projected @ ${TARGET_M} cycles:"
+    if [ -n "$LAMBDA_PROJECTED_H" ]; then
+        line="*Lambda VM:* ${LAMBDA_PROJECTED_H}h"
+        [ -n "$LAMBDA_R2" ] && line="$line (R2=${LAMBDA_R2})"
+        PROJ_MRKDWN="${PROJ_MRKDWN}\\n${line}"
+    fi
+    if [ -n "$SP1_PROJECTED_H" ]; then
+        line="*SP1 v6:* ${SP1_PROJECTED_H}h"
+        [ -n "$SP1_R2" ] && line="$line (R2=${SP1_R2})"
+        PROJ_MRKDWN="${PROJ_MRKDWN}\\n${line}"
+    fi
+    PROJ_SECTION=',{"type":"divider"},{"type":"header","text":{"type":"plain_text","text":"Linear Projection"}},{"type":"section","text":{"type":"mrkdwn","text":"'"$PROJ_MRKDWN"'"}}'
+fi
+
+curl -X POST "$WEBHOOK_URL" \
+    -H 'Content-Type: application/json; charset=utf-8' \
+    --data '{"blocks":[{"type":"header","text":{"type":"plain_text","text":"Lambda VM vs SP1 v6 - Nightly Benchmark"}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"'"$RESULTS_MRKDWN"'"}}'"$PROJ_SECTION"']}'

.github/workflows/bench-vs-nightly.yml

@@ -56,3 +56,9 @@ jobs:
 
       - name: Publish summary
         run: cat bench_vs_artifacts/summary.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Post results to Slack
+        if: always()
+        env:
+          SLACK_WEBHOOK: ${{ github.event_name == 'workflow_dispatch' && secrets.BENCH_VS_SLACK_WEBHOOK_TEST || secrets.BENCH_VS_SLACK_WEBHOOK }}
+        run: sh .github/scripts/publish_bench_vs.sh "$SLACK_WEBHOOK"