Skip to content

Commit 08b5301

Browse files
MDA2AVMDA2AV
committed
Add rounds and specs
1 parent d9ad69e commit 08b5301

20 files changed

Lines changed: 801 additions & 172 deletions

‎scripts/archive.sh‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
5+
ROOT_DIR="$SCRIPT_DIR/.."
6+
SITE_DATA="$ROOT_DIR/site/data"
7+
ROUNDS_DIR="$SITE_DATA/rounds"
8+
9+
usage() {
10+
echo "Usage:"
11+
echo " $0 create <name> Archive current results as a named round"
12+
echo " $0 list List all archived rounds"
13+
echo " $0 delete <id> Delete an archived round"
14+
echo ""
15+
echo "Examples:"
16+
echo " $0 create \"Round 1 — March 2026\""
17+
echo " $0 list"
18+
echo " $0 delete 1"
19+
exit 1
20+
}
21+
22+
CMD="${1:-}"
23+
[ -z "$CMD" ] && usage
24+
25+
case "$CMD" in
26+
create)
27+
NAME="${2:-}"
28+
[ -z "$NAME" ] && { echo "Error: round name required"; usage; }
29+
30+
mkdir -p "$ROUNDS_DIR"
31+
32+
# Determine next round ID
33+
NEXT_ID=1
34+
if [ -f "$ROUNDS_DIR/index.json" ]; then
35+
MAX_ID=$(python3 -c "
36+
import json, sys
37+
with open(sys.argv[1]) as f:
38+
rounds = json.load(f)
39+
ids = [r['id'] for r in rounds]
40+
print(max(ids) if ids else 0)
41+
" "$ROUNDS_DIR/index.json")
42+
NEXT_ID=$((MAX_ID + 1))
43+
fi
44+
45+
DATE=$(date +%Y-%m-%d)
46+
47+
# Read system info from current.json (written by benchmark.sh --save)
48+
CURRENT_JSON="$SITE_DATA/current.json"
49+
if [ -f "$CURRENT_JSON" ]; then
50+
CPU=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('cpu','unknown'))")
51+
CORES=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('cores','unknown'))")
52+
THREADS_PER_CORE=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('threads_per_core','unknown'))")
53+
RAM=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('ram','unknown'))")
54+
RAM_SPEED=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('ram_speed','unknown'))")
55+
GOVERNOR=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('governor','unknown'))")
56+
OS_INFO=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('os','unknown'))")
57+
KERNEL=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('kernel','unknown'))")
58+
DOCKER=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('docker','unknown'))")
59+
DOCKER_RUNTIME=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('docker_runtime','unknown'))")
60+
COMMIT=$(python3 -c "import json; print(json.load(open('$CURRENT_JSON')).get('commit','unknown'))")
61+
else
62+
echo "Warning: site/data/current.json not found — run benchmark.sh --save first"
63+
CPU=$(lscpu 2>/dev/null | awk -F: '/Model name/ {gsub(/^[ \t]+/, "", $2); print $2; exit}')
64+
CORES=$(nproc 2>/dev/null || echo "unknown")
65+
THREADS_PER_CORE=$(lscpu 2>/dev/null | awk -F: '/Thread\(s\) per core/ {gsub(/^[ \t]+/, "", $2); print $2; exit}')
66+
RAM=$(free -h 2>/dev/null | awk '/Mem:/ {print $2}')
67+
RAM_SPEED=$(sudo dmidecode -t memory 2>/dev/null | awk '/Configured Memory Speed:/ && /MHz/ {print $4 " MHz"; exit}')
68+
[ -z "$RAM_SPEED" ] && RAM_SPEED="unknown"
69+
GOVERNOR=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "unknown")
70+
OS_INFO=$(. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME" || uname -s)
71+
KERNEL=$(uname -r)
72+
DOCKER=$(docker version --format '{{.Server.Version}}' 2>/dev/null || echo "unknown")
73+
DOCKER_RUNTIME=$(docker info --format '{{.DefaultRuntime}}' 2>/dev/null || echo "unknown")
74+
COMMIT=$(git -C "$ROOT_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown")
75+
fi
76+
77+
# Bundle all result data into one JSON
78+
python3 -c "
79+
import json, glob, os, sys
80+
81+
site_data = sys.argv[1]
82+
round_file = sys.argv[2]
83+
84+
bundle = {}
85+
for f in sorted(glob.glob(os.path.join(site_data, '*.json'))):
86+
name = os.path.basename(f)
87+
if name in ('frameworks.json', 'langcolors.json'):
88+
continue
89+
if name.startswith('rounds'):
90+
continue
91+
key = os.path.splitext(name)[0]
92+
with open(f) as fh:
93+
bundle[key] = json.load(fh)
94+
95+
# Include frameworks metadata
96+
fw_path = os.path.join(site_data, 'frameworks.json')
97+
if os.path.exists(fw_path):
98+
with open(fw_path) as fh:
99+
bundle['_frameworks'] = json.load(fh)
100+
101+
with open(round_file, 'w') as fh:
102+
json.dump(bundle, fh, separators=(',', ':'))
103+
" "$SITE_DATA" "$ROUNDS_DIR/${NEXT_ID}.json"
104+
105+
# Update index
106+
python3 -c "
107+
import json, os, sys
108+
109+
index_path = sys.argv[1]
110+
round_id = int(sys.argv[2])
111+
name = sys.argv[3]
112+
date = sys.argv[4]
113+
cpu = sys.argv[5]
114+
cores = sys.argv[6]
115+
ram = sys.argv[7]
116+
ram_speed = sys.argv[8]
117+
os_info = sys.argv[9]
118+
kernel = sys.argv[10]
119+
commit = sys.argv[11]
120+
docker = sys.argv[12]
121+
governor = sys.argv[13]
122+
docker_runtime = sys.argv[14]
123+
threads_per_core = sys.argv[15]
124+
current_json = sys.argv[16]
125+
126+
rounds = []
127+
if os.path.exists(index_path):
128+
with open(index_path) as f:
129+
rounds = json.load(f)
130+
131+
smt = 'on' if threads_per_core == '2' else 'off' if threads_per_core == '1' else None
132+
133+
entry = {
134+
'id': round_id,
135+
'name': name,
136+
'date': date,
137+
'cpu': cpu,
138+
'cores': cores,
139+
'ram': ram,
140+
'os': os_info,
141+
'kernel': kernel,
142+
'docker': docker,
143+
'docker_runtime': docker_runtime,
144+
'governor': governor,
145+
'commit': commit
146+
}
147+
if ram_speed != 'unknown':
148+
entry['ram_speed'] = ram_speed
149+
if smt is not None:
150+
entry['smt'] = smt
151+
152+
# Copy tcp config from current.json
153+
if os.path.exists(current_json):
154+
with open(current_json) as f:
155+
cur = json.load(f)
156+
if 'tcp' in cur:
157+
entry['tcp'] = cur['tcp']
158+
159+
rounds.append(entry)
160+
161+
with open(index_path, 'w') as f:
162+
json.dump(rounds, f, indent=2)
163+
" "$ROUNDS_DIR/index.json" "$NEXT_ID" "$NAME" "$DATE" "$CPU" "$CORES" "$RAM" "$RAM_SPEED" "$OS_INFO" "$KERNEL" "$COMMIT" "$DOCKER" "$GOVERNOR" "$DOCKER_RUNTIME" "$THREADS_PER_CORE" "$CURRENT_JSON"
164+
165+
# Clear current results so the new ongoing round starts fresh
166+
rm -rf "$ROOT_DIR/results"/*
167+
# Rebuild site data (produces empty data files)
168+
for f in "$SITE_DATA"/*.json; do
169+
[ -f "$f" ] || continue
170+
fname=$(basename "$f")
171+
[ "$fname" = "langcolors.json" ] && continue
172+
[ "$fname" = "frameworks.json" ] && continue
173+
[ "$fname" = "current.json" ] && continue
174+
echo '[]' > "$f"
175+
done
176+
echo '{}' > "$SITE_DATA/frameworks.json"
177+
rm -f "$SITE_DATA/current.json"
178+
179+
echo "[archived] Round $NEXT_ID: $NAME ($DATE)"
180+
echo "[hardware] $CPU ($CORES cores), $RAM RAM"
181+
echo "[system] $OS_INFO (kernel $KERNEL)"
182+
echo "[commit] $COMMIT"
183+
echo "[file] site/data/rounds/${NEXT_ID}.json"
184+
echo "[reset] results/ cleared — new round started"
185+
;;
186+
187+
list)
188+
if [ ! -f "$ROUNDS_DIR/index.json" ]; then
189+
echo "No archived rounds."
190+
exit 0
191+
fi
192+
python3 -c "
193+
import json, os, sys
194+
with open(sys.argv[1]) as f:
195+
rounds = json.load(f)
196+
if not rounds:
197+
print('No archived rounds.')
198+
else:
199+
for r in rounds:
200+
size = os.path.getsize(os.path.join(sys.argv[2], str(r['id']) + '.json'))
201+
print(f\" #{r['id']:>2} {r['name']:<40} {r['date']} ({size // 1024}KB)\")
202+
" "$ROUNDS_DIR/index.json" "$ROUNDS_DIR"
203+
;;
204+
205+
delete)
206+
ID="${2:-}"
207+
[ -z "$ID" ] && { echo "Error: round ID required"; usage; }
208+
if [ ! -f "$ROUNDS_DIR/${ID}.json" ]; then
209+
echo "Error: round $ID not found"
210+
exit 1
211+
fi
212+
rm -f "$ROUNDS_DIR/${ID}.json"
213+
python3 -c "
214+
import json, sys
215+
with open(sys.argv[1]) as f:
216+
rounds = json.load(f)
217+
rounds = [r for r in rounds if r['id'] != int(sys.argv[2])]
218+
with open(sys.argv[1], 'w') as f:
219+
json.dump(rounds, f, indent=2)
220+
" "$ROUNDS_DIR/index.json" "$ID"
221+
echo "[deleted] Round $ID"
222+
;;
223+
224+
*)
225+
usage
226+
;;
227+
esac

‎scripts/benchmark.sh‎

Lines changed: 94 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,17 @@ declare -A PROFILES=(
3030
)
3131
PROFILE_ORDER=(baseline pipelined limited-conn json baseline-h2 static-h2)
3232

33-
# Usage: benchmark.sh [framework] [profile]
34-
FRAMEWORK="${1:-}"
35-
PROFILE_FILTER="${2:-}"
33+
# Parse flags
34+
SAVE_RESULTS=false
35+
POSITIONAL=()
36+
for arg in "$@"; do
37+
case "$arg" in
38+
--save) SAVE_RESULTS=true ;;
39+
*) POSITIONAL+=("$arg") ;;
40+
esac
41+
done
42+
FRAMEWORK="${POSITIONAL[0]:-}"
43+
PROFILE_FILTER="${POSITIONAL[1]:-}"
3644

3745
rebuild_site_data() {
3846
local site_data="$ROOT_DIR/site/data"
@@ -78,6 +86,64 @@ rebuild_site_data() {
7886
echo "[updated] site/data/${profile}-${conns}.json"
7987
done
8088
done
89+
90+
# Write current round system info
91+
local cpu=$(lscpu 2>/dev/null | awk -F: '/Model name/ {gsub(/^[ \t]+/, "", $2); print $2; exit}')
92+
local cores=$(nproc 2>/dev/null || echo "unknown")
93+
local threads_per_core=$(lscpu 2>/dev/null | awk -F: '/Thread\(s\) per core/ {gsub(/^[ \t]+/, "", $2); print $2; exit}')
94+
local ram=$(free -h 2>/dev/null | awk '/Mem:/ {print $2}')
95+
local ram_speed=$(sudo dmidecode -t memory 2>/dev/null | awk '/Configured Memory Speed:/ && /MHz/ {print $4 " MHz"; exit}')
96+
[ -z "$ram_speed" ] && ram_speed="unknown"
97+
local governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "unknown")
98+
local os_info=$(. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME" || uname -s)
99+
local kernel=$(uname -r)
100+
local docker_ver=$(docker version --format '{{.Server.Version}}' 2>/dev/null || echo "unknown")
101+
local docker_runtime=$(docker info --format '{{.DefaultRuntime}}' 2>/dev/null || echo "unknown")
102+
local commit=$(git -C "$ROOT_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown")
103+
local lo_mtu=$(ip link show lo 2>/dev/null | awk '/mtu/ {print $5}')
104+
local cur_date=$(date +%Y-%m-%d)
105+
python3 -c "
106+
import json, sys, subprocess
107+
108+
d = {'date': sys.argv[1], 'cpu': sys.argv[2], 'cores': sys.argv[3],
109+
'ram': sys.argv[4], 'os': sys.argv[5], 'kernel': sys.argv[6],
110+
'docker': sys.argv[7], 'commit': sys.argv[8],
111+
'governor': sys.argv[11], 'docker_runtime': sys.argv[12],
112+
'threads_per_core': sys.argv[13]}
113+
rs = sys.argv[10]
114+
if rs != 'unknown':
115+
d['ram_speed'] = rs
116+
117+
def sysctl(key):
118+
try:
119+
return subprocess.check_output(['sysctl', '-n', key], stderr=subprocess.DEVNULL).decode().strip()
120+
except:
121+
return None
122+
123+
tcp = {}
124+
lo_mtu = sys.argv[14]
125+
if lo_mtu:
126+
tcp['lo_mtu'] = lo_mtu
127+
cc = sysctl('net.ipv4.tcp_congestion_control')
128+
if cc:
129+
tcp['congestion'] = cc
130+
somaxconn = sysctl('net.core.somaxconn')
131+
if somaxconn:
132+
tcp['somaxconn'] = somaxconn
133+
rmem = sysctl('net.core.rmem_max')
134+
if rmem:
135+
tcp['rmem_max'] = rmem
136+
wmem = sysctl('net.core.wmem_max')
137+
if wmem:
138+
tcp['wmem_max'] = wmem
139+
140+
if tcp:
141+
d['tcp'] = tcp
142+
143+
with open(sys.argv[9], 'w') as f:
144+
json.dump(d, f, indent=2)
145+
" "$cur_date" "$cpu" "$cores" "$ram" "$os_info" "$kernel" "$docker_ver" "$commit" "$site_data/current.json" "$ram_speed" "$governor" "$docker_runtime" "$threads_per_core" "$lo_mtu"
146+
echo "[updated] site/data/current.json"
81147
}
82148

83149
# If no framework, run all enabled ones
@@ -96,9 +162,15 @@ if [ -z "$FRAMEWORK" ]; then
96162
fi
97163
fi
98164

99-
"$SCRIPT_DIR/benchmark.sh" "$fw" "$PROFILE_FILTER" || true
165+
if [ "$SAVE_RESULTS" = "true" ]; then
166+
"$SCRIPT_DIR/benchmark.sh" "$fw" "$PROFILE_FILTER" --save || true
167+
else
168+
"$SCRIPT_DIR/benchmark.sh" "$fw" "$PROFILE_FILTER" || true
169+
fi
100170
done
101-
rebuild_site_data
171+
if [ "$SAVE_RESULTS" = "true" ]; then
172+
rebuild_site_data
173+
fi
102174
exit 0
103175
fi
104176

@@ -309,9 +381,10 @@ for profile in "${profiles_to_run[@]}"; do
309381
reconnects=$(echo "$best_output" | grep -oP 'Reconnects: \K\d+' || echo "0")
310382
fi
311383

312-
# Save results — subdirectory per connection count
313-
mkdir -p "$RESULTS_DIR/$profile/$CONNS"
314-
cat > "$RESULTS_DIR/$profile/${CONNS}/${FRAMEWORK}.json" <<EOF
384+
# Save results only with --save flag
385+
if [ "$SAVE_RESULTS" = "true" ]; then
386+
mkdir -p "$RESULTS_DIR/$profile/$CONNS"
387+
cat > "$RESULTS_DIR/$profile/${CONNS}/${FRAMEWORK}.json" <<EOF
315388
{
316389
"framework": "$DISPLAY_NAME",
317390
"language": "$LANGUAGE",
@@ -327,13 +400,16 @@ for profile in "${profiles_to_run[@]}"; do
327400
"reconnects": $reconnects
328401
}
329402
EOF
330-
echo "[saved] results/$profile/${CONNS}/${FRAMEWORK}.json"
403+
echo "[saved] results/$profile/${CONNS}/${FRAMEWORK}.json"
331404

332-
# Save docker logs
333-
LOGS_DIR="$ROOT_DIR/site/static/logs/$profile/$CONNS"
334-
mkdir -p "$LOGS_DIR"
335-
docker logs "$CONTAINER_NAME" > "$LOGS_DIR/${FRAMEWORK}.log" 2>&1 || true
336-
echo "[saved] site/static/logs/$profile/${CONNS}/${FRAMEWORK}.log"
405+
# Save docker logs
406+
LOGS_DIR="$ROOT_DIR/site/static/logs/$profile/$CONNS"
407+
mkdir -p "$LOGS_DIR"
408+
docker logs "$CONTAINER_NAME" > "$LOGS_DIR/${FRAMEWORK}.log" 2>&1 || true
409+
echo "[saved] site/static/logs/$profile/${CONNS}/${FRAMEWORK}.log"
410+
else
411+
echo "[dry-run] Results not saved (use --save to persist)"
412+
fi
337413

338414
# Stop container before next connection count
339415
docker stop -t 5 "$CONTAINER_NAME" 2>/dev/null || true
@@ -342,5 +418,7 @@ EOF
342418
done # CONNS loop
343419
done
344420

345-
# Rebuild site data
346-
rebuild_site_data
421+
# Rebuild site data only with --save
422+
if [ "$SAVE_RESULTS" = "true" ]; then
423+
rebuild_site_data
424+
fi

0 commit comments

Comments
 (0)