Skip to content

Commit 9088d92

Browse files
TKorrTKorr
committed
Refactor benchmark documentation update process in GitHub Actions
- Replaced the manual benchmark summary generation script with a dedicated script to streamline the update of benchmark results in the documentation. - Introduced a new script for writing benchmark estimates and throughput data, improving maintainability and clarity. - Enhanced the existing workflow to utilize the new script, ensuring consistent and accurate documentation of benchmark results.
1 parent 66d4247 commit 9088d92

3 files changed

Lines changed: 252 additions & 104 deletions

File tree

‎.github/workflows/release.yml‎

Lines changed: 1 addition & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -30,110 +30,7 @@ jobs:
3030
run: cargo bench --no-fail-fast
3131
- name: Generate benchmark summary
3232
run: |
33-
set -euo pipefail
34-
criterion_dir="target/criterion"
35-
36-
mean_ns() {
37-
local est="$1/new/estimates.json"
38-
if [ ! -f "$est" ]; then
39-
echo "TBD"
40-
return
41-
fi
42-
jq -r '.mean.point_estimate' "$est"
43-
}
44-
45-
mems_from() {
46-
local bench_dir="$1"
47-
local bench_json="$bench_dir/new/benchmark.json"
48-
local est_json="$bench_dir/new/estimates.json"
49-
local elements
50-
local ns
51-
if [ ! -f "$bench_json" ] || [ ! -f "$est_json" ]; then
52-
echo "TBD"
53-
return
54-
fi
55-
elements=$(jq -r '.throughput.Elements // empty' "$bench_json")
56-
ns=$(jq -r '.mean.point_estimate // empty' "$est_json")
57-
if [ -z "$elements" ] || [ -z "$ns" ] || [ "$elements" = "0" ] || [ "$ns" = "0" ]; then
58-
echo "TBD"
59-
return
60-
fi
61-
awk -v e="$elements" -v ns="$ns" 'BEGIN { printf "%.2f", (e * 1e9 / ns) / 1e6 }'
62-
}
63-
64-
lru_get=$(mean_ns "$criterion_dir/lru_get_hit_ns")
65-
lru_insert=$(mean_ns "$criterion_dir/lru_insert_full_ns")
66-
lru_k_get=$(mean_ns "$criterion_dir/lru_k_get_hit_ns")
67-
lru_k_insert=$(mean_ns "$criterion_dir/lru_k_insert_full_ns")
68-
lfu_get=$(mean_ns "$criterion_dir/lfu_get_hit_ns")
69-
lfu_insert=$(mean_ns "$criterion_dir/lfu_insert_full_ns")
70-
lfu_touch=$(mean_ns "$criterion_dir/lfu_policy_only_touch_ns")
71-
72-
lru_insert_get=$(mems_from "$criterion_dir/lru_policy/insert_get")
73-
lru_eviction=$(mems_from "$criterion_dir/lru_policy/eviction_churn")
74-
lru_pop=$(mems_from "$criterion_dir/lru_policy/pop_lru")
75-
lru_hot=$(mems_from "$criterion_dir/lru_policy/touch_hotset")
76-
77-
lru_k_insert_get=$(mems_from "$criterion_dir/lru_k_policy/insert_get")
78-
lru_k_eviction=$(mems_from "$criterion_dir/lru_k_policy/eviction_churn")
79-
lru_k_pop=$(mems_from "$criterion_dir/lru_k_policy/pop_lru_k")
80-
lru_k_hot=$(mems_from "$criterion_dir/lru_k_policy/touch_hotset")
81-
82-
lfu_insert_get=$(mems_from "$criterion_dir/lfu_policy/insert_get")
83-
lfu_eviction=$(mems_from "$criterion_dir/lfu_policy/eviction_churn")
84-
lfu_pop=$(mems_from "$criterion_dir/lfu_pop_lfu_policy")
85-
lfu_hot=$(mems_from "$criterion_dir/lfu_get_hotset_policy")
86-
87-
lru_uniform=$(mems_from "$criterion_dir/lru_workload_hit_rate/uniform")
88-
lru_hotset=$(mems_from "$criterion_dir/lru_workload_hit_rate/hotset_90_10")
89-
lru_scan=$(mems_from "$criterion_dir/lru_workload_hit_rate/scan")
90-
91-
lru_k_uniform=$(mems_from "$criterion_dir/lru_k_workload_hit_rate/uniform")
92-
lru_k_hotset=$(mems_from "$criterion_dir/lru_k_workload_hit_rate/hotset_90_10")
93-
lru_k_scan=$(mems_from "$criterion_dir/lru_k_workload_hit_rate/scan")
94-
95-
lfu_uniform=$(mems_from "$criterion_dir/lfu_workload_hit_rate/uniform")
96-
lfu_hotset=$(mems_from "$criterion_dir/lfu_workload_hit_rate/hotset_90_10")
97-
lfu_scan=$(mems_from "$criterion_dir/lfu_workload_hit_rate/scan")
98-
99-
cat > /tmp/latest-run.md <<EOF
100-
Micro-ops (ns/op):
101-
102-
| Cache | get_hit | insert_full | policy_only_touch |
103-
| --- | --- | --- | --- |
104-
| LRU | ${lru_get%.*} | ${lru_insert%.*} | n/a |
105-
| LRU-K | ${lru_k_get%.*} | ${lru_k_insert%.*} | n/a |
106-
| LFU | ${lfu_get%.*} | ${lfu_insert%.*} | ${lfu_touch%.*} |
107-
108-
Policy throughput (Melem/s = million operations per second):
109-
110-
| Cache | insert_get | eviction_churn | pop | touch_hotset |
111-
| --- | --- | --- | --- | --- |
112-
| LRU | ${lru_insert_get} | ${lru_eviction} | ${lru_pop} | ${lru_hot} |
113-
| LRU-K | ${lru_k_insert_get} | ${lru_k_eviction} | ${lru_k_pop} | ${lru_k_hot} |
114-
| LFU | ${lfu_insert_get} | ${lfu_eviction} | ${lfu_pop} | ${lfu_hot} |
115-
116-
Workload throughput (Melem/s, 200k ops):
117-
118-
| Cache | uniform | hotset_90_10 | scan |
119-
| --- | --- | --- | --- |
120-
| LRU | ${lru_uniform} | ${lru_hotset} | ${lru_scan} |
121-
| LRU-K | ${lru_k_uniform} | ${lru_k_hotset} | ${lru_k_scan} |
122-
| LFU | ${lfu_uniform} | ${lfu_hotset} | ${lfu_scan} |
123-
EOF
124-
125-
awk '
126-
/<!-- LATEST_RUN_START -->/ {
127-
print;
128-
while ((getline line < "/tmp/latest-run.md") > 0) print line;
129-
in=1;
130-
next
131-
}
132-
/<!-- LATEST_RUN_END -->/ { in=0; print; next }
133-
!in { print }
134-
' docs/benchmarks.md > /tmp/benchmarks.md
135-
136-
mv /tmp/benchmarks.md docs/benchmarks.md
33+
./scripts/update_docs_benchmarks.sh target/criterion docs/benchmarks.md
13734
- name: Package benchmark results
13835
run: tar -czf criterion-results.tar.gz target/criterion
13936
- name: Build docs site with Jekyll

‎scripts/update_docs_benchmarks.sh‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
criterion_dir="${1:-target/criterion}"
5+
benchmarks_md="${2:-docs/benchmarks.md}"
6+
7+
mean_ns() {
8+
local dir="$1"
9+
local est="$dir/new/estimates.json"
10+
if [[ ! -f "$est" ]]; then
11+
echo "TBD"
12+
return 0
13+
fi
14+
jq -r '.mean.point_estimate' "$est"
15+
}
16+
17+
melems_from() {
18+
local dir="$1"
19+
local bench_json="$dir/new/benchmark.json"
20+
local est_json="$dir/new/estimates.json"
21+
local elements ns
22+
23+
if [[ ! -f "$bench_json" || ! -f "$est_json" ]]; then
24+
echo "TBD"
25+
return 0
26+
fi
27+
28+
elements="$(jq -r '.throughput.Elements // empty' "$bench_json")"
29+
ns="$(jq -r '.mean.point_estimate // empty' "$est_json")"
30+
if [[ -z "$elements" || -z "$ns" || "$elements" == "0" || "$ns" == "0" ]]; then
31+
echo "TBD"
32+
return 0
33+
fi
34+
35+
awk -v e="$elements" -v ns="$ns" 'BEGIN { printf "%.2f", (e * 1e9 / ns) / 1e6 }'
36+
}
37+
38+
melems_first_of() {
39+
local path
40+
for path in "$@"; do
41+
if [[ -d "$path" ]]; then
42+
melems_from "$path"
43+
return 0
44+
fi
45+
done
46+
echo "TBD"
47+
}
48+
49+
ns_int() {
50+
local v="$1"
51+
if [[ "$v" == "TBD" ]]; then
52+
echo "TBD"
53+
else
54+
echo "${v%.*}"
55+
fi
56+
}
57+
58+
if [[ ! -f "$benchmarks_md" ]]; then
59+
echo "benchmarks markdown not found: $benchmarks_md" >&2
60+
exit 1
61+
fi
62+
63+
lru_get="$(mean_ns "$criterion_dir/lru_get_hit_ns")"
64+
lru_insert="$(mean_ns "$criterion_dir/lru_insert_full_ns")"
65+
lru_k_get="$(mean_ns "$criterion_dir/lru_k_get_hit_ns")"
66+
lru_k_insert="$(mean_ns "$criterion_dir/lru_k_insert_full_ns")"
67+
lfu_get="$(mean_ns "$criterion_dir/lfu_get_hit_ns")"
68+
lfu_insert="$(mean_ns "$criterion_dir/lfu_insert_full_ns")"
69+
lfu_touch="$(mean_ns "$criterion_dir/lfu_policy_only_touch_ns")"
70+
71+
lru_insert_get="$(melems_from "$criterion_dir/lru_policy/insert_get")"
72+
lru_eviction="$(melems_from "$criterion_dir/lru_policy/eviction_churn")"
73+
lru_pop="$(melems_from "$criterion_dir/lru_policy/pop_lru")"
74+
lru_hot="$(melems_from "$criterion_dir/lru_policy/touch_hotset")"
75+
76+
lru_k_insert_get="$(melems_from "$criterion_dir/lru_k_policy/insert_get")"
77+
lru_k_eviction="$(melems_from "$criterion_dir/lru_k_policy/eviction_churn")"
78+
lru_k_pop="$(melems_from "$criterion_dir/lru_k_policy/pop_lru_k")"
79+
lru_k_hot="$(melems_from "$criterion_dir/lru_k_policy/touch_hotset")"
80+
81+
lfu_insert_get="$(melems_from "$criterion_dir/lfu_policy/insert_get")"
82+
lfu_eviction="$(melems_from "$criterion_dir/lfu_policy/eviction_churn")"
83+
lfu_pop="$(melems_first_of "$criterion_dir/lfu_pop_lfu_policy" "$criterion_dir/lfu_policy/pop_lfu")"
84+
lfu_hot="$(melems_first_of "$criterion_dir/lfu_get_hotset_policy" "$criterion_dir/lfu_policy/get_hotset")"
85+
86+
lru_uniform="$(melems_from "$criterion_dir/lru_workload_hit_rate/uniform")"
87+
lru_hotset="$(melems_from "$criterion_dir/lru_workload_hit_rate/hotset_90_10")"
88+
lru_scan="$(melems_from "$criterion_dir/lru_workload_hit_rate/scan")"
89+
90+
lru_k_uniform="$(melems_from "$criterion_dir/lru_k_workload_hit_rate/uniform")"
91+
lru_k_hotset="$(melems_from "$criterion_dir/lru_k_workload_hit_rate/hotset_90_10")"
92+
lru_k_scan="$(melems_from "$criterion_dir/lru_k_workload_hit_rate/scan")"
93+
94+
lfu_uniform="$(melems_from "$criterion_dir/lfu_workload_hit_rate/uniform")"
95+
lfu_hotset="$(melems_from "$criterion_dir/lfu_workload_hit_rate/hotset_90_10")"
96+
lfu_scan="$(melems_from "$criterion_dir/lfu_workload_hit_rate/scan")"
97+
98+
tmp_latest="$(mktemp)"
99+
cat >"$tmp_latest" <<EOF
100+
Micro-ops (ns/op):
101+
102+
| Cache | get_hit | insert_full | policy_only_touch |
103+
| --- | --- | --- | --- |
104+
| LRU | $(ns_int "$lru_get") | $(ns_int "$lru_insert") | n/a |
105+
| LRU-K | $(ns_int "$lru_k_get") | $(ns_int "$lru_k_insert") | n/a |
106+
| LFU | $(ns_int "$lfu_get") | $(ns_int "$lfu_insert") | $(ns_int "$lfu_touch") |
107+
108+
Policy throughput (Melem/s = million operations per second):
109+
110+
| Cache | insert_get | eviction_churn | pop | touch_hotset |
111+
| --- | --- | --- | --- | --- |
112+
| LRU | $lru_insert_get | $lru_eviction | $lru_pop | $lru_hot |
113+
| LRU-K | $lru_k_insert_get | $lru_k_eviction | $lru_k_pop | $lru_k_hot |
114+
| LFU | $lfu_insert_get | $lfu_eviction | $lfu_pop | $lfu_hot |
115+
116+
Workload throughput (Melem/s, 200k ops):
117+
118+
| Cache | uniform | hotset_90_10 | scan |
119+
| --- | --- | --- | --- |
120+
| LRU | $lru_uniform | $lru_hotset | $lru_scan |
121+
| LRU-K | $lru_k_uniform | $lru_k_hotset | $lru_k_scan |
122+
| LFU | $lfu_uniform | $lfu_hotset | $lfu_scan |
123+
EOF
124+
125+
tmp_out="$(mktemp)"
126+
awk -v latest="$tmp_latest" '
127+
/<!-- LATEST_RUN_START -->/ {
128+
print;
129+
while ((getline line < latest) > 0) print line;
130+
in_block=1;
131+
next
132+
}
133+
/<!-- LATEST_RUN_END -->/ {
134+
in_block=0;
135+
print;
136+
next
137+
}
138+
!in_block { print }
139+
' "$benchmarks_md" >"$tmp_out"
140+
141+
mv "$tmp_out" "$benchmarks_md"
142+
rm -f "$tmp_latest"

‎scripts/update_docs_benchmarks_test.sh‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
if ! command -v jq >/dev/null 2>&1; then
5+
echo "skipping: jq not found" >&2
6+
exit 0
7+
fi
8+
9+
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
10+
script="$repo_root/scripts/update_docs_benchmarks.sh"
11+
12+
tmp="$(mktemp -d)"
13+
trap 'rm -rf "$tmp"' EXIT
14+
15+
criterion_dir="$tmp/criterion"
16+
docs_dir="$tmp/docs"
17+
mkdir -p "$criterion_dir" "$docs_dir"
18+
19+
write_estimates() {
20+
local dir="$1"
21+
local mean="$2"
22+
mkdir -p "$dir/new"
23+
cat >"$dir/new/estimates.json" <<EOF
24+
{ "mean": { "point_estimate": $mean } }
25+
EOF
26+
}
27+
28+
write_throughput() {
29+
local dir="$1"
30+
local elements="$2"
31+
mkdir -p "$dir/new"
32+
cat >"$dir/new/benchmark.json" <<EOF
33+
{ "throughput": { "Elements": $elements } }
34+
EOF
35+
}
36+
37+
write_policy_bench() {
38+
local dir="$1"
39+
local elements="$2"
40+
local mean_ns="$3"
41+
write_throughput "$dir" "$elements"
42+
write_estimates "$dir" "$mean_ns"
43+
}
44+
45+
# Micro-ops (ns)
46+
write_estimates "$criterion_dir/lru_get_hit_ns" 30.9
47+
write_estimates "$criterion_dir/lru_insert_full_ns" 138.2
48+
write_estimates "$criterion_dir/lru_k_get_hit_ns" 43.1
49+
write_estimates "$criterion_dir/lru_k_insert_full_ns" 189.9
50+
write_estimates "$criterion_dir/lfu_get_hit_ns" 87.5
51+
write_estimates "$criterion_dir/lfu_insert_full_ns" 195.0
52+
write_estimates "$criterion_dir/lfu_policy_only_touch_ns" 66.7
53+
54+
# Policy throughput (Elements/op and mean ns/op -> Melem/s)
55+
write_policy_bench "$criterion_dir/lru_policy/insert_get" 2048 192900.0
56+
write_policy_bench "$criterion_dir/lru_policy/eviction_churn" 4096 630000.0
57+
write_policy_bench "$criterion_dir/lru_policy/pop_lru" 1024 58800.0
58+
write_policy_bench "$criterion_dir/lru_policy/touch_hotset" 1024 40800.0
59+
60+
write_policy_bench "$criterion_dir/lru_k_policy/insert_get" 2048 320000.0
61+
write_policy_bench "$criterion_dir/lru_k_policy/eviction_churn" 4096 950000.0
62+
write_policy_bench "$criterion_dir/lru_k_policy/pop_lru_k" 1024 90000.0
63+
write_policy_bench "$criterion_dir/lru_k_policy/touch_hotset" 1024 70000.0
64+
65+
write_policy_bench "$criterion_dir/lfu_policy/insert_get" 2048 270000.0
66+
write_policy_bench "$criterion_dir/lfu_policy/eviction_churn" 4096 980000.0
67+
68+
# Workload throughput
69+
write_policy_bench "$criterion_dir/lru_workload_hit_rate/uniform" 200000 28700000.0
70+
write_policy_bench "$criterion_dir/lru_workload_hit_rate/hotset_90_10" 200000 10000000.0
71+
write_policy_bench "$criterion_dir/lru_workload_hit_rate/scan" 200000 27500000.0
72+
73+
write_policy_bench "$criterion_dir/lru_k_workload_hit_rate/uniform" 200000 36000000.0
74+
write_policy_bench "$criterion_dir/lru_k_workload_hit_rate/hotset_90_10" 200000 12500000.0
75+
write_policy_bench "$criterion_dir/lru_k_workload_hit_rate/scan" 200000 40000000.0
76+
77+
write_policy_bench "$criterion_dir/lfu_workload_hit_rate/uniform" 200000 42000000.0
78+
write_policy_bench "$criterion_dir/lfu_workload_hit_rate/hotset_90_10" 200000 24000000.0
79+
write_policy_bench "$criterion_dir/lfu_workload_hit_rate/scan" 200000 41000000.0
80+
81+
# Benchmarks doc with replacement markers.
82+
benchmarks_md="$docs_dir/benchmarks.md"
83+
cat >"$benchmarks_md" <<'EOF'
84+
# Benchmarks
85+
86+
## Latest run (release)
87+
88+
<!-- LATEST_RUN_START -->
89+
placeholder
90+
<!-- LATEST_RUN_END -->
91+
92+
## Release summary
93+
EOF
94+
95+
"$script" "$criterion_dir" "$benchmarks_md"
96+
97+
grep -q "| LRU | 30 | 138 | n/a |" "$benchmarks_md"
98+
grep -q "| LRU-K | 43 | 189 | n/a |" "$benchmarks_md"
99+
grep -q "| LFU | 87 | 195 | 66 |" "$benchmarks_md"
100+
101+
# Ensure missing LFU pop/hotset benches become TBD (we didn't create them).
102+
grep -q "^| LFU | .* | .* | TBD | TBD |$" "$benchmarks_md"
103+
104+
# Ensure marker block still exists and placeholder is gone.
105+
grep -q "<!-- LATEST_RUN_START -->" "$benchmarks_md"
106+
grep -q "<!-- LATEST_RUN_END -->" "$benchmarks_md"
107+
! grep -q "^placeholder$" "$benchmarks_md"
108+
109+
echo "ok: update_docs_benchmarks.sh"

0 commit comments

Comments
 (0)