Skip to content

Commit 735730d

Browse files
authored
fix: benchmark annotations (#50)
1 parent ea3b3ee commit 735730d

File tree

4 files changed

+20
-9
lines changed

4 files changed

+20
-9
lines changed

.github/workflows/benchmarks.yml

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ jobs:
5151
- name: Get system info
5252
id: sysinfo
5353
run: |
54-
echo "python_version=$(python --version)" >> $GITHUB_OUTPUT
54+
echo "python_version=$(uv run python --version)" >> $GITHUB_OUTPUT
5555
echo "os=$(uname -s)" >> $GITHUB_OUTPUT
5656
echo "arch=$(uname -m)" >> $GITHUB_OUTPUT
5757
echo "cpu_count=$(nproc)" >> $GITHUB_OUTPUT
@@ -159,9 +159,10 @@ jobs:
159159
jq -r '
160160
"| Endpoint | Baseline | SDK (100%) | Overhead | SDK (10%) | Overhead |",
161161
"|----------|----------|------------|----------|-----------|----------|",
162-
(.comparison_100 | to_entries[] |
163-
"| \(.key) | \(.value.baseline_mean_ms | . * 10 | round / 10)ms | \(.value.sdk_mean_ms | . * 10 | round / 10)ms | +\(.value.mean_overhead_ms | . * 10 | round / 10)ms (\(.value.mean_overhead_pct | round)%) | - | - |"
164-
)
162+
(.comparison_100 | keys[]) as $key |
163+
(.comparison_100[$key]) as $c100 |
164+
(.comparison_10[$key] // null) as $c10 |
165+
"| \($key) | \($c100.baseline_mean_ms * 10 | round / 10)ms | \($c100.sdk_mean_ms * 10 | round / 10)ms | +\($c100.mean_overhead_ms * 10 | round / 10)ms (\($c100.mean_overhead_pct | round)%) | \(if $c10 then "\($c10.sdk_mean_ms * 10 | round / 10)ms" else "-" end) | \(if $c10 then "+\($c10.mean_overhead_ms * 10 | round / 10)ms (\($c10.mean_overhead_pct | round)%)" else "-" end) |"
165166
' benchmarks/results/realistic-workload.json >> "$SUMMARY_FILE"
166167
167168
cat >> "$SUMMARY_FILE" << 'EOF'
@@ -175,11 +176,15 @@ jobs:
175176
jq -r '
176177
"| QPS | Baseline | SDK (100%) | Overhead | SDK (10%) | Overhead |",
177178
"|-----|----------|------------|----------|-----------|----------|",
178-
(.baseline | to_entries[] |
179-
. as $b |
180-
($b.key | tostring) as $qps |
181-
"| \($qps) | \($b.value.mean_ms | . * 10 | round / 10)ms | - | - | - | - |"
182-
)
179+
(.baseline | keys[] | tonumber) as $qps |
180+
(.baseline["\($qps)"].mean_ms) as $base |
181+
(.sdk_100["\($qps)"].mean_ms // null) as $sdk100 |
182+
(.sdk_10["\($qps)"].mean_ms // null) as $sdk10 |
183+
(if $sdk100 then (($sdk100 - $base) * 10 | round / 10) else null end) as $overhead100 |
184+
(if $sdk10 then (($sdk10 - $base) * 10 | round / 10) else null end) as $overhead10 |
185+
(if $sdk100 then ((($sdk100 - $base) / $base * 100) | round) else null end) as $pct100 |
186+
(if $sdk10 then ((($sdk10 - $base) / $base * 100) | round) else null end) as $pct10 |
187+
"| \($qps) | \($base * 10 | round / 10)ms | \(if $sdk100 then "\($sdk100 * 10 | round / 10)ms" else "-" end) | \(if $overhead100 then "+\($overhead100)ms (\($pct100)%)" else "-" end) | \(if $sdk10 then "\($sdk10 * 10 | round / 10)ms" else "-" end) | \(if $overhead10 then "+\($overhead10)ms (\($pct10)%)" else "-" end) |"
183188
' benchmarks/results/fixed-qps-latency.json >> "$SUMMARY_FILE"
184189
185190
cat >> "$SUMMARY_FILE" << 'EOF'

benchmarks/bench/fixed_qps_latency.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
sends requests at a steady rate to measure realistic latency impact.
99
"""
1010

11+
from __future__ import annotations
12+
1113
import json
1214
import os
1315
import subprocess

benchmarks/bench/realistic_workload.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
These simulate production APIs that do actual work (I/O, validation, processing).
1010
"""
1111

12+
from __future__ import annotations
13+
1214
import json
1315
import os
1416
import subprocess

benchmarks/bench/sdk_sampling_rates.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/usr/bin/env python3
22
"""Benchmark: SDK with different sampling rates."""
33

4+
from __future__ import annotations
5+
46
import os
57
import shutil
68
import sys

0 commit comments

Comments
 (0)