results

Author: MDA2AV

.github/workflows/benchmark.yml

@@ -1,6 +1,10 @@
 name: Benchmark
 
 on:
+  push:
+    branches: [main]
+    paths:
+      - 'frameworks/**'
   workflow_dispatch:
     inputs:
       framework:
@@ -11,44 +15,47 @@ on:
 permissions:
   contents: write
 
+concurrency:
+  group: benchmark
+  cancel-in-progress: false
+
 jobs:
   benchmark:
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
 
-      - name: Determine frameworks
-        id: frameworks
+      - name: Detect modified frameworks
+        id: detect
         run: |
           if [ -n "${{ inputs.framework }}" ]; then
             echo "list=${{ inputs.framework }}" >> "$GITHUB_OUTPUT"
           else
-            echo "list=$(ls -d frameworks/*/ | xargs -n1 basename | tr '\n' ' ')" >> "$GITHUB_OUTPUT"
+            list=$(git diff --name-only HEAD~1 HEAD \
+              | grep '^frameworks/' \
+              | cut -d'/' -f2 \
+              | sort -u \
+              | tr '\n' ' ')
+            echo "list=$list" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Run benchmarks
+        if: steps.detect.outputs.list != ''
         run: |
-          for fw in ${{ steps.frameworks.outputs.list }}; do
+          for fw in ${{ steps.detect.outputs.list }}; do
             echo ">>> Benchmarking: $fw"
-            ./scripts/benchmark.sh "$fw" || echo "WARN: $fw benchmark failed"
+            ./scripts/benchmark.sh --save "$fw" || echo "WARN: $fw benchmark failed"
           done
 
-      - name: Update site data
-        run: |
-          mkdir -p site/data
-          echo '[' > site/data/results.json
-          first=true
-          for f in results/*.json; do
-            [ -f "$f" ] || continue
-            $first || echo ',' >> site/data/results.json
-            cat "$f" >> site/data/results.json
-            first=false
-          done
-          echo ']' >> site/data/results.json
-
-      - name: Commit results
+      - name: Commit and push results
+        if: steps.detect.outputs.list != ''
         run: |
           git config user.name "HttpArena Bot"
           git config user.email "bot@httparena"
-          git add site/data/results.json
-          git diff --cached --quiet || git commit -m "Update benchmark results" && git push
+          git add results/ site/data/ site/static/logs/
+          if ! git diff --cached --quiet; then
+            git commit -m "benchmark: update results for ${{ steps.detect.outputs.list }}"
+            git push
+          fi

frameworks/nginx/nginx.conf

@@ -41,6 +41,11 @@ http {
         ssl_certificate /certs/server.crt;
         ssl_certificate_key /certs/server.key;
         ssl_protocols TLSv1.3;
+        ssl_session_tickets on;
+        ssl_session_cache shared:SSL:10m;
+
+        http2_max_concurrent_streams 256;
+        http2_max_requests 10000000;
 
         add_header Alt-Svc 'h3=":8443"; ma=86400';
 

scripts/benchmark.sh

@@ -10,7 +10,8 @@ H2LOAD="${H2LOAD:-h2load}"
 OHA="${OHA:-$HOME/.cargo/bin/oha}"
 HARD_NOFILE=$(ulimit -Hn)
 ulimit -n "$HARD_NOFILE"
-THREADS="${THREADS:-12}"
+THREADS="${THREADS:-64}"
+H2THREADS="${H2THREADS:-128}"
 DURATION=5s
 RUNS=3
 PORT=8080
@@ -30,8 +31,8 @@ declare -A PROFILES=(
     [upload]="1|0||64,256,512|upload"
     [compression]="1|0||4096,16384|compression"
     [noisy]="1|0||512,4096,16384|noisy"
-    [baseline-h2]="1|0||64,256,1024|h2"
-    [static-h2]="1|0||64,256,1024|static-h2"
+    [baseline-h2]="1|0||256,1024|h2"
+    [static-h2]="1|0||256,1024|static-h2"
     [baseline-h3]="32|0||256,512|h3"
     [static-h3]="32|0||256,512|static-h3"
 )
@@ -208,11 +209,31 @@ cleanup() {
     docker stop -t 5 "$CONTAINER_NAME" 2>/dev/null || true
     docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
 }
-trap cleanup EXIT
+
+# Save original CPU governor
+ORIG_GOVERNOR=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "")
+
+restore_settings() {
+    docker stop -t 5 "$CONTAINER_NAME" 2>/dev/null || true
+    docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
+    if [ -n "$ORIG_GOVERNOR" ]; then
+        echo "[restore] Restoring CPU governor to $ORIG_GOVERNOR..."
+        for g in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
+            sudo sh -c "echo $ORIG_GOVERNOR > $g" 2>/dev/null || true
+        done
+    fi
+}
+trap restore_settings EXIT
 
 # Clean slate: stop containers, restart Docker, drop caches
 docker ps -q --filter "name=httparena-" | xargs -r docker stop -t 5 2>/dev/null || true
 docker ps -aq --filter "name=httparena-" | xargs -r docker rm -f 2>/dev/null || true
+
+echo "[tune] Setting CPU governor to performance..."
+for g in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
+    sudo sh -c "echo performance > $g" 2>/dev/null || true
+done
+
 echo "[clean] Restarting Docker daemon..."
 sudo systemctl restart docker
 sleep 3
@@ -327,12 +348,12 @@ for profile in "${profiles_to_run[@]}"; do
         USE_H2LOAD=true
         gc_args=("$H2LOAD"
             -i "$REQUESTS_DIR/static-h2-uris.txt"
-            -c "$CONNS" -m 100 -t "$THREADS" -D "$DURATION")
+            -c "$CONNS" -m 100 -t "$H2THREADS" -D "$DURATION")
     elif [ "$endpoint" = "h2" ]; then
         USE_H2LOAD=true
         gc_args=("$H2LOAD"
             "https://localhost:$H2PORT/baseline2?a=1&b=1"
-            -c "$CONNS" -m 100 -t "$THREADS" -D "$DURATION")
+            -c "$CONNS" -m 100 -t "$H2THREADS" -D "$DURATION")
     elif [ "$endpoint" = "pipeline" ]; then
         gc_args=("http://localhost:$PORT/pipeline"
             -c "$CONNS" -t "$THREADS" -d "$DURATION" -p "$pipeline")

site/content/docs/running-locally/running-benchmarks.md

@@ -54,3 +54,16 @@ With `--save`, it additionally:
 8. Rebuilds site data files in `site/data/`
 
 For HTTP/1.1 profiles (`baseline`, `pipelined`, `limited-conn`, `json`, `upload`, `compression`, `noisy`), the load generator is **gcannon**. For HTTP/2 profiles (`baseline-h2`, `static-h2`), the load generator is **h2load**. For HTTP/3 profiles (`baseline-h3`, `static-h3`), the load generator is **oha**.
+
+## Environment variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `THREADS` | `64` | Number of threads for **gcannon** (HTTP/1.1 load generator) |
+| `H2THREADS` | `128` | Number of threads for **h2load** (HTTP/2 load generator) |
+
+Example — run with custom thread counts:
+
+```bash
+THREADS=8 H2THREADS=128 ./scripts/benchmark.sh aspnet-minimal
+```

site/data/baseline-16384.json

@@ -162,18 +162,18 @@
 {
   "framework": "h2o",
   "language": "C",
-  "rps": 2903962,
-  "avg_latency": "5.47ms",
-  "p99_latency": "21.20ms",
-  "cpu": "6345.3%",
+  "rps": 2930585,
+  "avg_latency": "5.42ms",
+  "p99_latency": "22.50ms",
+  "cpu": "6318.3%",
   "memory": "192.7MiB",
   "connections": 16384,
   "threads": 64,
   "duration": "5s",
   "pipeline": 1,
-  "bandwidth": "285.42MB/s",
+  "bandwidth": "287.63MB/s",
   "reconnects": 0,
-  "status_2xx": 14548851,
+  "status_2xx": 14652926,
   "status_3xx": 0,
   "status_4xx": 0,
   "status_5xx": 0

site/data/baseline-4096.json

@@ -162,18 +162,18 @@
 {
   "framework": "h2o",
   "language": "C",
-  "rps": 3163543,
-  "avg_latency": "1.29ms",
-  "p99_latency": "8.28ms",
-  "cpu": "6851.9%",
-  "memory": "77.6MiB",
+  "rps": 3246907,
+  "avg_latency": "1.26ms",
+  "p99_latency": "8.05ms",
+  "cpu": "6849.2%",
+  "memory": "80.4MiB",
   "connections": 4096,
   "threads": 64,
   "duration": "5s",
   "pipeline": 1,
-  "bandwidth": "310.61MB/s",
+  "bandwidth": "318.76MB/s",
   "reconnects": 0,
-  "status_2xx": 15817715,
+  "status_2xx": 16234537,
   "status_3xx": 0,
   "status_4xx": 0,
   "status_5xx": 0

site/data/baseline-512.json

@@ -162,18 +162,18 @@
 {
   "framework": "h2o",
   "language": "C",
-  "rps": 2706107,
-  "avg_latency": "188us",
-  "p99_latency": "1.31ms",
-  "cpu": "6747.4%",
-  "memory": "55.8MiB",
+  "rps": 2768745,
+  "avg_latency": "184us",
+  "p99_latency": "1.20ms",
+  "cpu": "6565.5%",
+  "memory": "54.9MiB",
   "connections": 512,
   "threads": 64,
   "duration": "5s",
   "pipeline": 1,
-  "bandwidth": "265.73MB/s",
+  "bandwidth": "271.73MB/s",
   "reconnects": 0,
-  "status_2xx": 13530535,
+  "status_2xx": 13843729,
   "status_3xx": 0,
   "status_4xx": 0,
   "status_5xx": 0

site/data/baseline-h2-1024.json

@@ -22,18 +22,18 @@
 {
   "framework": "aspnet-minimal",
   "language": "C#",
-  "rps": 198526,
-  "avg_latency": "424.18ms",
-  "p99_latency": "424.18ms",
-  "cpu": "11525.2%",
-  "memory": "2.2GiB",
+  "rps": 204357,
+  "avg_latency": "373.55ms",
+  "p99_latency": "373.55ms",
+  "cpu": "11707.1%",
+  "memory": "1.9GiB",
   "connections": 1024,
   "threads": 64,
   "duration": "5s",
   "pipeline": 1,
-  "bandwidth": "5.35MB/s",
+  "bandwidth": "5.50MB/s",
   "reconnects": 0,
-  "status_2xx": 992634,
+  "status_2xx": 1021787,
   "status_3xx": 0,
   "status_4xx": 0,
   "status_5xx": 0
@@ -62,18 +62,18 @@
 {
   "framework": "drogon",
   "language": "C++",
-  "rps": 11253340,
-  "avg_latency": "9.27ms",
-  "p99_latency": "9.27ms",
-  "cpu": "4875.7%",
-  "memory": "364.7MiB",
+  "rps": 14386780,
+  "avg_latency": "6.79ms",
+  "p99_latency": "6.79ms",
+  "cpu": "5102.7%",
+  "memory": "390.4MiB",
   "connections": 1024,
   "threads": 64,
   "duration": "5s",
   "pipeline": 1,
-  "bandwidth": "1.56GB/s",
+  "bandwidth": "2.00GB/s",
   "reconnects": 0,
-  "status_2xx": 56266700,
+  "status_2xx": 71933900,
   "status_3xx": 0,
   "status_4xx": 0,
   "status_5xx": 0
@@ -82,18 +82,18 @@
 {
   "framework": "h2o",
   "language": "C",
-  "rps": 9387640,
-  "avg_latency": "5.02ms",
-  "p99_latency": "5.02ms",
-  "cpu": "2063.4%",
-  "memory": "200.9MiB",
+  "rps": 14329780,
+  "avg_latency": "3.58ms",
+  "p99_latency": "3.58ms",
+  "cpu": "2556.3%",
+  "memory": "198.0MiB",
   "connections": 1024,
   "threads": 64,
   "duration": "5s",
   "pipeline": 1,
-  "bandwidth": "232.78MB/s",
+  "bandwidth": "355.33MB/s",
   "reconnects": 0,
-  "status_2xx": 46938200,
+  "status_2xx": 71648900,
   "status_3xx": 0,
   "status_4xx": 0,
   "status_5xx": 0
@@ -142,18 +142,18 @@
 {
   "framework": "nginx",
   "language": "C",
-  "rps": 3315310,
-  "avg_latency": "20.26ms",
-  "p99_latency": "20.26ms",
-  "cpu": "6456.5%",
+  "rps": 3407369,
+  "avg_latency": "21.61ms",
+  "p99_latency": "21.61ms",
+  "cpu": "6927.2%",
   "memory": "3.5GiB",
   "connections": 1024,
   "threads": 64,
   "duration": "5s",
   "pipeline": 1,
-  "bandwidth": "271.92MB/s",
+  "bandwidth": "279.47MB/s",
   "reconnects": 0,
-  "status_2xx": 16576553,
+  "status_2xx": 17036845,
   "status_3xx": 0,
   "status_4xx": 0,
   "status_5xx": 0

site/data/baseline-h2-2048.json

@@ -0,0 +1,21 @@
+[
+{
+  "framework": "h2o",
+  "language": "C",
+  "rps": 13938980,
+  "avg_latency": "8.38ms",
+  "p99_latency": "8.38ms",
+  "cpu": "2680.3%",
+  "memory": "268.9MiB",
+  "connections": 2048,
+  "threads": 64,
+  "duration": "5s",
+  "pipeline": 1,
+  "bandwidth": "345.65MB/s",
+  "reconnects": 0,
+  "status_2xx": 69694900,
+  "status_3xx": 0,
+  "status_4xx": 0,
+  "status_5xx": 0
+}
+]