From 47f9aa7453a75117601cb765054fd0a4ecc45a40 Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Thu, 14 May 2026 17:32:20 +0300
Subject: [PATCH 01/11] chore: upgrade the benchmark

---
 tests/performance/chart.js             | 42 +++++++++++++++++---------
 tests/performance/make-test-command.sh |  6 ++--
 tests/performance/package.json         |  6 ++--
 3 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/tests/performance/chart.js b/tests/performance/chart.js
index 9cfcb8fb0c..9f6ddb29c6 100644
--- a/tests/performance/chart.js
+++ b/tests/performance/chart.js
@@ -2,16 +2,28 @@ import fs from 'node:fs';
 
 const content = fs.readFileSync('benchmark_check.json', 'utf8');
 const json = JSON.parse(content);
+
+/** Returns the median of a numeric array without mutating the input. */
+const median = (xs) => {
+  const sorted = [...xs].sort((a, b) => a - b);
+  const mid = sorted.length >> 1;
+  return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+};
+
+/** Median absolute deviation around a given centre — a robust alternative to stddev. */
+const medianAbsoluteDeviation = (xs, centre) => median(xs.map((x) => Math.abs(x - centre)));
+
 const arr = json.results.map((r) => [
   r.command.replace(/^node node_modules\/([^/]+)\/.*/, (_, cliVersion) => cliVersion),
-  r.mean,
-  r.stddev,
+  r.median,
+  medianAbsoluteDeviation(r.times, r.median),
 ]);
-const minMean = Math.min(...arr.map(([_, mean]) => mean));
+const minMedian = Math.min(...arr.map(([_, m]) => m));
 
-const constructBarForChart = (mean, min) => {
+/** Builds a unicode bar whose length scales with how much slower a result is than the fastest. */
+const constructBarForChart = (value, min) => {
   if (min <= 0) return 'N/A';
-  const slownessRatio = mean / min;
+  const slownessRatio = value / min;
   const slownessFactor = slownessRatio - 1;
   const maxBarLength = 30;
   const visualFactor = Math.min(1, slownessFactor);
@@ -20,19 +32,19 @@ const constructBarForChart = (mean, min) => {
 };
 
 const output = [
-  '| CLI Version | Mean Time ± Std Dev (s) | Relative Performance (Lower is Faster) |',
+  '| CLI Version | Median Time ± MAD (s) | Relative Performance (Lower is Faster) |',
   '|---|---|---|',
-  ...arr.map(([cliVersion, mean, stddev]) => {
-    const bar = constructBarForChart(mean, minMean);
-    const meanFormatted = mean.toFixed(3);
-    const stddevFormatted = stddev.toFixed(3);
-    const relativeSpeedFactor = (mean / minMean).toFixed(2);
-    const factorSuffix = mean === minMean ? 'x (Fastest)' : 'x';
-
-    const timeWithStddev = `${meanFormatted}s ± ${stddevFormatted}s`;
+  ...arr.map(([cliVersion, medianValue, madValue]) => {
+    const bar = constructBarForChart(medianValue, minMedian);
+    const medianFormatted = medianValue.toFixed(3);
+    const madFormatted = madValue.toFixed(3);
+    const relativeSpeedFactor = (medianValue / minMedian).toFixed(2);
+    const factorSuffix = medianValue === minMedian ? 'x (Fastest)' : 'x';
+
+    const timeWithSpread = `${medianFormatted}s ± ${madFormatted}s`;
     const performanceDisplay = `${bar} ${relativeSpeedFactor}${factorSuffix}`;
 
-    return `| ${cliVersion} | ${timeWithStddev} | ${performanceDisplay} |`;
+    return `| ${cliVersion} | ${timeWithSpread} | ${performanceDisplay} |`;
   }),
 ].join('\n');
 
diff --git a/tests/performance/make-test-command.sh b/tests/performance/make-test-command.sh
index 4b1d96cfd5..e468a7e661 100644
--- a/tests/performance/make-test-command.sh
+++ b/tests/performance/make-test-command.sh
@@ -6,8 +6,10 @@ set -eo pipefail # Fail on script errors
 git clone https://github.com/Rebilly/api-definitions.git
 cd api-definitions && pnpm install && cd ..
 
-# Store the command into a text file:
-echo REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
+# Store the command into a text file.
+# Pre-warm the OS page cache so the first measured run does not pay a cold-cache
+# penalty; with that done, --warmup 1 is enough to stabilise per-process state.
+echo "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null &&" REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
 
 # Put the command in the test section of the package.json:
 cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json
diff --git a/tests/performance/package.json b/tests/performance/package.json
index 0f5d76d56e..cb41fc636b 100644
--- a/tests/performance/package.json
+++ b/tests/performance/package.json
@@ -7,9 +7,9 @@
   "scripts": {
     "chart": "node chart.js > benchmark_chart.md",
     "make-test": "bash make-test-command.sh",
-    "test:bundle": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
-    "test:lint": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
-    "test:check-config": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
+    "test:bundle": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
+    "test:lint": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
+    "test:check-config": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
   },
   "_enhancedDependencies": {
     "cli-2.0.0": "npm:@redocly/cli@2.0.0",

From bdbb7d34e1d06f51235b525552ba294a4b3a0511 Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Thu, 14 May 2026 17:45:09 +0300
Subject: [PATCH 02/11] chore: add trigger by label

---
 .github/workflows/performance.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yaml b/.github/workflows/performance.yaml
index 9244de9708..3ca17f807f 100644
--- a/.github/workflows/performance.yaml
+++ b/.github/workflows/performance.yaml
@@ -56,8 +56,12 @@ jobs:
           npm run chart # Creates benchmark_chart.md with the performance bar chart.
 
       - name: Comment PR
+        # If the PR carries the `performance-benchmark` label, each push posts a
+        # fresh comment (so variance across reruns can be compared in-thread);
+        # otherwise the comment is overwritten in place so normal PRs keep a
+        # single result.
         if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1
         with:
           file-path: tests/performance/benchmark_chart.md
-          comment-tag: historical-versions-comparison
+          comment-tag: ${{ contains(github.event.pull_request.labels.*.name, 'performance-benchmark') && format('historical-versions-rerun-{0}', github.sha) || 'historical-versions-comparison' }}

From adfa2bfde20e7b800b8b54422b340291fe8b93c8 Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Thu, 14 May 2026 18:01:09 +0300
Subject: [PATCH 03/11] chore: add triggering by label

---
 .github/workflows/performance.yaml | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/performance.yaml b/.github/workflows/performance.yaml
index 3ca17f807f..57f090dff5 100644
--- a/.github/workflows/performance.yaml
+++ b/.github/workflows/performance.yaml
@@ -8,6 +8,10 @@ on:
   pull_request:
     branches:
       - main
+    # `labeled` lets applying the `performance-benchmark` label trigger a rerun
+    # without needing an empty commit; the job-level `if` filters out unrelated
+    # label changes so we don't burn CI on every label add.
+    types: [opened, synchronize, reopened, labeled]
 
 env:
   CI: true
@@ -16,6 +20,9 @@ env:
 jobs:
   historical-versions:
     runs-on: ubuntu-latest
+    # Skip the job when a `labeled` event fires for an unrelated label —
+    # opens/pushes/reopens always run; only the label firehose is filtered.
+    if: github.event.action != 'labeled' || github.event.label.name == 'performance-benchmark'
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Install pnpm for Rebilly/api-definitions
@@ -33,8 +40,9 @@ jobs:
         run: npm i -g hyperfine
 
       - name: Add more versions to test
-        # Run only on the release branch (changeset-release/main):
-        if: github.head_ref == 'changeset-release/main'
+        # Runs on the release PR (changeset-release/main) automatically, or on
+        # any PR carrying the `performance-benchmark` label for opt-in deep-dive.
+        if: github.head_ref == 'changeset-release/main' || contains(github.event.pull_request.labels.*.name, 'performance-benchmark')
         run: |
           cd tests/performance/
           cat package.json | jq ".dependencies = $(cat package.json | jq ._enhancedDependencies)" > package.json
@@ -64,4 +72,4 @@ jobs:
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1
         with:
           file-path: tests/performance/benchmark_chart.md
-          comment-tag: ${{ contains(github.event.pull_request.labels.*.name, 'performance-benchmark') && format('historical-versions-rerun-{0}', github.sha) || 'historical-versions-comparison' }}
+          comment-tag: ${{ contains(github.event.pull_request.labels.*.name, 'performance-benchmark') && format('historical-versions-rerun-{0}', github.run_id) || 'historical-versions-comparison' }}

From c827c6faef781d7525f5bab099d098f9a6f033b6 Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Tue, 26 May 2026 14:44:20 +0300
Subject: [PATCH 04/11] fix: remove find warmup, add warmup 2 and improve
 chart.js

---
 tests/performance/chart.js             | 2 +-
 tests/performance/make-test-command.sh | 6 ++----
 tests/performance/package.json         | 6 +++---
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/tests/performance/chart.js b/tests/performance/chart.js
index 9f6ddb29c6..4e523b6d59 100644
--- a/tests/performance/chart.js
+++ b/tests/performance/chart.js
@@ -6,7 +6,7 @@ const json = JSON.parse(content);
 /** Returns the median of a numeric array without mutating the input. */
 const median = (xs) => {
   const sorted = [...xs].sort((a, b) => a - b);
-  const mid = sorted.length >> 1;
+  const mid = Math.floor(sorted.length / 2);
   return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
 };
 
diff --git a/tests/performance/make-test-command.sh b/tests/performance/make-test-command.sh
index e468a7e661..32063957c7 100644
--- a/tests/performance/make-test-command.sh
+++ b/tests/performance/make-test-command.sh
@@ -6,10 +6,8 @@ set -eo pipefail # Fail on script errors
 git clone https://github.com/Rebilly/api-definitions.git
 cd api-definitions && pnpm install && cd ..
 
-# Store the command into a text file.
-# Pre-warm the OS page cache so the first measured run does not pay a cold-cache
-# penalty; with that done, --warmup 1 is enough to stabilise per-process state.
-echo "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null &&" REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
+# Store the command into a text file:
+echo REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
 
 # Put the command in the test section of the package.json:
 cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json
diff --git a/tests/performance/package.json b/tests/performance/package.json
index cb41fc636b..f7549f72f9 100644
--- a/tests/performance/package.json
+++ b/tests/performance/package.json
@@ -7,9 +7,9 @@
   "scripts": {
     "chart": "node chart.js > benchmark_chart.md",
     "make-test": "bash make-test-command.sh",
-    "test:bundle": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
-    "test:lint": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
-    "test:check-config": "find node_modules/cli-* -type f -print0 | xargs -0 cat > /dev/null && find api-definitions -type f -print0 | xargs -0 cat > /dev/null && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 --min-runs 10 --max-runs 15 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
+    "test:bundle": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
+    "test:lint": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
+    "test:check-config": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
   },
   "_enhancedDependencies": {
     "cli-2.0.0": "npm:@redocly/cli@2.0.0",

From 3c943495d37a6917228d777c0c64c6c7595b401e Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Tue, 26 May 2026 17:00:02 +0300
Subject: [PATCH 05/11] chore: remove prev version from benchmark

---
 tests/performance/package.json | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/tests/performance/package.json b/tests/performance/package.json
index f7549f72f9..8c62817918 100644
--- a/tests/performance/package.json
+++ b/tests/performance/package.json
@@ -13,20 +13,12 @@
   },
   "_enhancedDependencies": {
     "cli-2.0.0": "npm:@redocly/cli@2.0.0",
-    "cli-2.03.1": "npm:@redocly/cli@2.3.1",
-    "cli-2.08.0": "npm:@redocly/cli@2.8.0",
     "cli-2.11.1": "npm:@redocly/cli@2.11.1",
-    "cli-2.12.0": "npm:@redocly/cli@2.12.0",
-    "cli-2.12.2": "npm:@redocly/cli@2.12.2",
     "cli-2.13.0": "npm:@redocly/cli@2.13.0",
-    "cli-2.14.1": "npm:@redocly/cli@2.14.1",
     "cli-2.14.2": "npm:@redocly/cli@2.14.2",
-    "cli-2.19.2": "npm:@redocly/cli@2.19.2",
-    "cli-2.24.1": "npm:@redocly/cli@2.24.1",
-    "cli-2.25.0": "npm:@redocly/cli@2.25.0",
-    "cli-2.25.4": "npm:@redocly/cli@2.25.4",
     "cli-2.27.0": "npm:@redocly/cli@2.27.0",
     "cli-2.30.2": "npm:@redocly/cli@2.30.2",
+    "cli-2.31.0": "npm:@redocly/cli@2.31.0",
     "cli-latest": "npm:@redocly/cli@latest",
     "cli-next": "file:../../redocly-cli.tgz"
   },

From 4d82cb9f8f9945c741c5893ddbe98ceaa3261f37 Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Tue, 26 May 2026 17:21:10 +0300
Subject: [PATCH 06/11] feat: add running of lint command in perf benchmark

---
 tests/performance/.gitignore           |  6 ++-
 tests/performance/chart.js             | 61 +++++++++++++-------------
 tests/performance/make-test-command.sh |  7 ++-
 tests/performance/package.json         |  2 +-
 4 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/tests/performance/.gitignore b/tests/performance/.gitignore
index 7d90fe0bc1..de79c2e923 100644
--- a/tests/performance/.gitignore
+++ b/tests/performance/.gitignore
@@ -2,6 +2,8 @@ api-definitions
 node_modules
 package-lock.json
 test-command.txt
-benchmark_check.md
-benchmark_check.json
+benchmark_bundle.md
+benchmark_bundle.json
+benchmark_lint.md
+benchmark_lint.json
 benchmark_chart.md
diff --git a/tests/performance/chart.js b/tests/performance/chart.js
index 4e523b6d59..adeacaf642 100644
--- a/tests/performance/chart.js
+++ b/tests/performance/chart.js
@@ -1,51 +1,50 @@
 import fs from 'node:fs';
 
-const content = fs.readFileSync('benchmark_check.json', 'utf8');
-const json = JSON.parse(content);
-
-/** Returns the median of a numeric array without mutating the input. */
 const median = (xs) => {
   const sorted = [...xs].sort((a, b) => a - b);
   const mid = Math.floor(sorted.length / 2);
   return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
 };
 
-/** Median absolute deviation around a given centre — a robust alternative to stddev. */
 const medianAbsoluteDeviation = (xs, centre) => median(xs.map((x) => Math.abs(x - centre)));
 
-const arr = json.results.map((r) => [
-  r.command.replace(/^node node_modules\/([^/]+)\/.*/, (_, cliVersion) => cliVersion),
-  r.median,
-  medianAbsoluteDeviation(r.times, r.median),
-]);
-const minMedian = Math.min(...arr.map(([_, m]) => m));
-
-/** Builds a unicode bar whose length scales with how much slower a result is than the fastest. */
 const constructBarForChart = (value, min) => {
   if (min <= 0) return 'N/A';
-  const slownessRatio = value / min;
-  const slownessFactor = slownessRatio - 1;
+  const slownessFactor = value / min - 1;
   const maxBarLength = 30;
-  const visualFactor = Math.min(1, slownessFactor);
-  const length = Math.floor(visualFactor * maxBarLength);
+  const length = Math.floor(Math.min(1, slownessFactor) * maxBarLength);
   return '▓' + '▓'.repeat(length);
 };
 
-const output = [
-  '| CLI Version | Median Time ± MAD (s) | Relative Performance (Lower is Faster) |',
-  '|---|---|---|',
-  ...arr.map(([cliVersion, medianValue, madValue]) => {
-    const bar = constructBarForChart(medianValue, minMedian);
-    const medianFormatted = medianValue.toFixed(3);
-    const madFormatted = madValue.toFixed(3);
-    const relativeSpeedFactor = (medianValue / minMedian).toFixed(2);
-    const factorSuffix = medianValue === minMedian ? 'x (Fastest)' : 'x';
-
-    const timeWithSpread = `${medianFormatted}s ± ${madFormatted}s`;
-    const performanceDisplay = `${bar} ${relativeSpeedFactor}${factorSuffix}`;
+const renderSection = (jsonPath, title) => {
+  const json = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
+  const rows = json.results.map((r) => [
+    r.command.replace(/^node node_modules\/([^/]+)\/.*/, (_, cliVersion) => cliVersion),
+    r.median,
+    medianAbsoluteDeviation(r.times, r.median),
+  ]);
+  const fastest = Math.min(...rows.map(([, value]) => value));
+
+  return [
+    `### ${title}`,
+    '',
+    '| CLI Version | Median Time ± MAD (s) | Relative Performance (Lower is Faster) |',
+    '|---|---|---|',
+    ...rows.map(([cliVersion, medianValue, madValue]) => {
+      const bar = constructBarForChart(medianValue, fastest);
+      const factor = (medianValue / fastest).toFixed(2);
+      const suffix = medianValue === fastest ? 'x (Fastest)' : 'x';
+      return `| ${cliVersion} | ${medianValue.toFixed(3)}s ± ${madValue.toFixed(3)}s | ${bar} ${factor}${suffix} |`;
+    }),
+  ].join('\n');
+};
 
-    return `| ${cliVersion} | ${timeWithSpread} | ${performanceDisplay} |`;
-  }),
+const output = [
+  '## Performance Benchmark',
+  '',
+  renderSection('benchmark_bundle.json', 'Bundle'),
+  '',
+  renderSection('benchmark_lint.json', 'Lint'),
 ].join('\n');
 
 process.stdout.write(output);
diff --git a/tests/performance/make-test-command.sh b/tests/performance/make-test-command.sh
index 32063957c7..0397b1f51f 100644
--- a/tests/performance/make-test-command.sh
+++ b/tests/performance/make-test-command.sh
@@ -6,8 +6,11 @@ set -eo pipefail # Fail on script errors
 git clone https://github.com/Rebilly/api-definitions.git
 cd api-definitions && pnpm install && cd ..
 
-# Store the command into a text file:
-echo REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
+BUNDLE_CMDS=$(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs)
+LINT_CMDS=$(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'\''")' | jq 'join(" ")' | xargs)
+
+# Lint --prepare wipes the ignore file each run so iterations do equal work:
+echo "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 ${BUNDLE_CMDS} --export-markdown benchmark_bundle.md --export-json benchmark_bundle.json && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 --prepare 'rm -f api-definitions/.redocly.lint-ignore.yaml' ${LINT_CMDS} --export-markdown benchmark_lint.md --export-json benchmark_lint.json" > test-command.txt
 
 # Put the command in the test section of the package.json:
 cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json
diff --git a/tests/performance/package.json b/tests/performance/package.json
index 8c62817918..4560f9c52a 100644
--- a/tests/performance/package.json
+++ b/tests/performance/package.json
@@ -8,7 +8,7 @@
     "chart": "node chart.js > benchmark_chart.md",
     "make-test": "bash make-test-command.sh",
     "test:bundle": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
-    "test:lint": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
+    "test:lint": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 --prepare 'rm -f api-definitions/.redocly.lint-ignore.yaml' 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
     "test:check-config": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
   },
   "_enhancedDependencies": {

From 96f1c13c8c15f9fd7b51f2c037f1debcc50b6e7c Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Tue, 26 May 2026 17:48:18 +0300
Subject: [PATCH 07/11] feat: add missing benchmark files

---
 .github/workflows/performance.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/performance.yaml b/.github/workflows/performance.yaml
index 57f090dff5..3236b8f897 100644
--- a/.github/workflows/performance.yaml
+++ b/.github/workflows/performance.yaml
@@ -60,7 +60,7 @@ jobs:
         run: |
           cd tests/performance/
           npm run test # This command is generated and injected into package.json in the previous step.
-          cat benchmark_check.md
+          cat benchmark_bundle.md benchmark_lint.md
           npm run chart # Creates benchmark_chart.md with the performance bar chart.
 
       - name: Comment PR

From 987e85423d6901cdc213a57104cc83c0c41fda2e Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Thu, 28 May 2026 15:02:03 +0300
Subject: [PATCH 08/11] chore: refactor test command bash script

---
 tests/performance/make-test-command.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/performance/make-test-command.sh b/tests/performance/make-test-command.sh
index 0397b1f51f..7805873420 100644
--- a/tests/performance/make-test-command.sh
+++ b/tests/performance/make-test-command.sh
@@ -6,11 +6,11 @@ set -eo pipefail # Fail on script errors
 git clone https://github.com/Rebilly/api-definitions.git
 cd api-definitions && pnpm install && cd ..
 
-BUNDLE_CMDS=$(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'\''")' | jq 'join(" ")' | xargs)
-LINT_CMDS=$(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'\''")' | jq 'join(" ")' | xargs)
+build_cmds() {
+  jq -r --arg sub "$1" --arg extra "${2:-}" '.dependencies | keys | map("'\''node node_modules/" + . + "/bin/cli.js " + $sub + " all@latest --config=api-definitions/redocly.yaml" + $extra + "'\''") | join(" ")' package.json
+}
 
-# Lint --prepare wipes the ignore file each run so iterations do equal work:
-echo "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 ${BUNDLE_CMDS} --export-markdown benchmark_bundle.md --export-json benchmark_bundle.json && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 --prepare 'rm -f api-definitions/.redocly.lint-ignore.yaml' ${LINT_CMDS} --export-markdown benchmark_lint.md --export-json benchmark_lint.json" > test-command.txt
+echo "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 $(build_cmds 'bundle') --export-markdown benchmark_bundle.md --export-json benchmark_bundle.json && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 --prepare 'rm -f api-definitions/.redocly.lint-ignore.yaml' $(build_cmds 'lint' ' --generate-ignore-file') --export-markdown benchmark_lint.md --export-json benchmark_lint.json" > test-command.txt
 
 # Put the command in the test section of the package.json:
 cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json

From 9599c52df062af95df7d2759b339e45da4ea9cd9 Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Thu, 28 May 2026 15:05:40 +0300
Subject: [PATCH 09/11] chore: add comment

---
 tests/performance/make-test-command.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/performance/make-test-command.sh b/tests/performance/make-test-command.sh
index 7805873420..c1c4465f49 100644
--- a/tests/performance/make-test-command.sh
+++ b/tests/performance/make-test-command.sh
@@ -6,6 +6,7 @@ set -eo pipefail # Fail on script errors
 git clone https://github.com/Rebilly/api-definitions.git
 cd api-definitions && pnpm install && cd ..
 
+# Store the command into a text file:
 build_cmds() {
   jq -r --arg sub "$1" --arg extra "${2:-}" '.dependencies | keys | map("'\''node node_modules/" + . + "/bin/cli.js " + $sub + " all@latest --config=api-definitions/redocly.yaml" + $extra + "'\''") | join(" ")' package.json
 }

From 9a61a655e9588c0ddf985a09124c7b6234bb6d13 Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Thu, 28 May 2026 18:23:30 +0300
Subject: [PATCH 10/11] fix: based on review

---
 .github/workflows/performance.yaml     |  2 +-
 tests/performance/.gitignore           |  2 +
 tests/performance/chart.js             | 61 ++++++++++++++++----------
 tests/performance/make-test-command.sh |  4 +-
 tests/performance/package.json         |  6 +--
 5 files changed, 46 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/performance.yaml b/.github/workflows/performance.yaml
index 3236b8f897..e61264f2b1 100644
--- a/.github/workflows/performance.yaml
+++ b/.github/workflows/performance.yaml
@@ -60,7 +60,7 @@ jobs:
         run: |
           cd tests/performance/
           npm run test # This command is generated and injected into package.json in the previous step.
-          cat benchmark_bundle.md benchmark_lint.md
+          cat benchmark_bundle.md benchmark_lint.md benchmark_check-config.md
           npm run chart # Creates benchmark_chart.md with the performance bar chart.
 
       - name: Comment PR
diff --git a/tests/performance/.gitignore b/tests/performance/.gitignore
index de79c2e923..e59ba9b4a2 100644
--- a/tests/performance/.gitignore
+++ b/tests/performance/.gitignore
@@ -6,4 +6,6 @@ benchmark_bundle.md
 benchmark_bundle.json
 benchmark_lint.md
 benchmark_lint.json
+benchmark_check-config.md
+benchmark_check-config.json
 benchmark_chart.md
diff --git a/tests/performance/chart.js b/tests/performance/chart.js
index adeacaf642..4a3a618085 100644
--- a/tests/performance/chart.js
+++ b/tests/performance/chart.js
@@ -16,35 +16,50 @@ const constructBarForChart = (value, min) => {
   return '▓' + '▓'.repeat(length);
 };
 
-const renderSection = (jsonPath, title) => {
+const loadResults = (jsonPath) => {
   const json = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
-  const rows = json.results.map((r) => [
-    r.command.replace(/^node node_modules\/([^/]+)\/.*/, (_, cliVersion) => cliVersion),
-    r.median,
-    medianAbsoluteDeviation(r.times, r.median),
-  ]);
-  const fastest = Math.min(...rows.map(([, value]) => value));
-
-  return [
-    `### ${title}`,
-    '',
-    '| CLI Version | Median Time ± MAD (s) | Relative Performance (Lower is Faster) |',
-    '|---|---|---|',
-    ...rows.map(([cliVersion, medianValue, madValue]) => {
-      const bar = constructBarForChart(medianValue, fastest);
-      const factor = (medianValue / fastest).toFixed(2);
-      const suffix = medianValue === fastest ? 'x (Fastest)' : 'x';
-      return `| ${cliVersion} | ${medianValue.toFixed(3)}s ± ${madValue.toFixed(3)}s | ${bar} ${factor}${suffix} |`;
-    }),
-  ].join('\n');
+  return new Map(
+    json.results.map(({ command, median, times }) => {
+      const cliVersion = command.replace(/^node node_modules\/([^/]+)\/.*/, (_, v) => v);
+      return [cliVersion, { median, mad: medianAbsoluteDeviation(times, median) }];
+    })
+  );
 };
 
+const findFastest = (results) =>
+  [...results.values()].reduce((best, r) => (r.median < best.median ? r : best));
+
+const renderCell = (entry, fastest) => {
+  const bar = constructBarForChart(entry.median, fastest.median);
+  const factor = entry.median / fastest.median;
+  if (entry === fastest) {
+    return `${bar} ${factor.toFixed(2)}x (Fastest)`;
+  }
+  const relativeUnc =
+    factor * Math.sqrt((entry.mad / entry.median) ** 2 + (fastest.mad / fastest.median) ** 2);
+  return `${bar} ${factor.toFixed(2)}x ± ${relativeUnc.toFixed(2)}`;
+};
+
+const operations = [
+  { name: 'Bundle', file: 'benchmark_bundle.json' },
+  { name: 'Lint', file: 'benchmark_lint.json' },
+  { name: 'Check Config', file: 'benchmark_check-config.json' },
+];
+
+const columns = operations.map(({ name, file }) => {
+  const data = loadResults(file);
+  return { name, data, fastest: findFastest(data) };
+});
+const versions = [...columns[0].data.keys()];
+
 const output = [
   '## Performance Benchmark',
   '',
-  renderSection('benchmark_bundle.json', 'Bundle'),
-  '',
-  renderSection('benchmark_lint.json', 'Lint'),
+  `| CLI Version | ${columns.map((c) => c.name).join(' | ')} |`,
+  `|---|${columns.map(() => '---').join('|')}|`,
+  ...versions.map(
+    (v) => `| ${v} | ${columns.map((c) => renderCell(c.data.get(v), c.fastest)).join(' | ')} |`
+  ),
 ].join('\n');
 
 process.stdout.write(output);
diff --git a/tests/performance/make-test-command.sh b/tests/performance/make-test-command.sh
index c1c4465f49..2b5a42c34b 100644
--- a/tests/performance/make-test-command.sh
+++ b/tests/performance/make-test-command.sh
@@ -8,10 +8,10 @@ cd api-definitions && pnpm install && cd ..
 
 # Store the command into a text file:
 build_cmds() {
-  jq -r --arg sub "$1" --arg extra "${2:-}" '.dependencies | keys | map("'\''node node_modules/" + . + "/bin/cli.js " + $sub + " all@latest --config=api-definitions/redocly.yaml" + $extra + "'\''") | join(" ")' package.json
+  jq -r --arg suffix "$1" '.dependencies | keys | map("'\''node node_modules/" + . + "/bin/cli.js " + $suffix + "'\''") | join(" ")' package.json
 }
 
-echo "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 $(build_cmds 'bundle') --export-markdown benchmark_bundle.md --export-json benchmark_bundle.json && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 --prepare 'rm -f api-definitions/.redocly.lint-ignore.yaml' $(build_cmds 'lint' ' --generate-ignore-file') --export-markdown benchmark_lint.md --export-json benchmark_lint.json" > test-command.txt
+echo "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 $(build_cmds 'bundle all@latest --config=api-definitions/redocly.yaml') --export-markdown benchmark_bundle.md --export-json benchmark_bundle.json && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 $(build_cmds 'lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file') --export-markdown benchmark_lint.md --export-json benchmark_lint.json && REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 $(build_cmds 'check-config --config=api-definitions/redocly.yaml --lint-config=warn') --export-markdown benchmark_check-config.md --export-json benchmark_check-config.json" > test-command.txt
 
 # Put the command in the test section of the package.json:
 cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json
diff --git a/tests/performance/package.json b/tests/performance/package.json
index 4560f9c52a..c22bb87bdd 100644
--- a/tests/performance/package.json
+++ b/tests/performance/package.json
@@ -7,9 +7,9 @@
   "scripts": {
     "chart": "node chart.js > benchmark_chart.md",
     "make-test": "bash make-test-command.sh",
-    "test:bundle": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
-    "test:lint": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 --prepare 'rm -f api-definitions/.redocly.lint-ignore.yaml' 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
-    "test:check-config": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 2 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
+    "test:bundle": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml' 'node node_modules/cli-next/bin/cli.js bundle all@latest --config=api-definitions/redocly.yaml'",
+    "test:lint": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint all@latest --config=api-definitions/redocly.yaml --generate-ignore-file'",
+    "test:check-config": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 'node node_modules/cli-latest/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn' 'node node_modules/cli-next/bin/cli.js check-config --config=api-definitions/redocly.yaml --lint-config=warn'"
   },
   "_enhancedDependencies": {
     "cli-2.0.0": "npm:@redocly/cli@2.0.0",

From 86d0e6d80a311c71384e1fe14db51da97f108bbb Mon Sep 17 00:00:00 2001
From: Albina Blazhko <albina.blazhko@redocly.com>
Date: Thu, 28 May 2026 18:48:17 +0300
Subject: [PATCH 11/11] chore: remove destruction

---
 tests/performance/chart.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/performance/chart.js b/tests/performance/chart.js
index 4a3a618085..f8daade056 100644
--- a/tests/performance/chart.js
+++ b/tests/performance/chart.js
@@ -19,9 +19,9 @@ const constructBarForChart = (value, min) => {
 const loadResults = (jsonPath) => {
   const json = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
   return new Map(
-    json.results.map(({ command, median, times }) => {
-      const cliVersion = command.replace(/^node node_modules\/([^/]+)\/.*/, (_, v) => v);
-      return [cliVersion, { median, mad: medianAbsoluteDeviation(times, median) }];
+    json.results.map((r) => {
+      const cliVersion = r.command.replace(/^node node_modules\/([^/]+)\/.*/, (_, v) => v);
+      return [cliVersion, { median: r.median, mad: medianAbsoluteDeviation(r.times, r.median) }];
     })
   );
 };