WIP: TEST BENCHMARK!

Author: tatomyr

.github/workflows/performance.yaml

@@ -14,45 +14,45 @@ env:
   REDOCLY_TELEMETRY: off
 
 jobs:
-  latest-vs-next:
-    # Do not run on the release branch (changeset-release/main):
-    if: github.head_ref != 'changeset-release/main'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 24
-          cache: npm
-      - name: Install Dependencies
-        run: npm ci
-      - name: Install External
-        run: npm i -g hyperfine
-      - name: Prepare
-        run: |
-          npm run compile
-          npm run pack:prepare
-          cd tests/performance/
-          npm i
-          npm run make-test
+  # latest-vs-next:
+  #   # Do not run on the release branch (changeset-release/main):
+  #   if: github.head_ref != 'changeset-release/main'
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-node@v3
+  #       with:
+  #         node-version: 24
+  #         cache: npm
+  #     - name: Install Dependencies
+  #       run: npm ci
+  #     - name: Install External
+  #       run: npm i -g hyperfine
+  #     - name: Prepare
+  #       run: |
+  #         npm run compile
+  #         npm run pack:prepare
+  #         cd tests/performance/
+  #         npm i
+  #         npm run make-test
 
-      - name: Run Benchmark
-        run: |
-          cd tests/performance/
-          npm run compare_latest_vs_next
-          cat benchmark_check.md
-          npm run chart # Creates benchmark_chart.md with the performance bar chart.
+  #     - name: Run Benchmark
+  #       run: |
+  #         cd tests/performance/
+  #         npm run compare_latest_vs_next
+  #         cat benchmark_check.md
+  #         npm run chart # Creates benchmark_chart.md with the performance bar chart.
 
-      - name: Comment PR
-        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
-        uses: thollander/actions-comment-pull-request@v2
-        with:
-          filePath: tests/performance/benchmark_chart.md
-          comment_tag: latest-vs-next-comparison
+  #     - name: Comment PR
+  #       if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+  #       uses: thollander/actions-comment-pull-request@v2
+  #       with:
+  #         filePath: tests/performance/benchmark_chart.md
+  #         comment_tag: latest-vs-next-comparison
 
   historical-versions:
     # Run only on release branch (changeset-release/main):
-    if: github.head_ref == 'changeset-release/main'
+    # if: github.head_ref == 'changeset-release/main'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -64,6 +64,13 @@ jobs:
         run: npm ci
       - name: Install External
         run: npm i -g hyperfine
+
+      - name: Clean up versions
+        # Do not run on the release branch (changeset-release/main):
+        if: github.head_ref != 'changeset-release/main'
+        run: |
+          echo "Cleaning up versions..."
+
       - name: Prepare
         run: |
           npm run compile
@@ -75,7 +82,7 @@ jobs:
       - name: Run Benchmark
         run: |
           cd tests/performance/
-          npm run compare_historical_versions # This command is generated and injected into package.json in the previous step.
+          npm run test # This command is generated and injected into package.json in the previous step.
           cat benchmark_check.md
           npm run chart # Creates benchmark_chart.md with the performance bar chart.
 

tests/performance/make-test-command.sh

@@ -10,4 +10,4 @@ cd api-definitions && npm install && cd ..
 echo REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js lint core@public --config=api-definitions/redocly.yaml --generate-ignore-file'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
 
 # Put the command in the test section of the package.json:
-cat package.json | jq ".scripts.compare_historical_versions = \"$(cat test-command.txt)\"" > package.json
+cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json

tests/performance/package.json

@@ -6,10 +6,15 @@
   "scripts": {
     "chart": "node chart.js > benchmark_chart.md",
     "make-test": "bash make-test-command.sh",
-    "compare_latest_vs_next": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 'node node_modules/cli-latest/bin/cli.js lint core@public --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint core@public --config=api-definitions/redocly.yaml --generate-ignore-file' --export-markdown benchmark_check.md --export-json benchmark_check.json",
-    "compare_historical_versions": "Placeholder..."
+    "FIXME: remove compare_latest_vs_next": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 'node node_modules/cli-latest/bin/cli.js lint core@public --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint core@public --config=api-definitions/redocly.yaml --generate-ignore-file' --export-markdown benchmark_check.md --export-json benchmark_check.json",
+    "test": "Placeholder..."
   },
   "dependencies": {
+    "cli-2.0.0": "npm:@redocly/cli@2.0.0",
+    "cli-2.3.1": "npm:@redocly/cli@2.3.1",
+    "cli-2.8.0": "npm:@redocly/cli@2.8.0",
+    "cli-2.9.0": "npm:@redocly/cli@2.9.0",
+    "cli-2.11.1": "npm:@redocly/cli@2.11.1",
     "cli-latest": "npm:@redocly/cli@latest",
     "cli-next": "file:../../redocly-cli.tgz"
   }