WIP: TEST BENCHMARK!

Author: tatomyr

.github/workflows/performance.yaml

@@ -28,26 +28,27 @@ jobs:
         run: npm i -g hyperfine @redocly/cli@latest
       - name: Prepare
         run: |
-          jq '.bin = {"redocly-next":"bin/cli.js"} | .name = "@redocly/cli-next"' packages/cli/package.json > __tmp__.json
-          mv __tmp__.json packages/cli/package.json
-          cat packages/cli/package.json
+          npm run compile
           npm run pack:prepare
-          npm i -g redocly-cli.tgz
-      - run: redocly-next --version
-      - run: redocly --version
+          cd tests/performance/
+          npm i
+          npm run make-test
+
       - name: Run Benchmark
-        run: hyperfine -i --warmup 3 'redocly lint resources/rebilly.yaml' 'redocly-next lint resources/rebilly.yaml' --export-markdown benchmark_check.md --export-json benchmark_check.json
+        run: |
+          cd tests/performance/
+          npm run compare-latest-vs-next
 
       - name: Comment PR
         if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
         uses: thollander/actions-comment-pull-request@v2
         with:
-          filePath: benchmark_check.md
+          filePath: tests/performance/benchmark_chart.md
           comment_tag: latest-vs-next-comparison
 
   historical-versions:
     # Run only on release branch (changeset-release/main):
-    if: github.head_ref == 'changeset-release/main'
+    # if: github.head_ref == 'changeset-release/main' # FIXME: Uncomment this!
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -66,10 +67,11 @@ jobs:
           cd tests/performance/
           npm i
           npm run make-test
+
       - name: Run Benchmark
         run: |
           cd tests/performance/
-          npm test # This command is generated and injected into package.json in the previous step.
+          npm run compare-historical-versions # This command is generated and injected into package.json in the previous step.
           cat benchmark_check.md
           npm run chart # Creates benchmark_chart.md with the performance bar chart.
 

tests/performance/make-test-command.sh

@@ -7,7 +7,7 @@ git clone https://github.com/Rebilly/api-definitions.git
 cd api-definitions && npm install && cd ..
 
 # Store the command into a text file:
-echo REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 3 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js lint --config=api-definitions/redocly.yaml --generate-ignore-file'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
+echo REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js lint core@public --config=api-definitions/redocly.yaml --generate-ignore-file'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
 
 # Put the command in the test section of the package.json:
-cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json
+cat package.json | jq ".scripts.compare-historical-versions = \"$(cat test-command.txt)\"" > package.json

tests/performance/package.json

@@ -5,14 +5,12 @@
   "private": true,
   "scripts": {
     "chart": "node chart.js > benchmark_chart.md",
-    "make-test": "bash make-test-command.sh"
+    "make-test": "bash make-test-command.sh",
+    "compare-latest-vs-next": "REDOCLY_SUPPRESS_UPDATE_NOTICE=true hyperfine --warmup 1 'node node_modules/cli-latest/bin/cli.js lint core@public --config=api-definitions/redocly.yaml --generate-ignore-file' 'node node_modules/cli-next/bin/cli.js lint core@public --config=api-definitions/redocly.yaml --generate-ignore-file' --export-markdown benchmark_check.md --export-json benchmark_check.json",
+    "compare-historical-versions": "Placeholder..."
   },
   "dependencies": {
-    "cli-2.0.0": "npm:@redocly/cli@2.0.0",
-    "cli-2.3.1": "npm:@redocly/cli@2.3.1",
-    "cli-2.8.0": "npm:@redocly/cli@2.8.0",
-    "cli-2.9.0": "npm:@redocly/cli@2.9.0",
-    "cli-2.11.1": "npm:@redocly/cli@2.11.1",
+    "cli-latest": "npm:@redocly/cli@latest",
     "cli-next": "file:../../redocly-cli.tgz"
   }
 }