benchmarks.yml

name: Benchmarks

on:
  workflow_dispatch: {}

concurrency:
  group: benchmarks-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  setup:
    name: "Initialize Benchmark Report"
    if: github.event.action != 'closed' && github.event_name == 'pull_request' && github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Post initial pending comment
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const marker = '<!-- fusion-gateway-benchmark-report -->';
            const runId = String(context.runId);
            const dataPrefix = '<!-- benchmark-data:';

            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c => c.body.includes(marker));

            // --- Generate all-pending markdown ---
            const tests = [
              { key: 'no-recursion', title: 'Simple Composite Query' },
              { key: 'deep-recursion', title: 'Deep Recursion Query' },
              { key: 'variable-batch', title: 'Variable Batching Throughput' },
            ];
            const modes = [
              { key: 'constant', label: 'Constant', desc: '(50 VUs)' },
              { key: 'ramping', label: 'Ramping', desc: '(0-500-0 VUs)' },
            ];
            const runners = [
              { group: 'benchmarking-1', label: '1' },
            ];

            let md = '### Fusion Gateway Performance Results\n\n';
            md += `> **Progress: 0/6 benchmarks completed** — this report updates as each job finishes.\n\n`;

            for (const test of tests) {
              md += `#### ${test.title}\n\n`;
              md += '| | Req/s | Err% |\n|:--|--:|--:|\n';
              for (const mode of modes) {
                for (const runner of runners) {
                  md += `| **${mode.label} ${runner.label}** ${mode.desc} | *pending* | *pending* |\n`;
                }
              }
              md += '\n<details>\n<summary>Response Times</summary>\n\n';
              md += '| | Min | Med | Avg | P90 | P95 | Max |\n|:--|--:|--:|--:|--:|--:|--:|\n';
              for (const mode of modes) {
                for (const runner of runners) {
                  md += `| **${mode.label} ${runner.label}** | *pending* | *pending* | *pending* | *pending* | *pending* | *pending* |\n`;
                }
              }
              md += '\n</details>\n\n---\n\n';
            }
            md += '*Runner 1 = benchmarking-1*';

            const encodedData = Buffer.from(JSON.stringify({})).toString('base64');
            const timestamp = new Date().toUTCString();
            const commitSha = context.sha.substring(0, 7);
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;

            const commentBody = [
              marker,
              `<!-- run:${runId} -->`,
              `${dataPrefix}${encodedData} -->`,
              `<!-- completed:0 -->`,
              md,
              '---',
              `*Run [${runId}](${runUrl}) • Commit ${commitSha} • ${timestamp}*`,
            ].join('\n');

            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body: commentBody,
              });
              console.log('Reset existing comment to all-pending');
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: commentBody,
              });
              console.log('Created initial all-pending comment');
            }

  benchmark:
    name: "${{ matrix.test-name }} ${{ matrix.mode-label }} (${{ matrix.runner-label }})"
    needs: [setup]
    if: "!cancelled() && github.event.action != 'closed' && (github.event_name == 'push' || github.event.pull_request.draft == false)"
    runs-on:
      group: ${{ matrix.runner-group }}
    permissions:
      contents: write
      pull-requests: write
    strategy:
      fail-fast: false
      matrix:
        test: [no-recursion, deep-recursion, variable-batch]
        mode: [constant, ramping]
        runner-group: [benchmarking-1]
        include:
          - runner-group: benchmarking-1
            runner-label: "Runner 1"
          - test: no-recursion
            test-name: "Simple Composite Query"
          - test: deep-recursion
            test-name: "Deep Recursion"
          - test: variable-batch
            test-name: "Variable Batch"
          - mode: constant
            mode-label: "Constant"
          - mode: ramping
            mode-label: "Ramping"

    steps:
      - name: Checkout current repository
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          fetch-depth: 0
          show-progress: false

      - name: Install .NET
        run: |
          if ! command -v dotnet &> /dev/null; then
            echo "Installing .NET..."
            curl -sSL https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh
            chmod +x dotnet-install.sh
            ./dotnet-install.sh --version 11.0.100-preview.5.26302.115 --install-dir $HOME/.dotnet
            rm dotnet-install.sh
            echo "$HOME/.dotnet" >> $GITHUB_PATH
            export PATH="$HOME/.dotnet:$PATH"
          fi
          dotnet --version

      - name: Install k6
        run: |
          if ! command -v k6 &> /dev/null; then
            echo "Installing k6..."
            sudo gpg -k
            sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
            echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
            sudo apt-get update
            sudo apt-get install k6 -y
          fi
          k6 version

      - name: Install jq for result parsing
        run: |
          if ! command -v jq &> /dev/null; then
            sudo apt-get install jq -y
          fi

      - name: Make scripts executable
        working-directory: src/HotChocolate/Fusion/benchmarks/k6
        run: chmod +x *.sh

      - name: Run benchmark
        working-directory: src/HotChocolate/Fusion/benchmarks/k6
        run: |
          ./run-single-benchmark.sh \
            "${{ matrix.test }}" \
            "${{ matrix.mode }}" \
            "result.json" \
            "${{ matrix.runner-group }}" \
            "${{ matrix.runner-label }}"

      - name: Upload benchmark result
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: benchmark-${{ matrix.test }}-${{ matrix.mode }}-${{ matrix.runner-group }}
          path: src/HotChocolate/Fusion/benchmarks/k6/result.json
          retention-days: 30

      # Lightweight progressive PR update — reads only the local result.json
      # and accumulates data in a hidden JSON block inside the PR comment.
      # No artifact downloads, no external scripts — just a few API calls.
      - name: Update PR comment (progressive)
        if: github.event_name == 'pull_request'
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const fs = require('fs');

            // Read this job's result
            let result;
            try {
              result = JSON.parse(fs.readFileSync(
                'src/HotChocolate/Fusion/benchmarks/k6/result.json', 'utf8'));
            } catch (e) {
              console.log('No result.json found, skipping comment update');
              return;
            }

            const marker = '<!-- fusion-gateway-benchmark-report -->';
            const dataPrefix = '<!-- benchmark-data:';
            const runId = String(context.runId);

            // Fetch existing comment (created by the setup job)
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c => c.body.includes(marker));
            if (!existing) {
              console.log('No benchmark comment found, skipping update');
              return;
            }

            // Only accumulate data from the same run
            let allResults = {};
            const runMatch = existing.body.match(/<!-- run:(\d+) -->/);
            const commentRunId = runMatch ? runMatch[1] : null;
            if (commentRunId === runId) {
              const m = existing.body.match(/<!-- benchmark-data:(.*?) -->/);
              if (m) {
                try { allResults = JSON.parse(Buffer.from(m[1], 'base64').toString()); }
                catch (e) { console.log('Failed to parse existing data, starting fresh'); }
              }
            }

            // Merge our result
            const key = `${result.test}|${result.mode}|${result.runner_group}`;
            allResults[key] = result;
            const completed = Object.keys(allResults).length;

            // --- Generate markdown report inline ---
            const fmt = (n) => Number(n).toFixed(2);
            const tests = [
              { key: 'no-recursion', title: 'Simple Composite Query' },
              { key: 'deep-recursion', title: 'Deep Recursion Query' },
              { key: 'variable-batch', title: 'Variable Batching Throughput' },
            ];
            const modes = [
              { key: 'constant', label: 'Constant', desc: '(50 VUs)' },
              { key: 'ramping', label: 'Ramping', desc: '(0-500-0 VUs)' },
            ];
            const runners = [
              { group: 'benchmarking-1', label: '1' },
            ];

            let md = '### Fusion Gateway Performance Results\n\n';
            if (completed < 6) {
              md += `> **Progress: ${completed}/6 benchmarks completed** — this report updates as each job finishes.\n\n`;
            }

            for (const test of tests) {
              md += `#### ${test.title}\n\n`;
              md += '| | Req/s | Err% |\n|:--|--:|--:|\n';
              for (const mode of modes) {
                for (const runner of runners) {
                  const k = `${test.key}|${mode.key}|${runner.group}`;
                  const r = allResults[k];
                  const lbl = `**${mode.label} ${runner.label}**`;
                  if (r) {
                    md += `| ${lbl} ${mode.desc} | ${fmt(r.throughput.requests_per_second)} | ${fmt(r.reliability.error_rate)}% |\n`;
                  } else {
                    md += `| ${lbl} ${mode.desc} | *pending* | *pending* |\n`;
                  }
                }
              }
              md += '\n<details>\n<summary>Response Times</summary>\n\n';
              md += '| | Min | Med | Avg | P90 | P95 | Max |\n|:--|--:|--:|--:|--:|--:|--:|\n';
              for (const mode of modes) {
                for (const runner of runners) {
                  const k = `${test.key}|${mode.key}|${runner.group}`;
                  const r = allResults[k];
                  const lbl = `**${mode.label} ${runner.label}**`;
                  if (r) {
                    const t = r.response_time;
                    md += `| ${lbl} | ${fmt(t.min)}ms | ${fmt(t.p50)}ms | ${fmt(t.avg)}ms | ${fmt(t.p90)}ms | ${fmt(t.p95)}ms | ${fmt(t.max)}ms |\n`;
                  } else {
                    md += `| ${lbl} | *pending* | *pending* | *pending* | *pending* | *pending* | *pending* |\n`;
                  }
                }
              }
              md += '\n</details>\n\n---\n\n';
            }
            md += '*Runner 1 = benchmarking-1*';

            // Build comment body with hidden data block
            const encodedData = Buffer.from(JSON.stringify(allResults)).toString('base64');
            const timestamp = new Date().toUTCString();
            const commitSha = context.sha.substring(0, 7);
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

            const commentBody = [
              marker,
              `<!-- run:${runId} -->`,
              `${dataPrefix}${encodedData} -->`,
              `<!-- completed:${completed} -->`,
              md,
              '---',
              `*Run [${runId}](${runUrl}) • Commit ${commitSha} • ${timestamp}*`,
            ].join('\n');

            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: existing.id,
              body: commentBody,
            });
            console.log(`Updated comment (${completed}/6 completed)`);

  report:
    name: "Final Performance Report"
    needs: [setup, benchmark]
    if: "!cancelled() && github.event.action != 'closed' && github.event_name == 'pull_request'"
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - name: Download all benchmark results
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        continue-on-error: true
        with:
          pattern: benchmark-*
          path: benchmark-results

      # Merges artifact data with accumulated comment data so we never
      # overwrite progressive results with a less complete artifact set
      # (e.g., when cancel-in-progress killed some jobs mid-run).
      - name: Update PR comment with final report
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const fs = require('fs');
            const path = require('path');

            const marker = '<!-- fusion-gateway-benchmark-report -->';
            const dataPrefix = '<!-- benchmark-data:';
            const runId = String(context.runId);

            // --- Collect results from artifacts ---
            let artifactResults = {};
            const baseDir = 'benchmark-results';
            if (fs.existsSync(baseDir)) {
              for (const dir of fs.readdirSync(baseDir)) {
                const filePath = path.join(baseDir, dir, 'result.json');
                if (fs.existsSync(filePath)) {
                  try {
                    const r = JSON.parse(fs.readFileSync(filePath, 'utf8'));
                    const key = `${r.test}|${r.mode}|${r.runner_group}`;
                    artifactResults[key] = r;
                  } catch (e) { console.log(`Failed to parse ${filePath}:`, e.message); }
                }
              }
            }
            console.log(`Found ${Object.keys(artifactResults).length} result(s) from artifacts`);

            // --- Read accumulated data from existing PR comment (same run only) ---
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c => c.body.includes(marker));

            let commentResults = {};
            if (existing) {
              const runMatch = existing.body.match(/<!-- run:(\d+) -->/);
              const commentRunId = runMatch ? runMatch[1] : null;

              if (commentRunId === runId) {
                const m = existing.body.match(/<!-- benchmark-data:(.*?) -->/);
                if (m) {
                  try { commentResults = JSON.parse(Buffer.from(m[1], 'base64').toString()); }
                  catch (e) { console.log('Failed to parse comment data'); }
                }
                console.log(`Same run, loaded ${Object.keys(commentResults).length} result(s) from comment`);
              } else {
                console.log(`Different run (comment=${commentRunId}, current=${runId}), ignoring comment data`);
              }
            }

            // --- Merge: artifacts win over comment data, comment fills gaps ---
            const allResults = { ...commentResults, ...artifactResults };
            const completed = Object.keys(allResults).length;
            console.log(`Merged total: ${completed} result(s)`);

            // Only update if we have at least as much data as what's posted for the same run
            if (existing) {
              const runMatch2 = existing.body.match(/<!-- run:(\d+) -->/);
              const commentRunId2 = runMatch2 ? runMatch2[1] : null;
              if (commentRunId2 === runId) {
                const cm = existing.body.match(/<!-- completed:(\d+) -->/);
                const existingCompleted = cm ? parseInt(cm[1], 10) : 0;
                if (completed < existingCompleted) {
                  console.log(`Skipped final report (${completed} < ${existingCompleted})`);
                  return;
                }
              }
            }

            // --- Generate markdown (same logic as progressive updates) ---
            const fmt = (n) => Number(n).toFixed(2);
            const tests = [
              { key: 'no-recursion', title: 'Simple Composite Query' },
              { key: 'deep-recursion', title: 'Deep Recursion Query' },
              { key: 'variable-batch', title: 'Variable Batching Throughput' },
            ];
            const modes = [
              { key: 'constant', label: 'Constant', desc: '(50 VUs)' },
              { key: 'ramping', label: 'Ramping', desc: '(0-500-0 VUs)' },
            ];
            const runners = [
              { group: 'benchmarking-1', label: '1' },
            ];

            let md = '### Fusion Gateway Performance Results\n\n';
            if (completed < 6) {
              md += `> **Progress: ${completed}/6 benchmarks completed** — this report updates as each job finishes.\n\n`;
            }

            for (const test of tests) {
              md += `#### ${test.title}\n\n`;
              md += '| | Req/s | Err% |\n|:--|--:|--:|\n';
              for (const mode of modes) {
                for (const runner of runners) {
                  const k = `${test.key}|${mode.key}|${runner.group}`;
                  const r = allResults[k];
                  const lbl = `**${mode.label} ${runner.label}**`;
                  if (r) {
                    md += `| ${lbl} ${mode.desc} | ${fmt(r.throughput.requests_per_second)} | ${fmt(r.reliability.error_rate)}% |\n`;
                  } else {
                    md += `| ${lbl} ${mode.desc} | *pending* | *pending* |\n`;
                  }
                }
              }
              md += '\n<details>\n<summary>Response Times</summary>\n\n';
              md += '| | Min | Med | Avg | P90 | P95 | Max |\n|:--|--:|--:|--:|--:|--:|--:|\n';
              for (const mode of modes) {
                for (const runner of runners) {
                  const k = `${test.key}|${mode.key}|${runner.group}`;
                  const r = allResults[k];
                  const lbl = `**${mode.label} ${runner.label}**`;
                  if (r) {
                    const t = r.response_time;
                    md += `| ${lbl} | ${fmt(t.min)}ms | ${fmt(t.p50)}ms | ${fmt(t.avg)}ms | ${fmt(t.p90)}ms | ${fmt(t.p95)}ms | ${fmt(t.max)}ms |\n`;
                  } else {
                    md += `| ${lbl} | *pending* | *pending* | *pending* | *pending* | *pending* | *pending* |\n`;
                  }
                }
              }
              md += '\n</details>\n\n---\n\n';
            }
            md += '*Runner 1 = benchmarking-1*';

            // Build comment body
            const encodedData = Buffer.from(JSON.stringify(allResults)).toString('base64');
            const timestamp = new Date().toUTCString();
            const commitSha = context.sha.substring(0, 7);
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

            const commentBody = [
              marker,
              `<!-- run:${runId} -->`,
              `${dataPrefix}${encodedData} -->`,
              `<!-- completed:${completed} -->`,
              md,
              '---',
              `*Run [${runId}](${runUrl}) • Commit ${commitSha} • ${timestamp}*`,
            ].join('\n');

            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body: commentBody,
              });
              console.log(`Updated final report (${completed}/6 completed)`);
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: commentBody,
              });
              console.log(`Created final report (${completed}/6 completed)`);
            }

  store-results:
    name: "Store Baseline Results"
    needs: benchmark
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - name: Download all benchmark results
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: benchmark-*
          path: benchmark-results

      - name: Checkout performance data repository
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          repository: ChilliCream/graphql-platform-performance-data
          token: ${{ secrets.PERFORMANCE_DATA_TOKEN }}
          path: performance-data-repo
          fetch-depth: 1
          show-progress: false

      - name: Store results to external repository
        run: |
          mkdir -p performance-data-repo/fusion-gateway

          for dir in benchmark-results/benchmark-*/; do
            if [ -f "$dir/result.json" ]; then
              name=$(basename "$dir")
              key=${name#benchmark-}
              cp "$dir/result.json" "performance-data-repo/fusion-gateway/${key}.json"
            fi
          done

          cd performance-data-repo
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          git add fusion-gateway/

          if ! git diff --staged --quiet; then
            git commit -m "Update Fusion Gateway benchmark data from ${{ github.sha }}"
            git push
          else
            echo "No changes to performance data"
          fi