comment-profiling-changes.yaml

name: Profiling Changes

on:
  pull_request: {}

env:
  CARGO_TERM_COLOR: always

jobs:
  profile:
    runs-on: ubuntu-latest
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Install Valgrind
        run: |
          sudo apt update
          sudo apt install -y valgrind

      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          # Cache on Cargo.lock file
          cache-on-failure: true

      - name: Cache iai-callgrind binary
        id: cache-iai
        uses: actions/cache@v4
        with:
          path: ~/.cargo/bin/iai-callgrind-runner
          key: ${{ runner.os }}-iai-callgrind-runner-0.16.1

      - name: Install iai-callgrind
        if: steps.cache-iai.outputs.cache-hit != 'true'
        run: |
          cargo install iai-callgrind-runner@0.16.1

      - name: Checkout master branch
        run: |
          git fetch origin master:master
          git checkout master

      - name: Get master commit SHA
        id: master-sha
        run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT

      - name: Cache benchmark baselines
        id: cache-benchmark-baselines
        uses: actions/cache@v4
        with:
          path: target/iai
          key: ${{ runner.os }}-benchmark-baselines-master-${{ steps.master-sha.outputs.sha }}
          restore-keys: |
            ${{ runner.os }}-benchmark-baselines-master-

      - name: Run baseline benchmarks
        if: steps.cache-benchmark-baselines.outputs.cache-hit != 'true'
        run: |
          # Compile benchmarks
          cargo bench --bench compile_demo_art_iai -- --save-baseline=master

          # Runtime benchmarks
          cargo bench --bench update_executor_iai -- --save-baseline=master
          cargo bench --bench run_once_iai -- --save-baseline=master
          cargo bench --bench run_cached_iai -- --save-baseline=master

      - name: Checkout PR branch
        run: |
          git checkout ${{ github.event.pull_request.head.sha }}

      - name: Run PR benchmarks
        id: benchmark
        run: |
          # Compile benchmarks
          COMPILE_OUTPUT=$(cargo bench --bench compile_demo_art_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g')

          # Runtime benchmarks
          UPDATE_OUTPUT=$(cargo bench --bench update_executor_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g')
          RUN_ONCE_OUTPUT=$(cargo bench --bench run_once_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g')
          RUN_CACHED_OUTPUT=$(cargo bench --bench run_cached_iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g')

          # Store outputs
          echo "COMPILE_OUTPUT<<EOF" >> $GITHUB_OUTPUT
          echo "$COMPILE_OUTPUT" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

          echo "UPDATE_OUTPUT<<EOF" >> $GITHUB_OUTPUT
          echo "$UPDATE_OUTPUT" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

          echo "RUN_ONCE_OUTPUT<<EOF" >> $GITHUB_OUTPUT
          echo "$RUN_ONCE_OUTPUT" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

          echo "RUN_CACHED_OUTPUT<<EOF" >> $GITHUB_OUTPUT
          echo "$RUN_CACHED_OUTPUT" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: Make old comments collapsed by default
        uses: actions/github-script@v7
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const { data: comments } = await github.rest.issues.listComments({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
            });

            const botComments = comments.filter((comment) =>
              comment.user.type === 'Bot' && comment.body.includes('Performance Benchmark Results') && comment.body.includes('<details open>')
            );

            for (const comment of botComments) {
              // Edit the comment to remove the "open" attribute from the <details> tag
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: comment.id,
                body: comment.body.replace('<details open>', '<details>')
              });
            }

      - name: Comment PR
        uses: actions/github-script@v7
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const compileOutput = JSON.parse(`${{ steps.benchmark.outputs.COMPILE_OUTPUT }}`);
            const updateOutput = JSON.parse(`${{ steps.benchmark.outputs.UPDATE_OUTPUT }}`);
            const runOnceOutput = JSON.parse(`${{ steps.benchmark.outputs.RUN_ONCE_OUTPUT }}`);
            const runCachedOutput = JSON.parse(`${{ steps.benchmark.outputs.RUN_CACHED_OUTPUT }}`);

            let significantChanges = false;
            let commentBody = "";

            function formatNumber(num) {
              return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            }

            function formatPercentage(pct) {
              const sign = pct >= 0 ? '+' : '';
              return `${sign}${pct.toFixed(2)}%`;
            }

            function padRight(str, len) {
              return str.padEnd(len);
            }

            function padLeft(str, len) {
              return str.padStart(len);
            }

            function processBenchmarkOutput(benchmarkOutput, sectionTitle, isLast = false) {
              let sectionBody = "";
              let hasResults = false;
              let hasSignificantChanges = false;
              
              for (const benchmark of benchmarkOutput) {
                if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) {
                  const summary = benchmark.callgrind_summary.summaries[0];
                  const irDiff = summary.events.Ir;
                  
                  if (irDiff.diff_pct !== null) {
                    hasResults = true;
                    const changePercentage = formatPercentage(irDiff.diff_pct);
                    const color = irDiff.diff_pct > 0 ? "red" : "lime";
                    
                    sectionBody += `**${benchmark.module_path} ${benchmark.id}:${benchmark.details}**\n`;
                    sectionBody += `Instructions: \`${formatNumber(irDiff.old)}\` (master) → \`${formatNumber(irDiff.new)}\` (HEAD) : `;
                    sectionBody += `$$\\color{${color}}${changePercentage.replace("%", "\\\\%")}$$\n\n`;
                    
                    sectionBody += "<details>\n<summary>Detailed metrics</summary>\n\n```\n";
                    sectionBody += `Baselines:                master|       HEAD\n`;
                    
                    for (const [eventKind, costsDiff] of Object.entries(summary.events)) {
                      if (costsDiff.diff_pct !== null) {
                        const changePercentage = formatPercentage(costsDiff.diff_pct);
                        const line = `${padRight(eventKind, 20)} ${padLeft(formatNumber(costsDiff.old), 11)}|${padLeft(formatNumber(costsDiff.new), 11)} ${padLeft(changePercentage, 15)}`;
                        sectionBody += `${line}\n`;
                      }
                    }
                    
                    sectionBody += "```\n</details>\n\n";
                    
                    if (Math.abs(irDiff.diff_pct) > 5) {
                      significantChanges = true;
                      hasSignificantChanges = true;
                    }
                  }
                }
              }
              
              if (hasResults) {
                // Wrap section in collapsible details, open only if there are significant changes
                const openAttribute = hasSignificantChanges ? " open" : "";
                const ruler = isLast ? "" : "\n\n---";
                return `<details${openAttribute}>\n<summary><h2>${sectionTitle}</h2></summary>\n\n${sectionBody}${ruler}\n</details>`;
              }
              return "";
            }

            // Process each benchmark category
            const sections = [
              { output: compileOutput, title: "🔧 Graph Compilation" },
              { output: updateOutput, title: "🔄 Executor Update" },
              { output: runOnceOutput, title: "🚀 Render: Cold Execution" },
              { output: runCachedOutput, title: "⚡ Render: Cached Execution" }
            ];

            // Generate sections and determine which ones have results
            const generatedSections = sections.map(({ output, title }) => 
              processBenchmarkOutput(output, title, true) // temporarily mark all as last
            ).filter(section => section.length > 0);

            // Re-generate with correct isLast flags
            let sectionIndex = 0;
            const finalSections = sections.map(({ output, title }) => {
              const section = processBenchmarkOutput(output, title, true); // check if it has results
              if (section.length > 0) {
                const isLast = sectionIndex === generatedSections.length - 1;
                sectionIndex++;
                return processBenchmarkOutput(output, title, isLast);
              }
              return "";
            }).filter(section => section.length > 0);

            // Combine all sections
            commentBody = finalSections.join("\n\n");

            if (commentBody.length > 0) {
              const output = `<details open>\n<summary>Performance Benchmark Results</summary>\n\n${commentBody}\n</details>`;

              if (significantChanges) {
                github.rest.issues.createComment({
                  issue_number: context.issue.number,
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  body: output
                });
              } else {
                console.log("No significant performance changes detected. Skipping comment.");
                console.log(output);
              }
            } else {
              console.log("No benchmark results to display.");
            }