manual-benchmark.yml

# This workflow runs the desired benchmarks on demand.
#
# This workflow triggers whenever a comment is created in any PR.
# If the comment has this format: "/benchmark NAME [GHC]"
# The this workflow will run the benchmark with the given NAME, first against 
# the current branch and then comparing the results against the master branch.
# If the optional [GHC] is provided, the benchmark will be run with the given 
# GHC version (one of ghc96, ghc98, ghc910), otherwise the default GHC 
# version (ghc96) will be used.

name: "🚀 Manual Benchmark"

on:
  issue_comment:
    types: [created]

jobs:
  run:
    name: Run
    runs-on: [self-hosted, plutus-benchmark]
    permissions:
      pull-requests: write
  
    if: | 
      startsWith(github.event.comment.body, '/benchmark') && 
      github.event.issue.pull_request 

    steps:
      - name: Checkout
        uses: actions/checkout@v6.0.1
        with:
          # It's possible that new commits get merged into master since the PR 
          # was opened. We need a safe buffer to make sure that our use of merge-head 
          # leter always finds the true parent of the first PR commit.
          fetch-depth: 100

      - name: React With Rocket 
        uses: actions/github-script@v7.1.0
        with:
          script: |
            github.rest.reactions.createForIssueComment({
              owner: context.issue.owner,
              repo: context.issue.repo,
              comment_id: context.payload.comment.id,
              content: "rocket"
            });  

      - name: Extract Benchmark Name
        id: extract-benchmark
        uses: actions/github-script@v7.1.0
        with:
          script: |
            const regex = /^\/benchmark\s*([^\s]*)\s*(ghc96|ghc98|ghc910)?\s*$/;
            const comment = context.payload.comment.body;
            const match = comment.match(regex)
            if (match !== null && match.length == 3 && match[1] !== '') {
              core.setOutput('benchmark', match[1]);
              core.setOutput('ghc', match[2] || 'ghc96');
            } else {
              core.setFailed(`Unable to extract benchmark name and ghc version from comment '${comment}'`);
            }

      - name: Extract Branch Name
        id: extract-branch
        uses: actions/github-script@v7.1.0
        with:
          script: | 
            async function isPullRequest() {
              const result = await github.rest.issues.get({
                owner: context.issue.owner,
                repo: context.issue.repo,
                issue_number: context.issue.number
              });
              return !!result.data.pull_request;
            }

            async function getCommentHeadRef() {
              const query = `
                query pullRequestDetails($repo:String!, $owner:String!, $number:Int!) {
                  repository(name: $repo, owner: $owner) {
                    pullRequest(number: $number) {
                      headRef {
                        name
                      }
                    }
                  }
                }`;

              const result = await github.graphql(query, {
                owner: context.issue.owner,
                repo: context.issue.repo,
                number: context.issue.number
              });

              return result.repository.pullRequest.headRef.name;
            }

            try {
              if (!await isPullRequest()) {
                core.setFailed("Comment is not on a pull request");
              } else {  
                core.setOutput("head_ref", await getCommentHeadRef());
              }
            } catch (error) {
              core.setFailed(`Error: ${error}`);
            } 

      - name: Publish GH Action Link
        uses: actions/github-script@v7.1.0
        with:
          script: | 
            async function getJobUrl() {
              return `https://github.com/${context.issue.owner}/${context.issue.repo}/actions/runs/${context.runId}`;
            }

            await github.rest.issues.createComment({
              owner: context.issue.owner,
              repo: context.issue.repo,
              issue_number: context.issue.number,
              body: `Click [here](${await getJobUrl()}) to check the status of your benchmark.`
            });

      - name: Run Benchmark
        run: |
          nix develop .#${{ steps.extract-benchmark.outputs.ghc }} \
            --no-warn-dirty \
            --accept-flake-config \
            --command taskset -c 7 bash ./scripts/ci-plutus-benchmark.sh 
            # We use taskset to pin the benchmark to a single CPU core to improve repeatability

        env:
          BENCHMARK_NAME: ${{ steps.extract-benchmark.outputs.benchmark }}
          PR_NUMBER: ${{ github.event.issue.number }}
          PR_BRANCH: ${{ steps.extract-branch.outputs.head_ref }}


      - name: Publish Results
        uses: actions/github-script@v7.1.0
        with:
          script: | 
            const fs = require("fs");

            await github.rest.issues.createComment({
              owner: context.issue.owner,
              repo: context.issue.repo,
              issue_number: context.issue.number,
              // bench-compare-result.log is generated by ci-plutus-benchmark.sh
              body: fs.readFileSync("bench-compare-result.log", "utf-8").toString()
            });