ci-benchmark.yml

name: Benchmark Bot

on:
  issue_comment:
    types: [created]

permissions:
  issues: write
  pull-requests: write

jobs:
  reply-to-benchmark:
    if: |
      startsWith(github.event.comment.body, '@benchmark') &&
      github.event.issue.pull_request &&
      (
        github.event.comment.user.login == 'tchaton' ||
        github.event.comment.user.login == 'lantiga' ||
        github.event.comment.user.login == 'justusschock' ||
        github.event.comment.user.login == 'Borda' ||
        github.event.comment.user.login == 'bhimrazy' ||
        github.event.comment.user.login == 'deependujha'
      )
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - name: Extract make args
        id: extract
        uses: actions/github-script@v9
        with:
          script: |
            const fullComment = context.payload.comment.body;
            const parts = fullComment.trim().split(/\s+/);
            const makeargs = parts.slice(1).join(' '); // remove "@benchmark" and keep rest
            core.setOutput("make_args", makeargs);

      - name: Reply to PR comment and save ID
        id: comment
        uses: actions/github-script@v9
        with:
          script: |
            const fs = require('fs');
            const path = '.github/benchmark/greet.md';
            const replyTemplate = fs.readFileSync(path, 'utf8');
            const username = context.payload.comment.user.login;
            const prNumber = context.payload.issue.number;

            const pr = await github.rest.pulls.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: context.payload.issue.number,
            });
            const branch = pr.data.head.ref;

            // Replace placeholders
            const reply = replyTemplate
              .replace(/{{username}}/g, username)
              .replace(/{{pr_number}}/g, prNumber);

            const response = await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: prNumber,
              body: reply
            });

            core.setOutput("comment_id", response.data.id);
            core.setOutput("prNumber", prNumber);
            core.setOutput("branch", branch);
            core.setOutput("username", username);

      - name: run python
        run: |
          pip install -U lightning-sdk
          echo '"pr number: ${{ steps.comment.outputs.prNumber }}"; "branch: ${{ steps.comment.outputs.branch }}"'
          echo "make-args: ${{steps.extract.outputs.make_args}}"
          python .github/benchmark/benchmark.py \
            --pr "${{ steps.comment.outputs.prNumber }}" \
            --branch "${{ steps.comment.outputs.branch }}" \
            --make-args "${{ steps.extract.outputs.make_args }}" \
            2> error.txt || true
        env: # the following values are parsed from the repository secrets
          LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
          LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }}

      - name: Update the same comment
        uses: actions/github-script@v9
        env:
          COMMENT_ID: ${{ steps.comment.outputs.comment_id }}
          USERNAME: ${{ steps.comment.outputs.username }}
        with:
          script: |
            const fs = require('fs');
            const comment_id = Number(process.env.COMMENT_ID);

            let reply = '';
            if (fs.existsSync('result.md')) {
              reply = fs.readFileSync('result.md', 'utf8');
            } else if (fs.existsSync('error.txt') && fs.readFileSync('error.txt', 'utf8').trim().length > 0) {
              const err = fs.readFileSync('error.txt', 'utf8');
              reply = `❌ **Benchmark failed**\n\n\`\`\`\n${err}\n\`\`\``;
            } else {
              reply = '❌ Benchmark completed, but can\'t find `result.md` and `error.txt` file. Something went wrong.';
            }

            const updated_body = `Hi @${process.env.USERNAME}!\n\n${reply}\n\ncc: @tchaton @deependujha @bhimrazy`;

            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: comment_id,
              body: updated_body
            });