benchmark.yml

#-----------------------------------------------------------------------------#
# Launch an EC2 instance, run benchmarks, and publish results
#-----------------------------------------------------------------------------#

name: benchmark

on:
  # Manual trigger, optionally targeting a specific OpenMC ref
  workflow_dispatch:
    inputs:
      openmc_ref:
        description: 'OpenMC commit/branch to benchmark (default: develop)'
        required: false
        default: 'develop'

  # TODO: Trigger from openmc-dev/openmc via repository_dispatch
  # repository_dispatch:
  #   types: [run-perf]

permissions:
  contents: write
  id-token: write

jobs:
  # Spin up an ephemeral EC2 runner
  start-runner:
    runs-on: ubuntu-latest
    outputs:
      instance_id: ${{ steps.launch.outputs.instance_id }}
    steps:
      - uses: actions/checkout@v4

      - name: Configure AWS credentials from OIDC
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::882559813539:role/GithubActionsRole
          aws-region: us-east-2

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.14'

      - name: Install Python packages
        run: |
          pip install --upgrade pip
          pip install boto3

      - name: Get GitHub runner registration token
        id: reg-token
        run: |
          RESPONSE=$(curl -sS -X POST \
            -H "Authorization: Bearer ${{ secrets.GH_RUNNER_TOKEN }}" \
            -H "Accept: application/vnd.github+json" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            "https://api.github.com/repos/${{ github.repository }}/actions/runners/registration-token")
          TOKEN=$(echo "$RESPONSE" | jq -r '.token')
          if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
            echo "Failed to get runner token"
            echo "$RESPONSE" | jq .
            exit 1
          fi
          echo "token=$TOKEN" >> "$GITHUB_OUTPUT"

      - name: Launch EC2 instance
        id: launch
        run: |
          OUTPUT=$(python scripts/launch_ec2_instance.py \
            --runner-token "${{ steps.reg-token.outputs.token }}" \
            --runner-label "perf-ec2" \
            --github-repo "${{ github.repository }}")
          echo "$OUTPUT"
          INSTANCE_ID=$(echo "$OUTPUT" | grep '^instance_id=' | cut -d= -f2)
          echo "instance_id=$INSTANCE_ID" >> "$GITHUB_OUTPUT"

  # Run benchmarks on the EC2 runner
  run-benchmarks:
    needs: start-runner
    runs-on: [self-hosted, perf-ec2]
    timeout-minutes: 120
    env:
      MOAB_INSTALL_DIR: /opt/software/moab
      DAGMC_INSTALL_DIR: /opt/software/dagmc
      LIBMESH_INSTALL_DIR: /opt/software/libmesh
      OPENMC_CROSS_SECTIONS: /opt/data/endfb-vii.1-hdf5/cross_sections.xml
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0   # ASV needs full history for commit metadata

      - name: Clone results repo
        uses: actions/checkout@v4
        with:
          repository: openmc-dev/openmc-performance-bench-results
          path: results-repo
          token: ${{ secrets.RESULTS_REPO_TOKEN }}

      - name: Determine OpenMC ref
        id: ref
        run: |
          REF="${{ github.event.client_payload.sha
               || github.event.inputs.openmc_ref
               || 'develop' }}"
          echo "openmc_ref=$REF" >> "$GITHUB_OUTPUT"

      - name: Add tools and package directories to PATH
        run: |
          echo "/opt/tools-venv/bin" >> "$GITHUB_PATH"
          echo "$MOAB_INSTALL_DIR/bin" >> "$GITHUB_PATH"
          echo "$DAGMC_INSTALL_DIR/bin" >> "$GITHUB_PATH"
          echo "$LIBMESH_INSTALL_DIR/bin" >> "$GITHUB_PATH"

      - name: Register machine with ASV
        id: machine
        run: |
          # Request an IMDSv2 session token from the EC2 metadata service
          TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" \
            -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" -s)

          # Query the EC2 instance type using the metadata token
          INSTANCE_TYPE=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" \
            -s http://169.254.169.254/latest/meta-data/instance-type)

          # Use a fixed name that describes the hardware, not the ephemeral hostname
          MACHINE_NAME="ec2-${INSTANCE_TYPE//./-}"
          echo "machine_name=$MACHINE_NAME" >> "$GITHUB_OUTPUT"
          asv machine --yes --machine "$MACHINE_NAME"

      - name: Point ASV results at results repo
        run: |
          mkdir -p results-repo/results
          mkdir -p .asv
          rm -rf .asv/results
          ln -s "${{ github.workspace }}/results-repo/results" "${{ github.workspace }}/.asv/results"

      - name: Run benchmarks
        id: asv-run
        continue-on-error: true
        run: |
          PYTHONUNBUFFERED=1 asv run ${{ steps.ref.outputs.openmc_ref }}^! --show-stderr -vv

      - name: Inject metadata into results
        if: steps.asv-run.outcome == 'success' || steps.asv-run.outcome == 'failure'
        run: |
          python scripts/collect_metadata.py \
            --results-dir "results-repo/results/${{ steps.machine.outputs.machine_name }}" \
            --openmc-build-dir .asv/env/*/project/build \
            --dagmc-dir $DAGMC_INSTALL_DIR \
            --moab-dir $MOAB_INSTALL_DIR \
            --libmesh-dir $LIBMESH_INSTALL_DIR

      - name: Publish HTML report
        if: steps.asv-run.outcome == 'success' || steps.asv-run.outcome == 'failure'
        run: |
          if [ -d results-repo/results ]; then
            asv publish --html-dir results-repo/html
          else
            echo "No ASV results found; skipping publish"
          fi

      - name: Commit and push results
        if: steps.asv-run.outcome == 'success' || steps.asv-run.outcome == 'failure'
        run: |
          cd results-repo
          git config user.name  "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add results/
          git diff --cached --quiet || \
            git commit -m "Add benchmark results for ${{ steps.ref.outputs.openmc_ref }}"
          git push

      - name: Deploy HTML to GitHub Pages
        if: (steps.asv-run.outcome == 'success' || steps.asv-run.outcome == 'failure') && hashFiles('results-repo/html/index.html') != ''
        uses: peaceiris/actions-gh-pages@v4
        with:
          personal_token: ${{ secrets.RESULTS_REPO_TOKEN }}
          external_repository: openmc-dev/openmc-performance-bench-results
          publish_branch: gh-pages
          publish_dir: results-repo/html
          keep_files: true

      - name: Fail if any benchmarks failed
        if: steps.asv-run.outcome == 'failure'
        run: exit 1

  # Always terminate the EC2 instance, even if benchmarks fail
  stop-runner:
    if: always()
    needs: [start-runner, run-benchmarks]
    runs-on: ubuntu-latest
    steps:
      - name: Configure AWS credentials from OIDC
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::882559813539:role/GithubActionsRole
          aws-region: us-east-2

      - name: Terminate EC2 instance
        if: needs.start-runner.outputs.instance_id != ''
        run: |
          echo "Terminating ${{ needs.start-runner.outputs.instance_id }}"
          aws ec2 terminate-instances \
            --instance-ids "${{ needs.start-runner.outputs.instance_id }}"