benchmarks.yml

# Copyright 2010 New Relic, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
name: Benchmarks

on:
  pull_request:

permissions:
  contents: read

concurrency:
  group: ${{ github.ref || github.run_id }}-${{ github.workflow }}
  cancel-in-progress: true

jobs:
  # ===========================
  # Aggregate Benchmark Results
  # ===========================
  benchmarks:
    runs-on: ubuntu-24.04
    if: always() # Always run, even on cancellation or failure
    needs:
      - benchmark

    steps:
      - name: Status
        run: |
          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
            echo "Workflow cancelled."
            exit 1
          elif [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
            echo "One or more jobs failed."
            exit 1
          else
            echo "All jobs completed successfully."
            exit 0
          fi

  # ==================================
  # Benchmarks Run Directly On Runners
  # ==================================
  benchmark:
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]

    env:
      ASV_FACTOR: "1.1"
      BASE_SHA: ${{ github.event.pull_request.base.sha }}
      HEAD_SHA: ${{ github.event.pull_request.head.sha }}

    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # 7.0.0
        with:
          fetch-depth: 0

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # 6.2.0
        with:
          python-version: "${{ matrix.python }}"

      - name: Fetch git tags
        run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git fetch --tags origin

      - name: Install Dependencies
        run: |
          pip install --upgrade pip
          pip install asv virtualenv

      - name: Configure Machine Information
        run: |
          asv machine --yes

      - name: Run Benchmark
        run: |
          echo "Running continuous benchmarking between base commit ${BASE_SHA} and head commit ${HEAD_SHA}"
          asv continuous \
            --show-stderr \
            --split \
            --factor "${ASV_FACTOR}" \
            --python=${{ matrix.python }} \
            "${BASE_SHA}" "${HEAD_SHA}"