benchmarks.yml

name: Run snarkOS Benchmarks

on:
  push:
    branches:
      - 'staging'
  workflow_dispatch:

jobs:
  # Run benchmarks and stores the output to a file
  benchmark:
    name: Benchmark
    runs-on:
      labels: ubuntu-latest-m
    permissions:
      contents: write
    steps:
      - name: Checkout
        uses: actions/checkout@v5

      - name: Install Rust
        uses: dtolnay/rust-toolchain@1.88 # Update this when the MSRV changes.

      - name: Install required Debian packages
        run: |
          sudo apt-get update
          sudo apt-get install -y lld python3-pip

      - name: Set up Python virtual environment
        run: |
          pip install --upgrade pip
          python -m venv pyenv
          source pyenv/bin/activate
          pip install aiohttp

      - name: Set up Rust build caching
        uses: Swatinem/rust-cache@v2

      - name: Set up Google Cloud Integration
        env:
          GCLOUD_SERVICE_KEY: ${{ secrets.GCLOUD_SERVICE_KEY }}
        run: |
          curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
          echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
          sudo apt-get update
          sudo apt-get install google-cloud-cli -y
          echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-key.json
          gcloud auth activate-service-account --key-file=${HOME}/gcloud-key.json

      - name: Fetch Ledger Data
        run: |
          gcloud config set project protocol-development-sandbox
          # Created with `.ci/generate_ledger.sh 40 250 1`
          # NOTE: This needs to be updated when the consensus version changes.
          gcloud storage cp gs://ci_testdata/sync-ledger-val40-250-9ec2291c57.zip ledger.zip
          unzip ledger.zip
 
      - name: Install snarkOS (test_network)
        run: |
          # use `test_network` flag without snarkVM `dev_println`
          cargo install --path=. --locked --features=test_consensus_heights,test_targets

      # Download previous benchmark result from cache (if exists)
      - name: Download previous benchmark data
        uses: actions/cache@v4
        with:
          path: ./cache
          key: ${{ runner.os }}-benchmark

      - name: Create results file
        run: |
          # Close the array
          printf "[\n" | tee -a results.json

          echo "Generated results file:"
          cat results.json

      - name: Run REST API benchmark
        timeout-minutes: 20
        run: |
          source pyenv/bin/activate
          ./.ci/bench_rest_api.sh

      - name: Run P2P sync benchmark
        timeout-minutes: 60
        run:
          ./.ci/bench_p2p_sync.sh

      - name: Run BFT sync benchmark
        timeout-minutes: 60
        run:
          ./.ci/bench_bft_sync.sh

      - name: Install snarkOS (production)
        run:
          cargo install --path=. --locked

      - name: Run CDN sync benchmark
        timeout-minutes: 60
        run:
          ./.ci/bench_cdn_sync.sh

      - name: Finish results file
        run: |
          printf "]\n" | tee -a results.json
          echo "\n\nFile Contents:"
          cat results.json


      - name: Generate benchmark results
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: snarkOS Benchmarks
          alert-threshold: '150%'
          comment-on-alert: true
          # TODO: Set to `true` once the benchmarks have less noise.
          fail-on-alert: false
          github-token: ${{ secrets.GITHUB_TOKEN }}
          # Use generic JSON format
          # Note: there is no way to set smallerIsBetter for individual benchmarks right now,
          # but for all, except the sync variance, bigger is indeed better.
          tool: 'customBiggerIsBetter'
          output-file-path: results.json
          alert-comment-cc-users: '@kaimast'
          # Only push the results on staging (see below)
          auto-push: false
          # Enable Job Summary for PRs
          summary-always: true

      - name: Push benchmark result
        # Avoid pushing results on pull requests or experimental branches, to reduce noise in the data.
        # The results for all other runs are still accessible through the workflow summary.
        if: github.ref_name == 'staging'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: git push origin gh-pages