sl-sh/.github/workflows/pr_benchmarks.yml at main · sl-sh-dev/sl-sh · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
name: Bencher

on:
  push:
    branches:
      - main
  pull_request:
    types: [opened, reopened, synchronize, closed]
  workflow_dispatch:

env:
  RUST_BACKTRACE: 1

jobs:
  target_branch:
    runs-on: ubuntu-latest
    outputs:
      branch_name: ${{ steps.get_branch.outputs.branch_name }}
    steps:
      - id: get_branch
        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
        run: echo "branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | tee $GITHUB_OUTPUT

  backup_branch:
    runs-on: ubuntu-latest
    outputs:
      branch_name: ${{ steps.get_branch.outputs.branch_name }}
    steps:
      - id: get_branch
        if: github.event_name != 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
        run: echo "branch_name=main" >> "$GITHUB_OUTPUT"

  benchmark_pr_with_bencher_iai:
    needs: [target_branch, backup_branch]
    permissions:
      pull-requests: write
    name: Continuous Benchmarking with Bencher Iai
    runs-on: ubuntu-latest
    env:
      BENCHER_PROJECT: sl-sh
      BENCHER_TESTBED: ubuntu-latest
      BENCHER_ADAPTER: rust_iai_callgrind
      BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
      GITHUB_EVENT_NAME: ${{ github.event_name }}
      GITHUB_EVENT_PATH: ${{ github.event_path }}
      GITHUB_SHA: ${{ github.sha }}
      GITHUB_ACTIONS: true
      BRANCH: ${{ needs.target_branch.outputs.branch_name || needs.backup_branch.outputs.branch_name || github.event.pull_request.base.ref || 'main' }}
    steps:
      - run: sudo apt-get update
      - run: sudo apt install -y valgrind gnuplot
      - uses: actions/checkout@v4
      - uses: bencherdev/bencher@main
      - run: rustup toolchain install stable --profile minimal
      - uses: seepine/hash-files@v1
        id: get-hash
        with:
          patterns: 'Cargo.lock'
      - name: Rust cache
        uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
          cache-all-crates: true
          shared-key: "${{ steps.get-hash.outputs.hash }}"
      - run: cargo install iai-callgrind-runner --version 0.14.0
      - name: cargo test --no-run benches (debug mode) to check compile
        run: cargo test --benches --no-run
      - name: Track Benchmarks with Bencher
        run: |
          echo "Running bencher for branch ${BRANCH}."
          echo "Lower threshold: ${{ vars.BENCHER_LOWER_THRESHOLD_IAI }}"
          echo "Upper threshold: ${{ vars.BENCHER_UPPER_THRESHOLD_IAI }}"
          bencher run \
          --testbed "ubuntu-latest" \
          --github-actions "${{ secrets.GITHUB_TOKEN }}" \
          --token "${{ secrets.BENCHER_API_TOKEN }}" \
          --err \
          --branch "${BRANCH}" \
          --branch-start-point "main" \
          --threshold-measure estimated_cycles \
          --threshold-min-sample-size 4 \
          --threshold-test z_score \
          --threshold-lower-boundary ${{ vars.BENCHER_LOWER_THRESHOLD_IAI }} \
          --threshold-upper-boundary ${{ vars.BENCHER_UPPER_THRESHOLD_IAI }} \
          --branch-reset \
          "cargo bench --all"

  benchmark_pr_with_bencher_criterion:
    needs: [target_branch, backup_branch]
    permissions:
      pull-requests: write
    name: Continuous Benchmarking with Bencher Criterion
    runs-on: ubuntu-latest
    env:
      BENCHER_PROJECT: sl-sh
      BENCHER_TESTBED: ubuntu-latest
      BENCHER_ADAPTER: rust_criterion
      BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
      GITHUB_EVENT_NAME: ${{ github.event_name }}
      GITHUB_EVENT_PATH: ${{ github.event_path }}
      GITHUB_SHA: ${{ github.sha }}
      GITHUB_ACTIONS: true
      BRANCH: ${{ needs.target_branch.outputs.branch_name || needs.backup_branch.outputs.branch_name || github.event.pull_request.base.ref || 'main' }}
    steps:
      - run: sudo apt-get update
      - run: sudo apt install -y valgrind gnuplot
      - uses: actions/checkout@v4
      - uses: bencherdev/bencher@main
      - run: rustup toolchain install stable --profile minimal
      - uses: seepine/hash-files@v1
        id: get-hash
        with:
          patterns: 'Cargo.lock'
      - name: Rust cache
        uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
          cache-all-crates: true
          shared-key: "${{ steps.get-hash.outputs.hash }}"
      - run: cargo install iai-callgrind-runner --version 0.14.0
      - name: cargo test --no-run benches (debug mode) to check compile
        run: cargo test --benches --no-run

      - name: Track Benchmarks with Bencher
        run: |
          echo "Lower threshold: ${{ vars.BENCHER_LOWER_THRESHOLD_CRITERION }}"
          echo "Upper threshold: ${{ vars.BENCHER_UPPER_THRESHOLD_CRITERION }}"
          echo "Running bencher for branch $BRANCH."
          bencher run \
          --testbed "ubuntu-latest" \
          --github-actions "${{ secrets.GITHUB_TOKEN }}" \
          --token "${{ secrets.BENCHER_API_TOKEN }}" \
          --err \
          --branch "$BRANCH" \
          --branch-start-point "main" \
          --threshold-measure latency \
          --threshold-min-sample-size 4 \
          --threshold-test z_score \
          --threshold-lower-boundary ${{ vars.BENCHER_LOWER_THRESHOLD_CRITERION }} \
          --threshold-upper-boundary ${{ vars.BENCHER_UPPER_THRESHOLD_CRITERION }} \
          --branch-reset \
          "cargo bench --all"