DynamicPPL.jl/.github/workflows/Benchmarking.yml at main · TuringLang/DynamicPPL.jl · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
name: Benchmarking

on:
  pull_request:

permissions:
  pull-requests: write

# Cancel in-flight runs on the same PR when a new commit arrives. Benchmark
# jobs are slow (~10min each), so back-to-back force-pushes would otherwise
# spawn parallel runs that race to post the comment.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}

jobs:
  benchmark-pr:
    # OS pinned (rather than `ubuntu-latest`) so that successive runs land on
    # the same VM family — GitHub silently rotates `latest` and the noise
    # floor changes between runs. Julia version pinned for the same reason:
    # comparing timings under different compiler versions is meaningless.
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: julia-actions/setup-julia@v3
        with:
          version: '1.11'
      - uses: julia-actions/cache@v3

      - name: Run benchmarks
        working-directory: ./benchmarks
        run: |
          julia --project=. -e 'using Pkg; Pkg.instantiate()'
          julia -e 'using InteractiveUtils; versioninfo()' > version_info.txt
          # `tee` so the table also appears in the workflow log at-a-glance.
          julia --project=. benchmarks.jl markdown | tee results.md

      - uses: actions/upload-artifact@v7
        with:
          name: benchmark-pr
          path: |
            benchmarks/results.md
            benchmarks/version_info.txt

  benchmark-main:
    # Tracks main's moving HEAD — the displayed main SHA may shift between
    # successive re-runs of the same PR if main advances in the interim.
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    outputs:
      sha: ${{ steps.mainsha.outputs.sha }}
    steps:
      - uses: actions/checkout@v7
        with:
          ref: main
      - id: mainsha
        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
      - uses: julia-actions/setup-julia@v3
        with:
          version: '1.11'
      - uses: julia-actions/cache@v3

      - name: Run benchmarks
        working-directory: ./benchmarks
        run: |
          julia --project=. -e 'using Pkg; Pkg.instantiate()'
          julia --project=. benchmarks.jl markdown | tee results.md

      - uses: actions/upload-artifact@v7
        with:
          name: benchmark-main
          path: benchmarks/results.md

  post-comment:
    needs: [benchmark-pr, benchmark-main]
    # Post the comment as long as the PR-head bench succeeded. If the main
    # bench failed (e.g. transitionally, before this PR's bench changes are on
    # main), the comment still goes up with a note in place of main's numbers.
    if: ${{ !cancelled() && needs.benchmark-pr.result == 'success' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/download-artifact@v8
        with:
          name: benchmark-pr
          path: head
      - uses: actions/download-artifact@v8
        if: needs.benchmark-main.result == 'success'
        with:
          name: benchmark-main
          path: main

      - name: Build comment body
        run: |
          head_sha='${{ github.event.pull_request.head.sha }}'
          main_sha='${{ needs.benchmark-main.outputs.sha }}'
          main_status='${{ needs.benchmark-main.result }}'
          {
            echo "<!-- benchmark-report -->"
            echo "## Benchmarks @ ${head_sha}"
            echo ""
            echo "**Performance Ratio:** gradient time divided by log-density time."
            echo ""
            echo "For very small models these ratios are noisy across runs and machines; raw primal and gradient timings are more reliable. The benchmarks are aimed at DynamicPPL developers and mainly catch obvious allocation or type-stability regressions. See [benchmark notes](https://github.com/TuringLang/DynamicPPL.jl/tree/main/benchmarks#interpreting-results) for details."
            echo ""
            cat head/results.md
            echo ""
            if [[ "$main_status" == "success" ]]; then
              echo "<details><summary>Main @ ${main_sha}</summary>"
              echo ""
              cat main/results.md
              echo ""
              echo "</details>"
              echo ""
            else
              echo "Main benchmark job did not succeed (\`${main_status}\`) — see workflow logs."
              echo ""
            fi
            echo "<details><summary>Environment</summary>"
            echo "<pre>"
            cat head/version_info.txt
            echo "</pre>"
            echo "</details>"
            echo "<!-- benchmark-report -->"
          } > body.md

      - name: Find existing benchmark comment
        uses: peter-evans/find-comment@v4
        id: find_comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: github-actions[bot]
          body-includes: "<!-- benchmark-report -->"

      - name: Create or update benchmark comment
        uses: peter-evans/create-or-update-comment@v5
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body-path: body.md
          comment-id: ${{ steps.find_comment.outputs.comment-id }}
          edit-mode: replace