Skip to content

Commit cdf984b

Browse files
AdamGSAdamGS
committed
another attempt
Signed-off-by: Adam Gutglick <adam@spiraldb.com>
1 parent 9334719 commit cdf984b

4 files changed

Lines changed: 111 additions & 45 deletions

File tree

‎.github/workflows/bench-dispatch.yml‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,10 @@ on:
1111

1212
permissions:
1313
actions: write
14-
contents: read
15-
pull-requests: write # for label removal and PR comments
16-
id-token: write # enables AWS-GitHub OIDC
14+
pull-requests: write
1715

1816
jobs:
19-
remove-bench-label:
17+
dispatch-bench:
2018
runs-on: ubuntu-latest
2119
timeout-minutes: 2
2220
if: github.event.label.name == 'action/benchmark'
@@ -26,13 +24,19 @@ jobs:
2624
with:
2725
labels: action/benchmark
2826
fail_on_error: true
27+
- name: Dispatch PR Benchmarks workflow
28+
env:
29+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
30+
run: |
31+
gh workflow run bench-pr.yml \
32+
--repo "${{ github.repository }}" \
33+
--ref "${{ github.event.pull_request.head.ref }}" \
34+
-f pr_number="${{ github.event.pull_request.number }}" \
35+
-f pr_head_sha="${{ github.event.pull_request.head.sha }}" \
36+
-f is_fork="${{ github.event.pull_request.head.repo.fork }}" \
37+
-f head_ref="${{ github.event.pull_request.head.ref }}"
2938
30-
bench:
31-
needs: remove-bench-label
32-
uses: ./.github/workflows/bench-pr.yml
33-
secrets: inherit
34-
35-
remove-sql-label:
39+
dispatch-sql:
3640
runs-on: ubuntu-latest
3741
timeout-minutes: 2
3842
if: github.event.label.name == 'action/benchmark-sql'
@@ -42,8 +46,14 @@ jobs:
4246
with:
4347
labels: action/benchmark-sql
4448
fail_on_error: true
45-
46-
sql-bench:
47-
needs: remove-sql-label
48-
uses: ./.github/workflows/sql-pr.yml
49-
secrets: inherit
49+
- name: Dispatch PR SQL Benchmarks workflow
50+
env:
51+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
52+
run: |
53+
gh workflow run sql-pr.yml \
54+
--repo "${{ github.repository }}" \
55+
--ref "${{ github.event.pull_request.head.ref }}" \
56+
-f pr_number="${{ github.event.pull_request.number }}" \
57+
-f pr_head_sha="${{ github.event.pull_request.head.sha }}" \
58+
-f is_fork="${{ github.event.pull_request.head.repo.fork }}" \
59+
-f head_ref="${{ github.event.pull_request.head.ref }}"

‎.github/workflows/bench-pr.yml‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,34 @@
11
# Runs all benchmarks once for a pull request.
2-
# Called from bench-dispatch.yml when the `action/benchmark` label is added.
2+
# Dispatched from bench-dispatch.yml when the `action/benchmark` label is added.
33

44
name: PR Benchmarks
55

66
concurrency:
77
# The group causes runs to queue instead of running in parallel.
8-
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
8+
group: ${{ github.workflow }}-${{ inputs.head_ref || github.run_id }}
99
# Don't cancel benchmarks that are already running, instead just queue them up.
1010
cancel-in-progress: false
1111

1212
on:
13-
workflow_call: { }
14-
workflow_dispatch: { }
13+
workflow_dispatch:
14+
inputs:
15+
pr_number:
16+
description: "PR number for posting comments"
17+
required: false
18+
type: string
19+
pr_head_sha:
20+
description: "PR head SHA for checkout"
21+
required: false
22+
type: string
23+
is_fork:
24+
description: "Whether the PR is from a fork"
25+
required: false
26+
type: string
27+
default: "false"
28+
head_ref:
29+
description: "PR head ref for concurrency grouping"
30+
required: false
31+
type: string
1532

1633
permissions:
1734
contents: read
@@ -23,7 +40,7 @@ jobs:
2340
timeout-minutes: 120
2441
runs-on: >-
2542
${{ github.repository == 'vortex-data/vortex'
26-
&& format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
43+
&& format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, inputs.is_fork != 'true' && '/extras=s3-cache' || '')
2744
|| 'ubuntu-latest' }}
2845
strategy:
2946
matrix:
@@ -35,12 +52,12 @@ jobs:
3552
name: Compression
3653
steps:
3754
- uses: runs-on/action@v2
38-
if: github.event.pull_request.head.repo.fork == false
55+
if: inputs.is_fork != 'true'
3956
with:
4057
sccache: s3
4158
- uses: actions/checkout@v6
4259
with:
43-
ref: ${{ github.event.pull_request.head.sha }}
60+
ref: ${{ inputs.pr_head_sha || github.sha }}
4461
- name: Setup benchmark environment
4562
run: sudo bash scripts/setup-benchmark.sh
4663
- uses: ./.github/actions/setup-rust
@@ -63,7 +80,7 @@ jobs:
6380
cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }}
6481
6582
- name: Setup Polar Signals
66-
if: github.event.pull_request.head.repo.fork == false
83+
if: inputs.is_fork != 'true'
6784
uses: polarsignals/gh-actions-ps-profiling@v0.8.1
6885
with:
6986
polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
@@ -80,7 +97,7 @@ jobs:
8097
bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
8198
8299
- name: Setup AWS CLI
83-
if: github.event.pull_request.head.repo.fork == false
100+
if: inputs.is_fork != 'true'
84101
uses: aws-actions/configure-aws-credentials@v6
85102
with:
86103
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
@@ -114,27 +131,32 @@ jobs:
114131
cat comment.md >> $GITHUB_STEP_SUMMARY
115132
116133
- name: Comment PR
117-
if: github.event.pull_request.head.repo.fork == false
134+
if: inputs.is_fork != 'true' && inputs.pr_number != ''
118135
uses: thollander/actions-comment-pull-request@v3
119136
with:
120137
file-path: comment.md
121138
comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
139+
pr-number: ${{ inputs.pr_number }}
122140

123141
- name: Comment PR on failure
124-
if: failure() && github.event.pull_request.head.repo.fork == false
142+
if: failure() && inputs.is_fork != 'true' && inputs.pr_number != ''
125143
uses: thollander/actions-comment-pull-request@v3
126144
with:
127145
message: |
128146
# BENCHMARK FAILED
129147
130148
Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
131149
comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
150+
pr-number: ${{ inputs.pr_number }}
132151

133152
sql:
134153
uses: ./.github/workflows/sql-benchmarks.yml
135154
secrets: inherit
136155
with:
137156
mode: "pr"
157+
pr_head_sha: ${{ inputs.pr_head_sha }}
158+
is_fork: ${{ inputs.is_fork }}
159+
pr_number: ${{ inputs.pr_number }}
138160
benchmark_matrix: |
139161
[
140162
{

‎.github/workflows/sql-benchmarks.yml‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,18 @@ on:
1010
required: false
1111
type: string
1212
default: c6id.8xlarge
13+
pr_head_sha:
14+
required: false
15+
type: string
16+
default: ""
17+
is_fork:
18+
required: false
19+
type: string
20+
default: "false"
21+
pr_number:
22+
required: false
23+
type: string
24+
default: ""
1325
benchmark_matrix:
1426
required: false
1527
type: string
@@ -107,17 +119,17 @@ jobs:
107119

108120
runs-on: >-
109121
${{ github.repository == 'vortex-data/vortex'
110-
&& format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '')
122+
&& format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.id, (inputs.mode != 'pr' || inputs.is_fork != 'true') && '/extras=s3-cache' || '')
111123
|| 'ubuntu-latest' }}
112124
steps:
113125
- uses: runs-on/action@v2
114-
if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false
126+
if: inputs.mode != 'pr' || inputs.is_fork != 'true'
115127
with:
116128
sccache: s3
117129
- uses: actions/checkout@v6
118130
if: inputs.mode == 'pr'
119131
with:
120-
ref: ${{ github.event.pull_request.head.sha }}
132+
ref: ${{ inputs.pr_head_sha }}
121133

122134
- uses: actions/checkout@v6
123135
if: inputs.mode != 'pr'
@@ -166,14 +178,14 @@ jobs:
166178
$opts
167179
168180
- name: Setup AWS CLI
169-
if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false
181+
if: inputs.mode != 'pr' || inputs.is_fork != 'true'
170182
uses: aws-actions/configure-aws-credentials@v6
171183
with:
172184
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
173185
aws-region: us-east-1
174186

175187
- name: Upload data
176-
if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false)
188+
if: matrix.remote_storage != null && (inputs.mode != 'pr' || inputs.is_fork != 'true')
177189
shell: bash
178190
env:
179191
AWS_REGION: "us-east-1"
@@ -182,7 +194,7 @@ jobs:
182194
aws s3 cp --recursive ${{matrix.local_dir}} ${{ matrix.remote_storage }}
183195
184196
- name: Setup Polar Signals
185-
if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false
197+
if: inputs.mode != 'pr' || inputs.is_fork != 'true'
186198
uses: polarsignals/gh-actions-ps-profiling@v0.8.1
187199
with:
188200
polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
@@ -192,28 +204,28 @@ jobs:
192204
extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz
193205

194206
- name: Run ${{ matrix.name }} benchmark
195-
if: matrix.remote_storage == null || github.event.pull_request.head.repo.fork == true
207+
if: matrix.remote_storage == null || inputs.is_fork == 'true'
196208
shell: bash
197209
env:
198210
OTEL_SERVICE_NAME: "vortex-bench"
199211
OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf"
200-
OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}"
201-
OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}"
212+
OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || inputs.is_fork != 'true') && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}"
213+
OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || inputs.is_fork != 'true') && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}"
202214
OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}"
203215
run: |
204216
bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \
205217
${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \
206218
${{ matrix.scale_factor && format('--scale-factor {0}', matrix.scale_factor) || '' }}
207219
208220
- name: Run ${{ matrix.name }} benchmark (remote)
209-
if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false)
221+
if: matrix.remote_storage != null && (inputs.mode != 'pr' || inputs.is_fork != 'true')
210222
shell: bash
211223
env:
212224
AWS_REGION: "us-east-1"
213225
OTEL_SERVICE_NAME: "vortex-bench"
214226
OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf"
215-
OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}"
216-
OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}"
227+
OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || inputs.is_fork != 'true') && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}"
228+
OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || inputs.is_fork != 'true') && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}"
217229
OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}"
218230
run: |
219231
bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \
@@ -251,24 +263,26 @@ jobs:
251263
cat comment.md >> $GITHUB_STEP_SUMMARY
252264
253265
- name: Comment PR
254-
if: inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
266+
if: inputs.mode == 'pr' && inputs.is_fork != 'true' && inputs.pr_number != ''
255267
uses: thollander/actions-comment-pull-request@v3
256268
with:
257269
file-path: comment.md
258270
# There is exactly one comment per comment-tag. If a comment with this tag already exists,
259271
# this action will *update* the comment instead of posting a new comment. Therefore, each
260272
# unique benchmark configuration must have a unique comment-tag.
261273
comment-tag: bench-pr-comment-${{ matrix.id }}
274+
pr-number: ${{ inputs.pr_number }}
262275

263276
- name: Comment PR on failure
264-
if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
277+
if: failure() && inputs.mode == 'pr' && inputs.is_fork != 'true' && inputs.pr_number != ''
265278
uses: thollander/actions-comment-pull-request@v3
266279
with:
267280
message: |
268-
# 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨
281+
# SQL BENCHMARK FAILED
269282
270283
Benchmark `${{ matrix.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
271284
comment-tag: bench-pr-comment-${{ matrix.id }}
285+
pr-number: ${{ inputs.pr_number }}
272286

273287
- name: Upload Benchmark Results
274288
if: inputs.mode == 'develop'

‎.github/workflows/sql-pr.yml‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,32 @@
11
# Runs SQL benchmarks once for a pull request.
2-
# Called from bench-dispatch.yml when the `action/benchmark-sql` label is added.
2+
# Dispatched from bench-dispatch.yml when the `action/benchmark-sql` label is added.
33

44
name: PR SQL Benchmarks
55

66
concurrency:
7-
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
7+
group: ${{ github.workflow }}-${{ inputs.head_ref || github.run_id }}
88
cancel-in-progress: true
99

1010
on:
11-
workflow_call: { }
12-
workflow_dispatch: { }
11+
workflow_dispatch:
12+
inputs:
13+
pr_number:
14+
description: "PR number for posting comments"
15+
required: false
16+
type: string
17+
pr_head_sha:
18+
description: "PR head SHA for checkout"
19+
required: false
20+
type: string
21+
is_fork:
22+
description: "Whether the PR is from a fork"
23+
required: false
24+
type: string
25+
default: "false"
26+
head_ref:
27+
description: "PR head ref for concurrency grouping"
28+
required: false
29+
type: string
1330

1431
permissions:
1532
contents: read
@@ -22,3 +39,6 @@ jobs:
2239
secrets: inherit
2340
with:
2441
mode: "pr"
42+
pr_head_sha: ${{ inputs.pr_head_sha }}
43+
is_fork: ${{ inputs.is_fork }}
44+
pr_number: ${{ inputs.pr_number }}

0 commit comments

Comments
 (0)