Skip to content

Commit bd0f765

Browse files
Arbousier1Arbousier1
committed
Prepare benchmark stack once per campaign
1 parent d498572 commit bd0f765

1 file changed

Lines changed: 160 additions & 24 deletions

File tree

‎.github/workflows/upstream-runtime-comparison.yml‎

Lines changed: 160 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,14 @@ concurrency:
1313
cancel-in-progress: true
1414

1515
jobs:
16-
compare-upstream-runtime:
17-
name: Paper 26.1.2-74 ${{ matrix.scenario }} run ${{ matrix.run_number }}
16+
prepare-upstream-runtime:
17+
name: Prepare immutable comparison stack
1818
if: >-
1919
github.event_name == 'workflow_dispatch' ||
2020
github.event.action != 'labeled' ||
2121
github.event.label.name == 'upstream-runtime-formal'
2222
runs-on: ubuntu-latest
23-
timeout-minutes: 20
24-
strategy:
25-
fail-fast: false
26-
max-parallel: 24
27-
matrix:
28-
scenario: [dropped-items, block-active]
29-
run_number: ${{ fromJSON((github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '[1,2,3,4,5,6,7,8,9,10,11,12]' || '[1,2,3,4]') }}
23+
timeout-minutes: 15
3024
env:
3125
UPSTREAM_BUILD_NUMBER: "163"
3226
UPSTREAM_SOURCE_SHA: c7f9dd0457451537653bf4b4c0eb0e4298c51187
@@ -36,14 +30,6 @@ jobs:
3630
PAPER_BUILD: "74"
3731
PAPER_SHA256: 1d70b1dab9cf4a6de615209a536f3a45a2186240253c428213ce2188ab95e5f7
3832
COMPARE_RUNTIME_PROFILE: optimized-candidate
39-
CAMPAIGN_KIND: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && 'formal' || 'smoke' }}
40-
CAMPAIGN_RUNS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '12' || '4' }}
41-
CAMPAIGN_WARMUP_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '60' || '10' }}
42-
CAMPAIGN_SETTLE_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '20' || '5' }}
43-
CAMPAIGN_MEASURE_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '120' || '10' }}
44-
CAMPAIGN_SCENARIO: ${{ matrix.scenario }}
45-
CAMPAIGN_SCENE_SIZE: ${{ matrix.scenario == 'dropped-items' && '512' || '1024' }}
46-
CAMPAIGN_RUN_NUMBER: ${{ matrix.run_number }}
4733
EXPECTED_HARNESS_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
4834

4935
steps:
@@ -78,10 +64,10 @@ jobs:
7864
node tools/perf/analyze-phase2-protocol-trace.js --self-test
7965
pwsh -NoProfile -File tools/perf/analyze-phase2-abba.ps1 -SelfTest
8066
81-
- name: Build and test comparison harness
67+
- name: Build and test comparison harness once
8268
run: ./gradlew clean check runtimeComparisonJar --no-daemon --no-build-cache --rerun-tasks
8369

84-
- name: Build fixed production candidate in a detached worktree
70+
- name: Build fixed production candidate once
8571
run: |
8672
set -euo pipefail
8773
PRODUCTION_WORKTREE="$RUNNER_TEMP/interactionvisualizer-production-b3c2453"
@@ -95,7 +81,7 @@ jobs:
9581
./gradlew clean shadowJar --no-daemon --no-build-cache --rerun-tasks
9682
)
9783
98-
- name: Select candidate artifacts and create canonical configuration
84+
- name: Select candidate artifacts and canonical configuration
9985
run: |
10086
set -euo pipefail
10187
[[ "$COMPARE_RUNTIME_PROFILE" == optimized-candidate ]]
@@ -147,7 +133,7 @@ jobs:
147133
PY
148134
test -s compare-dependencies/canonical-config.yml
149135
150-
- name: Download and verify official upstream build 163
136+
- name: Download and verify official upstream once
151137
env:
152138
JENKINS_BUILD_URL: https://ci.loohpjames.com/job/InteractionVisualizer/163
153139
run: |
@@ -201,7 +187,7 @@ jobs:
201187
grep -Fxq 'main: com.loohp.interactionvisualizer.InteractionVisualizer' \
202188
compare-dependencies/upstream-plugin.yml
203189
204-
- name: Download and verify Paper 26.1.2 build 74
190+
- name: Download and verify Paper once
205191
env:
206192
PAPER_USER_AGENT: InteractionVisualizer-Upstream-Comparison/1.0 (https://github.com/EllanServer/InteractionVisualizer)
207193
run: |
@@ -235,12 +221,156 @@ jobs:
235221
printf '%s %s\n' "$PAPER_SHA256" compare-dependencies/paper.jar \
236222
| sha256sum --check --strict
237223
238-
- name: Prepare immutable protocol client
224+
- name: Prepare immutable protocol client once
239225
run: bash tools/perf/prepare-phase2-protocol-client.sh compare-dependencies/protocol-client
240226

227+
- name: Seal prepared comparison stack
228+
run: |
229+
set -euo pipefail
230+
harness_source_sha=$(git rev-parse HEAD)
231+
candidate_source_sha=$(git -C "$PRODUCTION_WORKTREE" rev-parse HEAD)
232+
[[ "$harness_source_sha" == "$EXPECTED_HARNESS_SHA" ]]
233+
[[ "$candidate_source_sha" == "$PRODUCTION_CANDIDATE_SHA" ]]
234+
sha256sum \
235+
compare-dependencies/upstream.jar \
236+
compare-dependencies/rewrite.jar \
237+
compare-dependencies/runtime-comparison-driver.jar \
238+
compare-dependencies/canonical-config.yml \
239+
compare-dependencies/paper.jar \
240+
compare-dependencies/protocol-client/client-build-manifest.json \
241+
> compare-dependencies/prepared-files.sha256
242+
python3 - compare-dependencies/prepared-stack.json \
243+
"$harness_source_sha" "$candidate_source_sha" "$COMPARE_RUNTIME_PROFILE" \
244+
"$UPSTREAM_SOURCE_SHA" "$UPSTREAM_ARTIFACT_SHA256" "$PAPER_SHA256" <<'PY'
245+
from pathlib import Path
246+
import json
247+
import sys
248+
249+
output, harness, candidate, profile, upstream_source, upstream_sha, paper_sha = sys.argv[1:]
250+
Path(output).write_text(json.dumps({
251+
"schemaVersion": 1,
252+
"harnessSourceSha": harness,
253+
"candidateSourceSha": candidate,
254+
"runtimeProfile": profile,
255+
"upstreamSourceSha": upstream_source,
256+
"upstreamArtifactSha256": upstream_sha,
257+
"paperSha256": paper_sha,
258+
"distribution": "single-build-artifact",
259+
}, indent=2) + "\n", encoding="utf-8")
260+
PY
261+
262+
- name: Package prepared comparison stack
263+
run: |
264+
set -euo pipefail
265+
mkdir prepared-transfer
266+
tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner \
267+
-czf prepared-transfer/comparison-stack.tar.gz compare-dependencies
268+
sha256sum prepared-transfer/comparison-stack.tar.gz \
269+
> prepared-transfer/comparison-stack.sha256
270+
271+
- name: Upload prepared comparison stack
272+
uses: actions/upload-artifact@v4
273+
with:
274+
name: upstream-runtime-prepared-${{ github.run_id }}
275+
path: prepared-transfer
276+
if-no-files-found: error
277+
retention-days: 1
278+
279+
compare-upstream-runtime:
280+
name: Paper 26.1.2-74 ${{ matrix.scenario }} run ${{ matrix.run_number }}
281+
if: >-
282+
github.event_name == 'workflow_dispatch' ||
283+
github.event.action != 'labeled' ||
284+
github.event.label.name == 'upstream-runtime-formal'
285+
runs-on: ubuntu-latest
286+
needs: prepare-upstream-runtime
287+
timeout-minutes: 20
288+
strategy:
289+
fail-fast: false
290+
max-parallel: 24
291+
matrix:
292+
scenario: [dropped-items, block-active]
293+
run_number: ${{ fromJSON((github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '[1,2,3,4,5,6,7,8,9,10,11,12]' || '[1,2,3,4]') }}
294+
env:
295+
UPSTREAM_BUILD_NUMBER: "163"
296+
UPSTREAM_SOURCE_SHA: c7f9dd0457451537653bf4b4c0eb0e4298c51187
297+
UPSTREAM_ARTIFACT_SIZE: "5799385"
298+
UPSTREAM_ARTIFACT_SHA256: a7ffc2ba053c74681feabc698e9fdb959ebd4f8252206fedd8801979e3de30c0
299+
PRODUCTION_CANDIDATE_SHA: c28db0146ec2f35eaf066b4202b33f08d06bbc4b
300+
PAPER_BUILD: "74"
301+
PAPER_SHA256: 1d70b1dab9cf4a6de615209a536f3a45a2186240253c428213ce2188ab95e5f7
302+
COMPARE_RUNTIME_PROFILE: optimized-candidate
303+
CAMPAIGN_KIND: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && 'formal' || 'smoke' }}
304+
CAMPAIGN_RUNS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '12' || '4' }}
305+
CAMPAIGN_WARMUP_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '60' || '10' }}
306+
CAMPAIGN_SETTLE_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '20' || '5' }}
307+
CAMPAIGN_MEASURE_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '120' || '10' }}
308+
CAMPAIGN_SCENARIO: ${{ matrix.scenario }}
309+
CAMPAIGN_SCENE_SIZE: ${{ matrix.scenario == 'dropped-items' && '512' || '1024' }}
310+
CAMPAIGN_RUN_NUMBER: ${{ matrix.run_number }}
311+
EXPECTED_HARNESS_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
312+
313+
steps:
314+
- name: Check out candidate head
315+
uses: actions/checkout@v4
316+
with:
317+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
318+
fetch-depth: 0
319+
persist-credentials: false
320+
321+
- name: Set up Java 25
322+
uses: actions/setup-java@v4
323+
with:
324+
distribution: temurin
325+
java-version: "25"
326+
327+
- name: Set up Node 24
328+
uses: actions/setup-node@v4
329+
with:
330+
node-version: "24"
331+
332+
- name: Distribute prepared comparison stack locally
333+
uses: actions/download-artifact@v4
334+
with:
335+
name: upstream-runtime-prepared-${{ github.run_id }}
336+
path: prepared-transfer
337+
338+
- name: Verify and unpack prepared comparison stack
339+
run: |
340+
set -euo pipefail
341+
sha256sum --check --strict prepared-transfer/comparison-stack.sha256
342+
tar -xzf prepared-transfer/comparison-stack.tar.gz
343+
241344
- name: Establish immutable campaign provenance
242345
run: |
243346
set -euo pipefail
347+
sha256sum --check --strict compare-dependencies/prepared-files.sha256
348+
(
349+
cd compare-dependencies/protocol-client
350+
sha256sum --check --strict client-files.sha256
351+
)
352+
python3 - compare-dependencies/prepared-stack.json \
353+
"$EXPECTED_HARNESS_SHA" "$PRODUCTION_CANDIDATE_SHA" \
354+
"$COMPARE_RUNTIME_PROFILE" "$UPSTREAM_SOURCE_SHA" \
355+
"$UPSTREAM_ARTIFACT_SHA256" "$PAPER_SHA256" <<'PY'
356+
import json
357+
import sys
358+
359+
source, harness, candidate, profile, upstream_source, upstream_sha, paper_sha = sys.argv[1:]
360+
data = json.load(open(source, encoding="utf-8"))
361+
expected = {
362+
"schemaVersion": 1,
363+
"harnessSourceSha": harness,
364+
"candidateSourceSha": candidate,
365+
"runtimeProfile": profile,
366+
"upstreamSourceSha": upstream_source,
367+
"upstreamArtifactSha256": upstream_sha,
368+
"paperSha256": paper_sha,
369+
"distribution": "single-build-artifact",
370+
}
371+
if data != expected:
372+
raise SystemExit(f"prepared comparison stack mismatch: {data!r} != {expected!r}")
373+
PY
244374
read -r available_cpu_count server_cpu_set client_cpu_set < <(
245375
python3 - <<'PY'
246376
import os
@@ -253,7 +383,7 @@ jobs:
253383
)
254384
harness_source_sha=$(git rev-parse HEAD)
255385
[[ "$harness_source_sha" == "$EXPECTED_HARNESS_SHA" ]]
256-
candidate_source_sha=$(git -C "$PRODUCTION_WORKTREE" rev-parse HEAD)
386+
candidate_source_sha="$PRODUCTION_CANDIDATE_SHA"
257387
[[ "$candidate_source_sha" == "$PRODUCTION_CANDIDATE_SHA" ]]
258388
rewrite_sha=$(sha256sum compare-dependencies/rewrite.jar | awk '{print $1}')
259389
driver_sha=$(sha256sum compare-dependencies/runtime-comparison-driver.jar | awk '{print $1}')
@@ -294,6 +424,7 @@ jobs:
294424
printf '%s\n' \
295425
"runtimeProfile=$COMPARE_RUNTIME_PROFILE" \
296426
'samplingMode=independent-runners' \
427+
'dependencyDistribution=single-build-artifact' \
297428
'requestedPacketOnlyStatic=true' \
298429
'requestedEventDrivenBlockUpdates=true'
299430
printf '%s\n' "harnessSourceSha=$harness_source_sha"
@@ -351,6 +482,7 @@ jobs:
351482
"protocolClientKeepAliveTimeoutMs": 300000,
352483
"runtimeProfile": runtime_profile,
353484
"samplingMode": "independent-runners",
485+
"dependencyDistribution": "single-build-artifact",
354486
"requestedFlags": {
355487
"packetOnlyStatic": runtime_profile == "optimized-candidate",
356488
"eventDrivenBlockUpdates": runtime_profile == "optimized-candidate",
@@ -856,6 +988,7 @@ jobs:
856988
"runs": int(runs),
857989
"runtimeProfile": "optimized-candidate",
858990
"samplingMode": "independent-runners",
991+
"dependencyDistribution": "single-build-artifact",
859992
"networkTimeoutSeconds": 300,
860993
"protocolClientKeepAliveTimeoutMs": 300000,
861994
}
@@ -1199,6 +1332,7 @@ jobs:
11991332
"conclusion": conclusion,
12001333
"formalComplete": formal,
12011334
"samplingMode": "independent-runners",
1335+
"dependencyDistribution": "single-build-artifact",
12021336
"registeredGates": {
12031337
"primaryMeanImprovement": {
12041338
"medianBRatioToAMaximum": 0.90,
@@ -1332,6 +1466,8 @@ jobs:
13321466
raise SystemExit(f"{scenario} runtime profile mismatch")
13331467
if document.get("samplingMode") != "independent-runners":
13341468
raise SystemExit(f"{scenario} sampling mode mismatch")
1469+
if document.get("dependencyDistribution") != "single-build-artifact":
1470+
raise SystemExit(f"{scenario} dependency distribution mismatch")
13351471
if document.get("requestedFlags") != {
13361472
"packetOnlyStatic": True,
13371473
"eventDrivenBlockUpdates": True,

0 commit comments

Comments
 (0)