Skip to content

Commit f91c470

Browse files
Arbousier1Arbousier1
committed
Parallelize independent runtime samples
1 parent d99826e commit f91c470

2 files changed

Lines changed: 282 additions & 40 deletions

File tree

‎.github/workflows/upstream-runtime-comparison.yml‎

Lines changed: 193 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,19 @@ concurrency:
1414

1515
jobs:
1616
compare-upstream-runtime:
17-
name: Paper 26.1.2-74 ${{ matrix.scenario }} upstream vs rewrite
17+
name: Paper 26.1.2-74 ${{ matrix.scenario }} run ${{ matrix.run_number }}
1818
if: >-
1919
github.event_name == 'workflow_dispatch' ||
2020
github.event.action != 'labeled' ||
2121
github.event.label.name == 'upstream-runtime-formal'
2222
runs-on: ubuntu-latest
23-
timeout-minutes: 100
23+
timeout-minutes: 20
2424
strategy:
2525
fail-fast: false
26+
max-parallel: 24
2627
matrix:
27-
include:
28-
- scenario: dropped-items
29-
scene_size: 512
30-
- scenario: block-active
31-
scene_size: 1024
28+
scenario: [dropped-items, block-active]
29+
run_number: ${{ fromJSON((github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '[1,2,3,4,5,6,7,8,9,10,11,12]' || '[1,2,3,4]') }}
3230
env:
3331
UPSTREAM_BUILD_NUMBER: "163"
3432
UPSTREAM_SOURCE_SHA: c7f9dd0457451537653bf4b4c0eb0e4298c51187
@@ -44,7 +42,8 @@ jobs:
4442
CAMPAIGN_SETTLE_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '20' || '5' }}
4543
CAMPAIGN_MEASURE_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '120' || '10' }}
4644
CAMPAIGN_SCENARIO: ${{ matrix.scenario }}
47-
CAMPAIGN_SCENE_SIZE: ${{ matrix.scene_size }}
45+
CAMPAIGN_SCENE_SIZE: ${{ matrix.scenario == 'dropped-items' && '512' || '1024' }}
46+
CAMPAIGN_RUN_NUMBER: ${{ matrix.run_number }}
4847
EXPECTED_HARNESS_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
4948

5049
steps:
@@ -288,6 +287,7 @@ jobs:
288287
"clientCpuSet=$client_cpu_set"
289288
printf '%s\n' \
290289
"runtimeProfile=$COMPARE_RUNTIME_PROFILE" \
290+
'samplingMode=independent-runners' \
291291
'requestedPacketOnlyStatic=true' \
292292
'requestedEventDrivenBlockUpdates=true'
293293
printf '%s\n' "harnessSourceSha=$harness_source_sha"
@@ -342,6 +342,7 @@ jobs:
342342
"settleSeconds": int(settle),
343343
"measureSeconds": int(measure),
344344
"runtimeProfile": runtime_profile,
345+
"samplingMode": "independent-runners",
345346
"requestedFlags": {
346347
"packetOnlyStatic": runtime_profile == "optimized-candidate",
347348
"eventDrivenBlockUpdates": runtime_profile == "optimized-candidate",
@@ -381,6 +382,7 @@ jobs:
381382
> compare-dependencies/campaign-files.sha256
382383
383384
- name: Run full-scene protocol preflight for both artifacts
385+
if: matrix.run_number == 1
384386
run: |
385387
set -euo pipefail
386388
preflight_root="compare-results/$CAMPAIGN_SCENARIO/preflight"
@@ -570,7 +572,7 @@ jobs:
570572
PY
571573
done
572574
573-
- name: Run restart-isolated ABBA campaign
575+
- name: Run one restart-isolated independent sample
574576
run: |
575577
set -euo pipefail
576578
result_root="compare-results/$CAMPAIGN_SCENARIO/$CAMPAIGN_KIND"
@@ -579,21 +581,21 @@ jobs:
579581
printf 'Scenario,Block,Position,Variant,RunId,StackSha256,ArtifactSha256,CaptureMethod,SourcePath\n' \
580582
> "$manifest"
581583
582-
for run_number in $(seq 1 "$CAMPAIGN_RUNS"); do
583-
block=$(( (run_number - 1) / 4 + 1 ))
584-
position=$(( (run_number - 1) % 4 + 1 ))
585-
if (( block % 2 == 1 )); then pattern=ABBA; else pattern=BAAB; fi
586-
variant=${pattern:$((position - 1)):1}
587-
run_id=$(printf '%s_%s_%02d' "${CAMPAIGN_SCENARIO//-/_}" "$variant" "$run_number")
588-
if [[ "$variant" == A ]]; then
589-
target=compare-dependencies/upstream.jar
590-
expected_artifact_sha="$UPSTREAM_ARTIFACT_SHA256"
591-
else
592-
target=compare-dependencies/rewrite.jar
593-
expected_artifact_sha="$REWRITE_ARTIFACT_SHA256"
594-
fi
584+
run_number="$CAMPAIGN_RUN_NUMBER"
585+
block=$(( (run_number - 1) / 4 + 1 ))
586+
position=$(( (run_number - 1) % 4 + 1 ))
587+
if (( block % 2 == 1 )); then pattern=ABBA; else pattern=BAAB; fi
588+
variant=${pattern:$((position - 1)):1}
589+
run_id=$(printf '%s_%s_%02d' "${CAMPAIGN_SCENARIO//-/_}" "$variant" "$run_number")
590+
if [[ "$variant" == A ]]; then
591+
target=compare-dependencies/upstream.jar
592+
expected_artifact_sha="$UPSTREAM_ARTIFACT_SHA256"
593+
else
594+
target=compare-dependencies/rewrite.jar
595+
expected_artifact_sha="$REWRITE_ARTIFACT_SHA256"
596+
fi
595597
596-
COMPARE_PLUGIN_JAR="$target" \
598+
COMPARE_PLUGIN_JAR="$target" \
597599
COMPARE_DRIVER_JAR=compare-dependencies/runtime-comparison-driver.jar \
598600
COMPARE_CONFIG_FILE=compare-dependencies/canonical-config.yml \
599601
COMPARE_PAPER_JAR=compare-dependencies/paper.jar \
@@ -608,9 +610,9 @@ jobs:
608610
COMPARE_SETTLE_SECONDS="$CAMPAIGN_SETTLE_SECONDS" \
609611
COMPARE_MEASURE_SECONDS="$CAMPAIGN_MEASURE_SECONDS" \
610612
COMPARE_PROTOCOL_TRACE_ENABLED=0 \
611-
bash tools/perf/run-upstream-runtime-once.sh
613+
bash tools/perf/run-upstream-runtime-once.sh
612614
613-
python3 - "$manifest" "$result_root" "$run_id" "$CAMPAIGN_SCENARIO" \
615+
python3 - "$manifest" "$result_root" "$run_id" "$CAMPAIGN_SCENARIO" \
614616
"$block" "$position" "$variant" "$CAMPAIGN_SCENE_SIZE" \
615617
"$CAMPAIGN_STACK_SHA256" "$expected_artifact_sha" \
616618
"$CANONICAL_CONFIG_SHA256" "$DRIVER_SHA256" "$PAPER_SHA256" \
@@ -716,8 +718,160 @@ jobs:
716718
"paper-server-tick-end-event", f"{run_id}/iv-compare.json",
717719
])
718720
PY
721+
722+
- name: Upload independent run evidence
723+
if: always()
724+
uses: actions/upload-artifact@v4
725+
with:
726+
name: upstream-runtime-run-${{ matrix.scenario }}-${{ env.CAMPAIGN_KIND }}-${{ github.run_id }}-${{ matrix.run_number }}
727+
path: |
728+
compare-results/${{ matrix.scenario }}
729+
compare-dependencies/campaign-provenance.json
730+
compare-dependencies/campaign-files.sha256
731+
compare-dependencies/upstream-build-163.json
732+
compare-dependencies/upstream-plugin.yml
733+
compare-dependencies/paper-build-74.json
734+
compare-dependencies/canonical-config.yml
735+
compare-dependencies/protocol-client/client-build-manifest.json
736+
compare-dependencies/protocol-client/client-files.sha256
737+
compare-dependencies/protocol-client/node-minecraft-protocol/package-lock.json
738+
compare-dependencies/protocol-client/node-minecraft-protocol/production-lock-inventory.json
739+
if-no-files-found: warn
740+
retention-days: 3
741+
742+
aggregate-upstream-runtime:
743+
name: Aggregate ${{ matrix.scenario }} independent samples
744+
needs: [compare-upstream-runtime]
745+
if: ${{ needs.compare-upstream-runtime.result == 'success' }}
746+
runs-on: ubuntu-latest
747+
timeout-minutes: 15
748+
strategy:
749+
fail-fast: false
750+
matrix:
751+
scenario: [dropped-items, block-active]
752+
env:
753+
UPSTREAM_BUILD_NUMBER: "163"
754+
UPSTREAM_SOURCE_SHA: c7f9dd0457451537653bf4b4c0eb0e4298c51187
755+
UPSTREAM_ARTIFACT_SHA256: a7ffc2ba053c74681feabc698e9fdb959ebd4f8252206fedd8801979e3de30c0
756+
PAPER_BUILD: "74"
757+
PAPER_SHA256: 1d70b1dab9cf4a6de615209a536f3a45a2186240253c428213ce2188ab95e5f7
758+
COMPARE_RUNTIME_PROFILE: optimized-candidate
759+
CAMPAIGN_KIND: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && 'formal' || 'smoke' }}
760+
CAMPAIGN_RUNS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '12' || '4' }}
761+
CAMPAIGN_WARMUP_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '60' || '10' }}
762+
CAMPAIGN_SETTLE_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '20' || '5' }}
763+
CAMPAIGN_MEASURE_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '120' || '10' }}
764+
CAMPAIGN_SCENARIO: ${{ matrix.scenario }}
765+
CAMPAIGN_SCENE_SIZE: ${{ matrix.scenario == 'dropped-items' && '512' || '1024' }}
766+
767+
steps:
768+
- name: Check out harness source
769+
uses: actions/checkout@v4
770+
with:
771+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
772+
fetch-depth: 1
773+
persist-credentials: false
774+
775+
- name: Download all independent run artifacts
776+
uses: actions/download-artifact@v4
777+
with:
778+
pattern: upstream-runtime-run-${{ matrix.scenario }}-${{ env.CAMPAIGN_KIND }}-${{ github.run_id }}-*
779+
path: parallel-runs
780+
781+
- name: Assemble and verify independent campaign
782+
run: |
783+
set -euo pipefail
784+
result_root="compare-results/$CAMPAIGN_SCENARIO/$CAMPAIGN_KIND"
785+
mkdir -p "$result_root" compare-dependencies
786+
manifest="$result_root/abba-manifest.csv"
787+
printf 'Scenario,Block,Position,Variant,RunId,StackSha256,ArtifactSha256,CaptureMethod,SourcePath\n' \
788+
> "$manifest"
789+
790+
mapfile -t manifests < <(find parallel-runs -type f \
791+
-path "*/compare-results/$CAMPAIGN_SCENARIO/$CAMPAIGN_KIND/abba-manifest.csv" | sort)
792+
[[ "${#manifests[@]}" == "$CAMPAIGN_RUNS" ]]
793+
for source_manifest in "${manifests[@]}"; do
794+
run_id=$(python3 - "$source_manifest" <<'PY'
795+
import csv
796+
import sys
797+
798+
rows = list(csv.DictReader(open(sys.argv[1], encoding="utf-8", newline="")))
799+
if len(rows) != 1:
800+
raise SystemExit(f"per-run manifest must have one row: {sys.argv[1]}")
801+
print(rows[0]["RunId"])
802+
PY
803+
)
804+
source_root=$(dirname "$source_manifest")
805+
[[ -d "$source_root/$run_id" ]]
806+
[[ ! -e "$result_root/$run_id" ]]
807+
cp -a "$source_root/$run_id" "$result_root/$run_id"
808+
tail -n +2 "$source_manifest" >> "$manifest"
719809
done
720810
811+
mapfile -t preflight_roots < <(find parallel-runs -type d \
812+
-path "*/compare-results/$CAMPAIGN_SCENARIO/preflight" | sort)
813+
[[ "${#preflight_roots[@]}" == 1 ]]
814+
cp -a "${preflight_roots[0]}" "compare-results/$CAMPAIGN_SCENARIO/preflight"
815+
816+
while IFS= read -r relative; do
817+
mapfile -t sources < <(find parallel-runs -type f \
818+
-path "*/compare-dependencies/$relative" | sort)
819+
[[ "${#sources[@]}" == "$CAMPAIGN_RUNS" ]]
820+
hash_count=$(sha256sum "${sources[@]}" | awk '{print $1}' | sort -u | wc -l)
821+
[[ "$hash_count" == 1 ]]
822+
destination="compare-dependencies/$relative"
823+
mkdir -p "$(dirname "$destination")"
824+
cp "${sources[0]}" "$destination"
825+
done <<'EOF'
826+
campaign-provenance.json
827+
campaign-files.sha256
828+
upstream-build-163.json
829+
upstream-plugin.yml
830+
paper-build-74.json
831+
canonical-config.yml
832+
protocol-client/client-build-manifest.json
833+
protocol-client/client-files.sha256
834+
protocol-client/node-minecraft-protocol/package-lock.json
835+
protocol-client/node-minecraft-protocol/production-lock-inventory.json
836+
EOF
837+
838+
python3 - compare-dependencies/campaign-provenance.json "$GITHUB_ENV" \
839+
"$CAMPAIGN_SCENARIO" "$CAMPAIGN_KIND" "$CAMPAIGN_RUNS" <<'PY'
840+
import json
841+
import sys
842+
843+
source, output, scenario, kind, runs = sys.argv[1:]
844+
data = json.load(open(source, encoding="utf-8"))
845+
expected = {
846+
"scenario": scenario,
847+
"campaignKind": kind,
848+
"runs": int(runs),
849+
"runtimeProfile": "optimized-candidate",
850+
"samplingMode": "independent-runners",
851+
}
852+
for field, value in expected.items():
853+
if data.get(field) != value:
854+
raise SystemExit(
855+
f"campaign provenance mismatch {field}: {data.get(field)!r} != {value!r}")
856+
cpu = data["cpuIsolation"]
857+
values = {
858+
"CANDIDATE_SOURCE_SHA": data["variantB"]["sourceSha"],
859+
"REWRITE_ARTIFACT_SHA256": data["variantB"]["artifactSha256"],
860+
"DRIVER_SHA256": data["driverSha256"],
861+
"CANONICAL_CONFIG_SHA256": data["canonicalConfigSha256"],
862+
"CLIENT_MANIFEST_SHA256": data["protocolClientManifestSha256"],
863+
"RUNNER_SHA256": data["runnerSha256"],
864+
"JVM_ARGUMENTS_SHA256": data["jvmArgumentsSha256"],
865+
"CAMPAIGN_STACK_SHA256": data["stackSha256"],
866+
"AVAILABLE_CPU_COUNT": cpu["availableCpuCount"],
867+
"SERVER_CPU_SET": ",".join(map(str, cpu["serverCpuSet"])),
868+
"CLIENT_CPU_SET": cpu["clientCpuSet"][0],
869+
}
870+
with open(output, "a", encoding="utf-8") as stream:
871+
for key, value in values.items():
872+
stream.write(f"{key}={value}\n")
873+
PY
874+
721875
- name: Validate campaign and analyze MSPT and TPS
722876
run: |
723877
set -euo pipefail
@@ -860,12 +1014,14 @@ jobs:
8601014
pwsh -NoProfile -File tools/perf/analyze-phase2-abba.ps1 "$manifest" \
8611015
-Scenario "$CAMPAIGN_SCENARIO" -Metric "$metric" \
8621016
-Direction LowerIsBetter -MinimumSeconds "$minimum_seconds" \
863-
"${incomplete[@]}" -OutputJson "$result_root/$metric.analysis.json" -Overwrite
1017+
-IndependentSamples "${incomplete[@]}" \
1018+
-OutputJson "$result_root/$metric.analysis.json" -Overwrite
8641019
done
8651020
pwsh -NoProfile -File tools/perf/analyze-phase2-abba.ps1 "$manifest" \
8661021
-Scenario "$CAMPAIGN_SCENARIO" -Metric observedTps \
8671022
-Direction HigherIsBetter -MinimumSeconds "$minimum_seconds" \
868-
"${incomplete[@]}" -OutputJson "$result_root/observedTps.analysis.json" -Overwrite
1023+
-IndependentSamples "${incomplete[@]}" \
1024+
-OutputJson "$result_root/observedTps.analysis.json" -Overwrite
8691025
8701026
- name: Publish comparison summary
8711027
if: success()
@@ -910,6 +1066,9 @@ jobs:
9101066
result = results[0]
9111067
if result.get("scenario") != scenario or result.get("metric") != metric:
9121068
raise SystemExit(f"{metric} analysis scenario/metric mismatch")
1069+
if (result.get("samplingMode") != "independent-runners"
1070+
or result.get("pairCount") != 0):
1071+
raise SystemExit(f"{metric} analysis did not use independent runners")
9131072
analyses[metric] = result
9141073
9151074
formal = kind == "formal"
@@ -951,7 +1110,8 @@ jobs:
9511110
lines = [
9521111
f"### Upstream runtime comparison: `{scenario}`",
9531112
"",
954-
f"Mode: `{kind}`; scene size: `{scene_size}`; restart-isolated runs: `{runs}`.",
1113+
f"Mode: `{kind}`; scene size: `{scene_size}`; fully parallel, "
1114+
f"restart-isolated independent runs: `{runs}`.",
9551115
f"Runtime profile: `{runtime_profile}`; shared config requests "
9561116
"`PacketOnlyStatic=true` and `EventDriven=true` for both artifacts.",
9571117
"Runtime assertion: A reports both flags as `unsupported-legacy`; "
@@ -1026,6 +1186,7 @@ jobs:
10261186
"passed": scenario_passed if formal else None,
10271187
"conclusion": conclusion,
10281188
"formalComplete": formal,
1189+
"samplingMode": "independent-runners",
10291190
"registeredGates": {
10301191
"primaryMeanImprovement": {
10311192
"medianBRatioToAMaximum": 0.90,
@@ -1102,8 +1263,8 @@ jobs:
11021263

11031264
summarize-upstream-runtime:
11041265
name: Global upstream runtime verdict
1105-
needs: [compare-upstream-runtime]
1106-
if: ${{ needs.compare-upstream-runtime.result == 'success' }}
1266+
needs: [aggregate-upstream-runtime]
1267+
if: ${{ needs.aggregate-upstream-runtime.result == 'success' }}
11071268
runs-on: ubuntu-latest
11081269
timeout-minutes: 10
11091270
env:
@@ -1157,6 +1318,8 @@ jobs:
11571318
raise SystemExit(f"{scenario} smoke verdict must not declare a pass state")
11581319
if document.get("runtimeProfile") != "optimized-candidate":
11591320
raise SystemExit(f"{scenario} runtime profile mismatch")
1321+
if document.get("samplingMode") != "independent-runners":
1322+
raise SystemExit(f"{scenario} sampling mode mismatch")
11601323
if document.get("requestedFlags") != {
11611324
"packetOnlyStatic": True,
11621325
"eventDrivenBlockUpdates": True,

0 commit comments

Comments
 (0)