Skip to content

Commit dd35a49

Browse files
Arbousier1Arbousier1
committed
Benchmark optimized runtime profile
1 parent c28db01 commit dd35a49

5 files changed

Lines changed: 433 additions & 27 deletions

File tree

‎.github/workflows/upstream-runtime-comparison.yml‎

Lines changed: 142 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,10 @@ jobs:
3434
UPSTREAM_SOURCE_SHA: c7f9dd0457451537653bf4b4c0eb0e4298c51187
3535
UPSTREAM_ARTIFACT_SIZE: "5799385"
3636
UPSTREAM_ARTIFACT_SHA256: a7ffc2ba053c74681feabc698e9fdb959ebd4f8252206fedd8801979e3de30c0
37-
PRODUCTION_CANDIDATE_SHA: b3c245386304c809e0d40e0530300872aec343f5
37+
PRODUCTION_CANDIDATE_SHA: c28db0146ec2f35eaf066b4202b33f08d06bbc4b
3838
PAPER_BUILD: "74"
3939
PAPER_SHA256: 1d70b1dab9cf4a6de615209a536f3a45a2186240253c428213ce2188ab95e5f7
40+
COMPARE_RUNTIME_PROFILE: optimized-candidate
4041
CAMPAIGN_KIND: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && 'formal' || 'smoke' }}
4142
CAMPAIGN_RUNS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '12' || '4' }}
4243
CAMPAIGN_WARMUP_SECONDS: ${{ (github.event_name == 'workflow_dispatch' || (github.event.action == 'labeled' && github.event.label.name == 'upstream-runtime-formal')) && '60' || '10' }}
@@ -98,6 +99,7 @@ jobs:
9899
- name: Select candidate artifacts and create canonical configuration
99100
run: |
100101
set -euo pipefail
102+
[[ "$COMPARE_RUNTIME_PROFILE" == optimized-candidate ]]
101103
mkdir -p compare-dependencies
102104
mapfile -t candidate_jars < <(find "$PRODUCTION_WORKTREE/build/libs" -maxdepth 1 -type f \
103105
-name 'InteractionVisualizer-*.jar' \
@@ -122,6 +124,7 @@ jobs:
122124
> compare-dependencies/canonical-config.yml
123125
python3 - compare-dependencies/canonical-config.yml <<'PY'
124126
from pathlib import Path
127+
import os
125128
import re
126129
import sys
127130
@@ -132,6 +135,15 @@ jobs:
132135
text, count = re.subn(pattern, rf"\1false", text)
133136
if count != 1:
134137
raise SystemExit(f"Expected exactly one true {key} setting, found {count}")
138+
runtime_profile = os.environ["COMPARE_RUNTIME_PROFILE"]
139+
if runtime_profile not in {"legacy-parity", "optimized-candidate"}:
140+
raise SystemExit(f"Unsupported runtime profile: {runtime_profile!r}")
141+
requested_value = "true" if runtime_profile == "optimized-candidate" else "false"
142+
for key in ("PacketOnlyStatic", "EventDriven"):
143+
pattern = rf"(?m)^(\s*{key}:\s*)(?:true|false)\s*$"
144+
text, count = re.subn(pattern, rf"\g<1>{requested_value}", text)
145+
if count != 1:
146+
raise SystemExit(f"Expected exactly one {key} setting, found {count}")
135147
path.write_text(text, encoding="utf-8", newline="\n")
136148
PY
137149
test -s compare-dependencies/canonical-config.yml
@@ -274,6 +286,10 @@ jobs:
274286
"availableCpuCount=$available_cpu_count" \
275287
"serverCpuSet=$server_cpu_set" \
276288
"clientCpuSet=$client_cpu_set"
289+
printf '%s\n' \
290+
"runtimeProfile=$COMPARE_RUNTIME_PROFILE" \
291+
'requestedPacketOnlyStatic=true' \
292+
'requestedEventDrivenBlockUpdates=true'
277293
printf '%s\n' "harnessSourceSha=$harness_source_sha"
278294
} | sha256sum | awk '{print $1}'
279295
)
@@ -303,7 +319,7 @@ jobs:
303319
"$UPSTREAM_BUILD_NUMBER" "$UPSTREAM_SOURCE_SHA" \
304320
"$UPSTREAM_ARTIFACT_SHA256" "$PAPER_BUILD" "$PAPER_SHA256" \
305321
"$available_cpu_count" "$server_cpu_set" "$client_cpu_set" \
306-
"$harness_source_sha" <<'PY'
322+
"$harness_source_sha" "$COMPARE_RUNTIME_PROFILE" <<'PY'
307323
from pathlib import Path
308324
import json
309325
import sys
@@ -314,7 +330,7 @@ jobs:
314330
jvm_sha, stack_sha, scenario, scene_size, kind, runs, warmup,
315331
settle, measure, upstream_build, upstream_source, upstream_sha,
316332
paper_build, paper_sha, available_cpu_count, server_cpu_set,
317-
client_cpu_set, harness_source_sha,
333+
client_cpu_set, harness_source_sha, runtime_profile,
318334
) = sys.argv[1:]
319335
Path(output).write_text(json.dumps({
320336
"schemaVersion": 1,
@@ -325,6 +341,11 @@ jobs:
325341
"warmupSeconds": int(warmup),
326342
"settleSeconds": int(settle),
327343
"measureSeconds": int(measure),
344+
"runtimeProfile": runtime_profile,
345+
"requestedFlags": {
346+
"packetOnlyStatic": runtime_profile == "optimized-candidate",
347+
"eventDrivenBlockUpdates": runtime_profile == "optimized-candidate",
348+
},
328349
"preflight": {"runs": 2, "warmupSeconds": 10, "settleSeconds": 5,
329350
"measureSeconds": 10, "protocolTraceEnabled": True},
330351
"variantA": {"meaning": "official-upstream", "jenkinsBuild": int(upstream_build),
@@ -382,6 +403,7 @@ jobs:
382403
COMPARE_RUN_ID="$run_id" \
383404
COMPARE_SCENARIO="$CAMPAIGN_SCENARIO" \
384405
COMPARE_VARIANT="$variant" \
406+
COMPARE_RUNTIME_PROFILE="$COMPARE_RUNTIME_PROFILE" \
385407
COMPARE_SCENE_SIZE="$CAMPAIGN_SCENE_SIZE" \
386408
COMPARE_WARMUP_SECONDS=10 \
387409
COMPARE_SETTLE_SECONDS=5 \
@@ -433,10 +455,36 @@ jobs:
433455
for field, expected in expected_metrics.items():
434456
if metrics.get(field) != expected:
435457
raise SystemExit(f"preflight metrics mismatch {field}: {metrics.get(field)!r} != {expected!r}")
458+
expected_requested_flags = {
459+
"packetOnlyStatic": True,
460+
"eventDrivenBlockUpdates": True,
461+
}
462+
expected_effective_flags = {
463+
"packetOnlyStatic": {
464+
"status": "unsupported-legacy" if variant == "A" else "runtime-field",
465+
"value": None if variant == "A" else True,
466+
"field": "packetOnlyStaticVirtualItems",
467+
},
468+
"eventDrivenBlockUpdates": {
469+
"status": "unsupported-legacy" if variant == "A" else "runtime-field",
470+
"value": None if variant == "A" else True,
471+
"field": "eventDrivenBlockUpdates",
472+
},
473+
}
474+
if metrics.get("requestedFlags") != expected_requested_flags:
475+
raise SystemExit("preflight requested optimization flags mismatch")
476+
if metrics.get("effectiveFlags") != expected_effective_flags:
477+
raise SystemExit("preflight effective optimization flags mismatch")
436478
if manifest.get("artifactSha256") != artifact_sha:
437479
raise SystemExit("preflight artifact SHA mismatch")
438480
if manifest.get("canonicalConfigSha256") != config_sha:
439481
raise SystemExit("preflight canonical config SHA mismatch")
482+
if manifest.get("runtimeProfile") != "optimized-candidate":
483+
raise SystemExit("preflight runtime profile mismatch")
484+
if manifest.get("requestedFlags") != expected_requested_flags:
485+
raise SystemExit("preflight manifest requested flags mismatch")
486+
if manifest.get("effectiveFlags") != expected_effective_flags:
487+
raise SystemExit("preflight manifest effective flags mismatch")
440488
expected_packet_allowlist = [
441489
"bundle_delimiter", "entity_destroy", "spawn_entity"
442490
]
@@ -496,10 +544,6 @@ jobs:
496544
if scenario == "dropped-items" and variant == "B" and spawn_observations <= scene_size:
497545
raise SystemExit(
498546
"rewritten dropped-item preflight produced no visual spawns beyond source items")
499-
if scenario == "block-active" and spawn_observations < scene_size:
500-
raise SystemExit(
501-
f"block-active preflight observed only {spawn_observations} spawns "
502-
f"for scene size {scene_size}")
503547
if scenario == "block-active":
504548
block_guards = {
505549
"furnaceBlocks": 205,
@@ -513,6 +557,16 @@ jobs:
513557
if metrics.get(field) != expected:
514558
raise SystemExit(
515559
f"block preflight mismatch {field}: {metrics.get(field)!r} != {expected!r}")
560+
minimum_visual_spawns = 2 * (
561+
metrics["beehiveBlocks"] + metrics["beeNestBlocks"]
562+
)
563+
if minimum_visual_spawns != 818:
564+
raise SystemExit(
565+
f"block preflight TextDisplay guard drifted: {minimum_visual_spawns!r}")
566+
if spawn_observations < minimum_visual_spawns:
567+
raise SystemExit(
568+
f"block-active preflight observed only {spawn_observations} spawns; "
569+
f"expected at least {minimum_visual_spawns} for two TextDisplays per bee block")
516570
PY
517571
done
518572
@@ -548,6 +602,7 @@ jobs:
548602
COMPARE_RUN_ID="$run_id" \
549603
COMPARE_SCENARIO="$CAMPAIGN_SCENARIO" \
550604
COMPARE_VARIANT="$variant" \
605+
COMPARE_RUNTIME_PROFILE="$COMPARE_RUNTIME_PROFILE" \
551606
COMPARE_SCENE_SIZE="$CAMPAIGN_SCENE_SIZE" \
552607
COMPARE_WARMUP_SECONDS="$CAMPAIGN_WARMUP_SECONDS" \
553608
COMPARE_SETTLE_SECONDS="$CAMPAIGN_SETTLE_SECONDS" \
@@ -560,7 +615,8 @@ jobs:
560615
"$CAMPAIGN_STACK_SHA256" "$expected_artifact_sha" \
561616
"$CANONICAL_CONFIG_SHA256" "$DRIVER_SHA256" "$PAPER_SHA256" \
562617
"$CLIENT_MANIFEST_SHA256" "$RUNNER_SHA256" "$JVM_ARGUMENTS_SHA256" \
563-
"$AVAILABLE_CPU_COUNT" "$SERVER_CPU_SET" "$CLIENT_CPU_SET" <<'PY'
618+
"$AVAILABLE_CPU_COUNT" "$SERVER_CPU_SET" "$CLIENT_CPU_SET" \
619+
"$COMPARE_RUNTIME_PROFILE" <<'PY'
564620
import csv
565621
import json
566622
from pathlib import Path
@@ -569,7 +625,7 @@ jobs:
569625
(manifest_text, root_text, run_id, scenario, block, position, variant,
570626
scene_size_text, stack_sha, artifact_sha, config_sha, driver_sha,
571627
paper_sha, client_sha, runner_sha, jvm_sha, available_cpu_count,
572-
server_cpu_set, client_cpu_set) = sys.argv[1:]
628+
server_cpu_set, client_cpu_set, runtime_profile) = sys.argv[1:]
573629
root = Path(root_text)
574630
metrics_path = root / run_id / "iv-compare.json"
575631
run_manifest_path = root / run_id / "run-manifest.json"
@@ -597,6 +653,28 @@ jobs:
597653
for field, expected in expected_metrics.items():
598654
if metrics.get(field) != expected:
599655
raise SystemExit(f"metrics mismatch {run_id}/{field}: {metrics.get(field)!r} != {expected!r}")
656+
expected_requested_flags = {
657+
"packetOnlyStatic": True,
658+
"eventDrivenBlockUpdates": True,
659+
}
660+
expected_effective_flags = {
661+
"packetOnlyStatic": {
662+
"status": "unsupported-legacy" if variant == "A" else "runtime-field",
663+
"value": None if variant == "A" else True,
664+
"field": "packetOnlyStaticVirtualItems",
665+
},
666+
"eventDrivenBlockUpdates": {
667+
"status": "unsupported-legacy" if variant == "A" else "runtime-field",
668+
"value": None if variant == "A" else True,
669+
"field": "eventDrivenBlockUpdates",
670+
},
671+
}
672+
if runtime_profile != "optimized-candidate":
673+
raise SystemExit(f"unexpected runtime profile: {runtime_profile!r}")
674+
if metrics.get("requestedFlags") != expected_requested_flags:
675+
raise SystemExit(f"metrics requested flags mismatch for {run_id}")
676+
if metrics.get("effectiveFlags") != expected_effective_flags:
677+
raise SystemExit(f"metrics effective flags mismatch for {run_id}")
600678
expected_manifest = {
601679
"runId": run_id,
602680
"scenario": scenario,
@@ -612,6 +690,9 @@ jobs:
612690
"availableCpuCount": int(available_cpu_count),
613691
"serverCpuSet": expected_server_cpu_set,
614692
"clientCpuSet": expected_client_cpu_set,
693+
"runtimeProfile": runtime_profile,
694+
"requestedFlags": expected_requested_flags,
695+
"effectiveFlags": expected_effective_flags,
615696
}
616697
for field, expected in expected_manifest.items():
617698
if run_manifest.get(field) != expected:
@@ -646,15 +727,16 @@ jobs:
646727
"$CAMPAIGN_SCENARIO" "$CAMPAIGN_SCENE_SIZE" \
647728
"$CAMPAIGN_STACK_SHA256" "$UPSTREAM_ARTIFACT_SHA256" \
648729
"$REWRITE_ARTIFACT_SHA256" "$CANONICAL_CONFIG_SHA256" \
649-
"$AVAILABLE_CPU_COUNT" "$SERVER_CPU_SET" "$CLIENT_CPU_SET" <<'PY'
730+
"$AVAILABLE_CPU_COUNT" "$SERVER_CPU_SET" "$CLIENT_CPU_SET" \
731+
"$COMPARE_RUNTIME_PROFILE" <<'PY'
650732
import csv
651733
import json
652734
from pathlib import Path
653735
import sys
654736
655737
(manifest_text, root_text, runs_text, scenario, scene_size_text,
656738
stack_sha, upstream_sha, rewrite_sha, config_sha, available_cpu_count,
657-
server_cpu_set, client_cpu_set) = sys.argv[1:]
739+
server_cpu_set, client_cpu_set, runtime_profile) = sys.argv[1:]
658740
expected_runs = int(runs_text)
659741
scene_size = int(scene_size_text)
660742
root = Path(root_text).resolve()
@@ -713,6 +795,33 @@ jobs:
713795
if metrics.get(field) != expected:
714796
raise SystemExit(
715797
f"final gate mismatch {row['RunId']}/{field}: {metrics.get(field)!r} != {expected!r}")
798+
expected_requested_flags = {
799+
"packetOnlyStatic": True,
800+
"eventDrivenBlockUpdates": True,
801+
}
802+
expected_effective_flags = {
803+
"packetOnlyStatic": {
804+
"status": "unsupported-legacy" if variant == "A" else "runtime-field",
805+
"value": None if variant == "A" else True,
806+
"field": "packetOnlyStaticVirtualItems",
807+
},
808+
"eventDrivenBlockUpdates": {
809+
"status": "unsupported-legacy" if variant == "A" else "runtime-field",
810+
"value": None if variant == "A" else True,
811+
"field": "eventDrivenBlockUpdates",
812+
},
813+
}
814+
if runtime_profile != "optimized-candidate":
815+
raise SystemExit(f"unexpected runtime profile: {runtime_profile!r}")
816+
for document_name, document in (("metrics", metrics), ("manifest", run_manifest)):
817+
if document.get("requestedFlags") != expected_requested_flags:
818+
raise SystemExit(
819+
f"{document_name} requested flags drifted in {row['RunId']}")
820+
if document.get("effectiveFlags") != expected_effective_flags:
821+
raise SystemExit(
822+
f"{document_name} effective flags drifted in {row['RunId']}")
823+
if run_manifest.get("runtimeProfile") != runtime_profile:
824+
raise SystemExit(f"runtime profile drifted in {row['RunId']}")
716825
if scenario == "block-active":
717826
block_guards = {
718827
"furnaceBlocks": 205,
@@ -767,7 +876,8 @@ jobs:
767876
"$CAMPAIGN_SCENE_SIZE" "$CAMPAIGN_KIND" "$CAMPAIGN_RUNS" \
768877
"$UPSTREAM_SOURCE_SHA" "$UPSTREAM_ARTIFACT_SHA256" \
769878
"$CANDIDATE_SOURCE_SHA" "$REWRITE_ARTIFACT_SHA256" \
770-
"$PAPER_SHA256" "$CAMPAIGN_STACK_SHA256" <<'PY'
879+
"$PAPER_SHA256" "$CAMPAIGN_STACK_SHA256" \
880+
"$COMPARE_RUNTIME_PROFILE" <<'PY'
771881
import csv
772882
import json
773883
from pathlib import Path
@@ -776,7 +886,7 @@ jobs:
776886
777887
(root_text, summary_text, scenario, scene_size, kind, runs,
778888
upstream_source, upstream_artifact, candidate_source, candidate_artifact,
779-
paper_sha, stack_sha) = sys.argv[1:]
889+
paper_sha, stack_sha, runtime_profile) = sys.argv[1:]
780890
root = Path(root_text)
781891
rows = list(csv.DictReader((root / "abba-manifest.csv").open(encoding="utf-8")))
782892
metrics_by_variant = {"A": [], "B": []}
@@ -837,8 +947,13 @@ jobs:
837947
f"### Upstream runtime comparison: `{scenario}`",
838948
"",
839949
f"Mode: `{kind}`; scene size: `{scene_size}`; restart-isolated runs: `{runs}`.",
950+
f"Runtime profile: `{runtime_profile}`; shared config requests "
951+
"`PacketOnlyStatic=true` and `EventDriven=true` for both artifacts.",
952+
"Runtime assertion: A reports both flags as `unsupported-legacy`; "
953+
"B exposes both runtime fields as `true`.",
840954
"",
841-
"A is official upstream Jenkins #163; B is production candidate `b3c2453`. ",
955+
f"A is official upstream Jenkins #163; B is production candidate "
956+
f"`{candidate_source[:7]}`. ",
842957
"Smoke evidence is exploratory and never declares a winner.",
843958
"",
844959
"| Metric | Upstream median | Rewrite median | Median B/A | Ratio 95% CI | Registered use |",
@@ -925,6 +1040,11 @@ jobs:
9251040
"candidateArtifactSha256": candidate_artifact,
9261041
"paperSha256": paper_sha,
9271042
"stackSha256": stack_sha,
1043+
"runtimeProfile": runtime_profile,
1044+
"requestedFlags": {
1045+
"packetOnlyStatic": True,
1046+
"eventDrivenBlockUpdates": True,
1047+
},
9281048
}
9291049
(root / "scenario-verdict.json").write_text(
9301050
json.dumps(verdict, indent=2) + "\n", encoding="utf-8")
@@ -1005,6 +1125,13 @@ jobs:
10051125
raise SystemExit(f"{scenario} formal verdict lacks a boolean pass state")
10061126
if kind == "smoke" and document.get("passed") is not None:
10071127
raise SystemExit(f"{scenario} smoke verdict must not declare a pass state")
1128+
if document.get("runtimeProfile") != "optimized-candidate":
1129+
raise SystemExit(f"{scenario} runtime profile mismatch")
1130+
if document.get("requestedFlags") != {
1131+
"packetOnlyStatic": True,
1132+
"eventDrivenBlockUpdates": True,
1133+
}:
1134+
raise SystemExit(f"{scenario} requested flags mismatch")
10081135
verdicts[scenario] = document
10091136
10101137
if kind == "smoke":
@@ -1024,6 +1151,7 @@ jobs:
10241151
"passed": global_passed,
10251152
"conclusion": conclusion,
10261153
"requiredScenarios": ["dropped-items", "block-active"],
1154+
"runtimeProfile": "optimized-candidate",
10271155
"scenarioVerdicts": {
10281156
scenario: {
10291157
"passed": verdict["passed"],

0 commit comments

Comments
 (0)