Skip to content

Commit 5da7468

Browse files
Arbousier1Arbousier1
committed
Bound saturated benchmark shutdown
1 parent 45dcfc2 commit 5da7468

2 files changed

Lines changed: 144 additions & 22 deletions

File tree

‎.github/workflows/upstream-runtime-comparison.yml‎

Lines changed: 81 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,7 @@ jobs:
554554
COMPARE_SCENARIO="$CAMPAIGN_SCENARIO" \
555555
COMPARE_VARIANT="$variant" \
556556
COMPARE_RUNTIME_PROFILE="$COMPARE_RUNTIME_PROFILE" \
557+
COMPARE_CAMPAIGN_KIND=preflight \
557558
COMPARE_SCENE_SIZE="$CAMPAIGN_SCENE_SIZE" \
558559
COMPARE_WARMUP_SECONDS=10 \
559560
COMPARE_SETTLE_SECONDS=5 \
@@ -631,6 +632,17 @@ jobs:
631632
raise SystemExit("preflight canonical config SHA mismatch")
632633
if manifest.get("runtimeProfile") != "optimized-candidate":
633634
raise SystemExit("preflight runtime profile mismatch")
635+
if manifest.get("campaignKind") != "preflight":
636+
raise SystemExit("preflight campaign kind mismatch")
637+
expected_shutdown = {
638+
"mode": "clean",
639+
"stopGraceSeconds": 120,
640+
"termGraceSeconds": 10,
641+
"forcedAllowed": False,
642+
}
643+
if manifest.get("serverShutdown") != expected_shutdown:
644+
raise SystemExit(
645+
f"preflight server shutdown mismatch: {manifest.get('serverShutdown')!r}")
634646
if manifest.get("requestedFlags") != expected_requested_flags:
635647
raise SystemExit("preflight manifest requested flags mismatch")
636648
if manifest.get("effectiveFlags") != expected_effective_flags:
@@ -753,6 +765,7 @@ jobs:
753765
COMPARE_SCENARIO="$CAMPAIGN_SCENARIO" \
754766
COMPARE_VARIANT="$variant" \
755767
COMPARE_RUNTIME_PROFILE="$COMPARE_RUNTIME_PROFILE" \
768+
COMPARE_CAMPAIGN_KIND="$CAMPAIGN_KIND" \
756769
COMPARE_SCENE_SIZE="$CAMPAIGN_SCENE_SIZE" \
757770
COMPARE_WARMUP_SECONDS="$CAMPAIGN_WARMUP_SECONDS" \
758771
COMPARE_SETTLE_SECONDS="$CAMPAIGN_SETTLE_SECONDS" \
@@ -766,7 +779,7 @@ jobs:
766779
"$CANONICAL_CONFIG_SHA256" "$DRIVER_SHA256" "$PAPER_SHA256" \
767780
"$CLIENT_MANIFEST_SHA256" "$RUNNER_SHA256" "$JVM_ARGUMENTS_SHA256" \
768781
"$AVAILABLE_CPU_COUNT" "$SERVER_CPU_SET" "$CLIENT_CPU_SET" \
769-
"$COMPARE_RUNTIME_PROFILE" <<'PY'
782+
"$COMPARE_RUNTIME_PROFILE" "$CAMPAIGN_KIND" <<'PY'
770783
import csv
771784
import json
772785
from pathlib import Path
@@ -775,7 +788,7 @@ jobs:
775788
(manifest_text, root_text, run_id, scenario, block, position, variant,
776789
scene_size_text, stack_sha, artifact_sha, config_sha, driver_sha,
777790
paper_sha, client_sha, runner_sha, jvm_sha, available_cpu_count,
778-
server_cpu_set, client_cpu_set, runtime_profile) = sys.argv[1:]
791+
server_cpu_set, client_cpu_set, runtime_profile, campaign_kind) = sys.argv[1:]
779792
root = Path(root_text)
780793
metrics_path = root / run_id / "iv-compare.json"
781794
run_manifest_path = root / run_id / "run-manifest.json"
@@ -841,13 +854,33 @@ jobs:
841854
"serverCpuSet": expected_server_cpu_set,
842855
"clientCpuSet": expected_client_cpu_set,
843856
"runtimeProfile": runtime_profile,
857+
"campaignKind": campaign_kind,
844858
"requestedFlags": expected_requested_flags,
845859
"effectiveFlags": expected_effective_flags,
846860
}
847861
for field, expected in expected_manifest.items():
848862
if run_manifest.get(field) != expected:
849863
raise SystemExit(
850864
f"run manifest mismatch {run_id}/{field}: {run_manifest.get(field)!r} != {expected!r}")
865+
expected_forced_shutdown = (
866+
campaign_kind == "formal" and scenario == "block-active" and variant == "A")
867+
expected_shutdown = {
868+
"stopGraceSeconds": 120,
869+
"termGraceSeconds": 10,
870+
"forcedAllowed": expected_forced_shutdown,
871+
}
872+
shutdown = run_manifest.get("serverShutdown")
873+
if not isinstance(shutdown, dict):
874+
raise SystemExit(f"server shutdown evidence missing for {run_id}")
875+
for field, expected in expected_shutdown.items():
876+
if shutdown.get(field) != expected:
877+
raise SystemExit(
878+
f"server shutdown mismatch {run_id}/{field}: "
879+
f"{shutdown.get(field)!r} != {expected!r}")
880+
if shutdown.get("mode") not in {"clean", "forced-after-stop-timeout"}:
881+
raise SystemExit(f"invalid server shutdown mode for {run_id}: {shutdown.get('mode')!r}")
882+
if shutdown.get("mode") != "clean" and not expected_forced_shutdown:
883+
raise SystemExit(f"unexpected forced server shutdown for {run_id}")
851884
expected_affinity = {
852885
"availableCpuCount": int(available_cpu_count),
853886
"serverCpuSet": expected_server_cpu_set,
@@ -1033,15 +1066,15 @@ jobs:
10331066
"$CAMPAIGN_STACK_SHA256" "$UPSTREAM_ARTIFACT_SHA256" \
10341067
"$REWRITE_ARTIFACT_SHA256" "$CANONICAL_CONFIG_SHA256" \
10351068
"$AVAILABLE_CPU_COUNT" "$SERVER_CPU_SET" "$CLIENT_CPU_SET" \
1036-
"$COMPARE_RUNTIME_PROFILE" <<'PY'
1069+
"$COMPARE_RUNTIME_PROFILE" "$CAMPAIGN_KIND" <<'PY'
10371070
import csv
10381071
import json
10391072
from pathlib import Path
10401073
import sys
10411074
10421075
(manifest_text, root_text, runs_text, scenario, scene_size_text,
10431076
stack_sha, upstream_sha, rewrite_sha, config_sha, available_cpu_count,
1044-
server_cpu_set, client_cpu_set, runtime_profile) = sys.argv[1:]
1077+
server_cpu_set, client_cpu_set, runtime_profile, kind) = sys.argv[1:]
10451078
expected_runs = int(runs_text)
10461079
scene_size = int(scene_size_text)
10471080
root = Path(root_text).resolve()
@@ -1129,6 +1162,28 @@ jobs:
11291162
raise SystemExit(f"runtime profile drifted in {row['RunId']}")
11301163
if run_manifest.get("networkTimeoutSeconds") != 300:
11311164
raise SystemExit(f"network timeout drifted in {row['RunId']}")
1165+
if run_manifest.get("campaignKind") != kind:
1166+
raise SystemExit(f"campaign kind drifted in {row['RunId']}")
1167+
expected_forced_shutdown = (
1168+
kind == "formal" and scenario == "block-active" and variant == "A")
1169+
expected_shutdown = {
1170+
"stopGraceSeconds": 120,
1171+
"termGraceSeconds": 10,
1172+
"forcedAllowed": expected_forced_shutdown,
1173+
}
1174+
shutdown = run_manifest.get("serverShutdown")
1175+
if not isinstance(shutdown, dict):
1176+
raise SystemExit(f"server shutdown evidence missing in {row['RunId']}")
1177+
for field, expected in expected_shutdown.items():
1178+
if shutdown.get(field) != expected:
1179+
raise SystemExit(
1180+
f"server shutdown drift {row['RunId']}/{field}: "
1181+
f"{shutdown.get(field)!r} != {expected!r}")
1182+
if shutdown.get("mode") not in {"clean", "forced-after-stop-timeout"}:
1183+
raise SystemExit(
1184+
f"invalid server shutdown mode in {row['RunId']}: {shutdown.get('mode')!r}")
1185+
if shutdown.get("mode") != "clean" and not expected_forced_shutdown:
1186+
raise SystemExit(f"unexpected forced server shutdown in {row['RunId']}")
11321187
if scenario == "block-active":
11331188
block_guards = {
11341189
"furnaceBlocks": 205,
@@ -1199,9 +1254,22 @@ jobs:
11991254
root = Path(root_text)
12001255
rows = list(csv.DictReader((root / "abba-manifest.csv").open(encoding="utf-8")))
12011256
metrics_by_variant = {"A": [], "B": []}
1257+
shutdown_modes_by_variant = {"A": [], "B": []}
12021258
for row in rows:
12031259
metrics = json.loads((root / row["SourcePath"]).read_text(encoding="utf-8"))
12041260
metrics_by_variant[row["Variant"]].append(metrics)
1261+
run_manifest = json.loads(
1262+
(root / row["SourcePath"]).with_name("run-manifest.json").read_text(
1263+
encoding="utf-8"))
1264+
shutdown_modes_by_variant[row["Variant"]].append(
1265+
run_manifest["serverShutdown"]["mode"])
1266+
shutdown_mode_counts = {
1267+
variant: {
1268+
mode: modes.count(mode)
1269+
for mode in ("clean", "forced-after-stop-timeout")
1270+
}
1271+
for variant, modes in shutdown_modes_by_variant.items()
1272+
}
12051273
metric_specs = [
12061274
("msptMean", "ms"),
12071275
("msptP95", "ms"),
@@ -1269,6 +1337,11 @@ jobs:
12691337
"`PacketOnlyStatic=true` and `EventDriven=true` for both artifacts.",
12701338
"Runtime assertion: A reports both flags as `unsupported-legacy`; "
12711339
"B exposes both runtime fields as `true`.",
1340+
("Server shutdown evidence: "
1341+
f"A clean `{shutdown_mode_counts['A']['clean']}`, forced "
1342+
f"`{shutdown_mode_counts['A']['forced-after-stop-timeout']}`; "
1343+
f"B clean `{shutdown_mode_counts['B']['clean']}`, forced "
1344+
f"`{shutdown_mode_counts['B']['forced-after-stop-timeout']}`."),
12721345
"",
12731346
f"A is official upstream Jenkins #163; B is production candidate "
12741347
f"`{candidate_source[:7]}`. ",
@@ -1371,6 +1444,10 @@ jobs:
13711444
100 if formal and scenario == "block-active" else None
13721445
),
13731446
},
1447+
"serverShutdown": {
1448+
"forcedTerminationAllowedOnlyForFormalBlockActiveUpstream": True,
1449+
"modeCountsByVariant": shutdown_mode_counts,
1450+
},
13741451
"metrics": {
13751452
metric: {
13761453
"medianBRatioToA": float(result["medianBRatioToA"]),

‎tools/perf/run-upstream-runtime-once.sh‎

Lines changed: 63 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ for variable in \
99
COMPARE_PLUGIN_JAR COMPARE_DRIVER_JAR COMPARE_PAPER_JAR COMPARE_CONFIG_FILE \
1010
COMPARE_CLIENT_ROOT \
1111
COMPARE_OUTPUT_ROOT COMPARE_RUN_ID COMPARE_SCENARIO COMPARE_VARIANT \
12-
COMPARE_RUNTIME_PROFILE \
12+
COMPARE_RUNTIME_PROFILE COMPARE_CAMPAIGN_KIND \
1313
COMPARE_SCENE_SIZE COMPARE_WARMUP_SECONDS COMPARE_SETTLE_SECONDS \
1414
COMPARE_MEASURE_SECONDS; do
1515
[[ -n "${!variable:-}" ]] || { echo "$variable is required" >&2; exit 64; }
@@ -25,6 +25,7 @@ run_id="$COMPARE_RUN_ID"
2525
scenario="$COMPARE_SCENARIO"
2626
variant="$COMPARE_VARIANT"
2727
runtime_profile="$COMPARE_RUNTIME_PROFILE"
28+
campaign_kind="$COMPARE_CAMPAIGN_KIND"
2829
scene_size="$COMPARE_SCENE_SIZE"
2930
warmup_seconds="$COMPARE_WARMUP_SECONDS"
3031
settle_seconds="$COMPARE_SETTLE_SECONDS"
@@ -49,6 +50,8 @@ case "$runtime_profile" in
4950
esac
5051
[[ "$scenario" == dropped-items || "$scenario" == block-active ]] \
5152
|| { echo "Unsupported comparison scenario: $scenario" >&2; exit 64; }
53+
[[ "$campaign_kind" == preflight || "$campaign_kind" == smoke || "$campaign_kind" == formal ]] \
54+
|| { echo "COMPARE_CAMPAIGN_KIND must be preflight, smoke, or formal" >&2; exit 64; }
5255
[[ "$protocol_trace_enabled" == 0 || "$protocol_trace_enabled" == 1 ]] \
5356
|| { echo "COMPARE_PROTOCOL_TRACE_ENABLED must be 0 or 1" >&2; exit 64; }
5457
[[ "$protocol_trace_max_events" =~ ^[0-9]+$ ]] \
@@ -262,6 +265,27 @@ client_pid=""
262265
console_open=0
263266
cleanup_complete=0
264267

268+
wait_for_pid_exit() {
269+
local pid="$1"
270+
local timeout_seconds="$2"
271+
for _ in $(seq 1 "$timeout_seconds"); do
272+
kill -0 "$pid" 2>/dev/null || return 0
273+
sleep 1
274+
done
275+
! kill -0 "$pid" 2>/dev/null
276+
}
277+
278+
terminate_pid_bounded() {
279+
local pid="$1"
280+
local term_grace_seconds="$2"
281+
kill -0 "$pid" 2>/dev/null || return 0
282+
kill -TERM "$pid" 2>/dev/null || true
283+
if ! wait_for_pid_exit "$pid" "$term_grace_seconds"; then
284+
kill -KILL "$pid" 2>/dev/null || true
285+
fi
286+
wait "$pid" 2>/dev/null || true
287+
}
288+
265289
prune_runtime_payload() {
266290
rm -rf -- \
267291
"$run_directory/cache" \
@@ -286,19 +310,13 @@ cleanup() {
286310
cleanup_complete=1
287311
set +e
288312
if [[ -n "$client_pid" ]] && kill -0 "$client_pid" 2>/dev/null; then
289-
kill -TERM "$client_pid"
290-
wait "$client_pid"
313+
terminate_pid_bounded "$client_pid" 5
291314
fi
292315
if [[ "$console_open" == 1 ]]; then
293316
printf 'stop\n' >&3
294317
fi
295318
if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then
296-
for _ in $(seq 1 60); do
297-
kill -0 "$server_pid" 2>/dev/null || break
298-
sleep 1
299-
done
300-
kill -TERM "$server_pid" 2>/dev/null || true
301-
wait "$server_pid" 2>/dev/null || true
319+
wait_for_pid_exit "$server_pid" 60 || terminate_pid_bounded "$server_pid" 10
302320
fi
303321
if [[ "$console_open" == 1 ]]; then
304322
exec 3>&-
@@ -648,7 +666,7 @@ python3 - "$run_directory/run-manifest.json" "$run_id" "$scenario" "$variant" \
648666
"$protocol_trace_aggregate_packet_allowlist" \
649667
"$trace_window_start_epoch_ms" "$trace_window_end_epoch_ms" \
650668
"$available_cpu_count" "$server_cpu_set" "$client_cpu_set" \
651-
"$runtime_profile" "$network_timeout_seconds" "$metrics_path" <<'PY'
669+
"$runtime_profile" "$campaign_kind" "$network_timeout_seconds" "$metrics_path" <<'PY'
652670
from pathlib import Path
653671
import json
654672
import sys
@@ -660,7 +678,7 @@ import sys
660678
trace_enabled, trace_packet_allowlist, trace_aggregate_packet_allowlist,
661679
trace_window_start_epoch_ms, trace_window_end_epoch_ms,
662680
available_cpu_count, server_cpu_set, client_cpu_set,
663-
runtime_profile, network_timeout_seconds, metrics_path,
681+
runtime_profile, campaign_kind, network_timeout_seconds, metrics_path,
664682
) = sys.argv[1:]
665683
metrics = json.load(open(metrics_path, encoding="utf-8"))
666684
Path(output).write_text(json.dumps({
@@ -670,6 +688,7 @@ Path(output).write_text(json.dumps({
670688
"variant": variant,
671689
"variantMeaning": "official-upstream" if variant == "A" else "rewritten-candidate",
672690
"runtimeProfile": runtime_profile,
691+
"campaignKind": campaign_kind,
673692
"networkTimeoutSeconds": int(network_timeout_seconds),
674693
"requestedFlags": metrics["requestedFlags"],
675694
"effectiveFlags": metrics["effectiveFlags"],
@@ -815,18 +834,44 @@ PY
815834
fi
816835

817836
send_console "stop"
818-
for _ in $(seq 1 120); do
819-
kill -0 "$server_pid" 2>/dev/null || break
820-
sleep 1
821-
done
837+
server_shutdown_mode=clean
838+
server_shutdown_forced_allowed=false
839+
if [[ "$campaign_kind" == formal && "$scenario" == block-active && "$variant" == A ]]; then
840+
server_shutdown_forced_allowed=true
841+
fi
822842
if kill -0 "$server_pid" 2>/dev/null; then
823-
echo "Paper did not stop cleanly" >&2
824-
exit 1
843+
if ! wait_for_pid_exit "$server_pid" 120; then
844+
server_shutdown_mode=forced-after-stop-timeout
845+
echo "Paper did not stop within 120 seconds; forcing bounded termination" >&2
846+
terminate_pid_bounded "$server_pid" 10
847+
fi
848+
fi
849+
if [[ "$server_shutdown_mode" == clean ]]; then
850+
wait "$server_pid"
825851
fi
826-
wait "$server_pid"
827852
server_pid=""
828853
exec 3>&-
829854
console_open=0
830855
rm -f -- "$console_fifo"
856+
python3 - "$run_directory/run-manifest.json" "$server_shutdown_mode" \
857+
"$server_shutdown_forced_allowed" <<'PY'
858+
from pathlib import Path
859+
import json
860+
import sys
861+
862+
path, mode, forced_allowed = sys.argv[1:]
863+
data = json.loads(Path(path).read_text(encoding="utf-8"))
864+
data["serverShutdown"] = {
865+
"mode": mode,
866+
"stopGraceSeconds": 120,
867+
"termGraceSeconds": 10,
868+
"forcedAllowed": forced_allowed == "true",
869+
}
870+
Path(path).write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
871+
PY
831872
prune_runtime_payload
832873
cleanup_complete=1
874+
if [[ "$server_shutdown_mode" != clean && "$server_shutdown_forced_allowed" != true ]]; then
875+
echo "Forced Paper termination is not allowed for this comparison sample" >&2
876+
exit 1
877+
fi

0 commit comments

Comments
 (0)