Skip to content

Commit 9756600

Browse files
Arbousier1Arbousier1
committed
Fix dropped-item culling benchmark workload
1 parent a8fd4c9 commit 9756600

6 files changed

Lines changed: 589 additions & 143 deletions

File tree

‎.github/workflows/phase2-runtime-ab.yml‎

Lines changed: 15 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ jobs:
6868
CAMPAIGN_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.scenario || 'static-steady' }}
6969
CAMPAIGN_RUNS: ${{ github.event_name == 'workflow_dispatch' && inputs.runs || github.event.action == 'labeled' && '12' || '4' }}
7070
CAMPAIGN_ITEMS: ${{ github.event_name == 'workflow_dispatch' && inputs.items || github.event.action == 'labeled' && '4096' || '1024' }}
71+
CAMPAIGN_DROPPED_NEARBY_ITEMS: "128"
7172
CAMPAIGN_WARMUP_SECONDS: ${{ github.event_name == 'workflow_dispatch' && inputs.warmup_seconds || github.event.action == 'labeled' && '120' || '10' }}
7273
CAMPAIGN_SETTLE_SECONDS: ${{ github.event_name == 'workflow_dispatch' && inputs.settle_seconds || github.event.action == 'labeled' && '20' || '5' }}
7374
CAMPAIGN_MEASURE_SECONDS: ${{ github.event_name == 'workflow_dispatch' && inputs.measure_seconds || github.event.action == 'labeled' && '180' || '10' }}
@@ -99,6 +100,7 @@ jobs:
99100
bash -n tools/perf/prepare-phase2-protocol-client.sh
100101
bash -n tools/perf/run-phase2-runtime-once.sh
101102
bash tools/perf/run-phase2-runtime-once.sh --self-test
103+
python3 tools/perf/evaluate-dropped-item-gate.py --self-test
102104
node --check tools/perf/phase2-protocol-client.js
103105
pwsh -NoProfile -File tools/perf/analyze-phase2-abba.ps1 -SelfTest
104106
pwsh -NoProfile -File tools/perf/analyze-phase2-pcap.ps1 -SelfTest
@@ -145,6 +147,7 @@ jobs:
145147
SCENARIO: ${{ env.CAMPAIGN_SCENARIO }}
146148
RUNS: ${{ env.CAMPAIGN_RUNS }}
147149
ITEMS: ${{ env.CAMPAIGN_ITEMS }}
150+
DROPPED_NEARBY_ITEMS: ${{ env.CAMPAIGN_DROPPED_NEARBY_ITEMS }}
148151
WARMUP_SECONDS: ${{ env.CAMPAIGN_WARMUP_SECONDS }}
149152
SETTLE_SECONDS: ${{ env.CAMPAIGN_SETTLE_SECONDS }}
150153
MEASURE_SECONDS: ${{ env.CAMPAIGN_MEASURE_SECONDS }}
@@ -160,6 +163,7 @@ jobs:
160163
"$SCENARIO" == dropped-items ]]
161164
[[ "$RUNS" =~ ^(4|8|12)$ ]]
162165
[[ "$ITEMS" =~ ^[0-9]+$ ]]
166+
[[ "$DROPPED_NEARBY_ITEMS" =~ ^[0-9]+$ ]]
163167
if [[ "$SCENARIO" == block-* ]]; then
164168
# TileEntityUpdate.CheckingRange=1 covers the centered 32x32
165169
# block footprint completely; larger command scenes are valid but
@@ -207,6 +211,10 @@ jobs:
207211
echo "dropped-item-section-candidates A/B requires exactly 2048 dropped items" >&2
208212
exit 64
209213
fi
214+
if (( DROPPED_NEARBY_ITEMS != 128 )); then
215+
echo "dropped-item-section-candidates A/B requires exactly 128 nearby labels" >&2
216+
exit 64
217+
fi
210218
elif [[ "$SCENARIO" == dropped-items ]]; then
211219
echo "dropped-items is reserved for dropped-item-section-candidates A/B" >&2
212220
exit 64
@@ -252,9 +260,11 @@ jobs:
252260
sha256sum phase2-dependencies/paper.jar
253261
sha256sum phase2-dependencies/protocol-client/client-build-manifest.json
254262
sha256sum tools/perf/run-phase2-runtime-once.sh
263+
sha256sum tools/perf/evaluate-dropped-item-gate.py
255264
java -version 2>&1
256265
printf '%s\n' \
257-
"abFactor=$AB_FACTOR" "scenario=$SCENARIO" "sparkProfile=$SPARK_PROFILE_MODE"
266+
"abFactor=$AB_FACTOR" "scenario=$SCENARIO" \
267+
"droppedNearbyItems=$DROPPED_NEARBY_ITEMS" "sparkProfile=$SPARK_PROFILE_MODE"
258268
} | sha256sum | awk '{print $1}'
259269
)
260270
@@ -287,6 +297,7 @@ jobs:
287297
PHASE2_VARIANT="$variant" \
288298
PHASE2_AB_FACTOR="$AB_FACTOR" \
289299
PHASE2_ITEM_COUNT="$ITEMS" \
300+
PHASE2_DROPPED_NEARBY_ITEM_COUNT="$DROPPED_NEARBY_ITEMS" \
290301
PHASE2_WARMUP_SECONDS="$WARMUP_SECONDS" \
291302
PHASE2_SETTLE_SECONDS="$SETTLE_SECONDS" \
292303
PHASE2_MEASURE_SECONDS="$MEASURE_SECONDS" \
@@ -413,107 +424,9 @@ jobs:
413424
-MinimumSeconds "$minimum_seconds" "${incomplete[@]}" \
414425
-OutputJson "$EVIDENCE_ROOT/droppedItemMs.analysis.json" -Overwrite
415426
416-
python3 - "$MANIFEST" "$EVIDENCE_ROOT" <<'PY'
417-
import csv
418-
import json
419-
import os
420-
from pathlib import Path
421-
import sys
422-
423-
manifest_path = Path(sys.argv[1])
424-
evidence_root = Path(sys.argv[2])
425-
426-
def load_analysis(name):
427-
return json.loads((evidence_root / f"{name}.analysis.json").read_text(encoding="utf-8"))
428-
429-
dropped = load_analysis("droppedItemMs")
430-
p95 = load_analysis("msptP95")
431-
p99 = load_analysis("msptP99")
432-
checks = {
433-
"droppedItemMedianRatioAtMost0_50": dropped["medianBRatioToA"] <= 0.50,
434-
"droppedItemCiExcludesRegression": dropped["ratioBootstrap95Ci"][1] < 1.0,
435-
"msptP95CiUpperAtMost1_02": p95["ratioBootstrap95Ci"][1] <= 1.02,
436-
"msptP99CiUpperAtMost1_05": p99["ratioBootstrap95Ci"][1] <= 1.05,
437-
}
438-
439-
candidate_runs = []
440-
with manifest_path.open(encoding="utf-8", newline="") as stream:
441-
for row in csv.DictReader(stream):
442-
metrics_path = manifest_path.parent / row["SourcePath"]
443-
metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
444-
candidate_runs.append({
445-
"runId": row["RunId"],
446-
"variant": row["Variant"],
447-
"sourceOwned": metrics["droppedSourceOwnedSectionCandidates"],
448-
"trackedItemsMax": metrics["droppedTrackedItemsMax"],
449-
"labelsMax": metrics["droppedLabelsMax"],
450-
"fullScanCandidates": metrics["droppedFullScanCandidates"],
451-
"spatialCandidates": metrics["droppedSpatialCandidates"],
452-
"viewerDistanceChecks": metrics["droppedViewerDistanceChecks"],
453-
})
454-
checks["allRunsTracked2048Labels"] = all(
455-
run["trackedItemsMax"] == 2048 and run["labelsMax"] == 2048
456-
for run in candidate_runs
457-
)
458-
checks["candidateHasZeroFullScans"] = all(
459-
run["fullScanCandidates"] == 0
460-
for run in candidate_runs if run["variant"] == "B"
461-
)
462-
checks["baselineExercisedFullScans"] = all(
463-
run["fullScanCandidates"] > 0
464-
for run in candidate_runs if run["variant"] == "A"
465-
)
466-
checks["candidateTreatmentIsolated"] = all(
467-
run["sourceOwned"] is (run["variant"] == "B")
468-
for run in candidate_runs
469-
)
470-
passed = all(checks.values())
471-
gate = {
472-
"schemaVersion": 1,
473-
"scenario": "dropped-items",
474-
"abFactor": "dropped-item-section-candidates",
475-
"formalComplete": dropped.get("formalComplete") is True,
476-
"serverRuntimeGatePassed": passed,
477-
"checks": checks,
478-
"ratios": {
479-
"droppedItemMs": {
480-
"medianBRatioToA": dropped["medianBRatioToA"],
481-
"ratioBootstrap95Ci": dropped["ratioBootstrap95Ci"],
482-
},
483-
"msptP95": {
484-
"medianBRatioToA": p95["medianBRatioToA"],
485-
"ratioBootstrap95Ci": p95["ratioBootstrap95Ci"],
486-
},
487-
"msptP99": {
488-
"medianBRatioToA": p99["medianBRatioToA"],
489-
"ratioBootstrap95Ci": p99["ratioBootstrap95Ci"],
490-
},
491-
},
492-
"runs": candidate_runs,
493-
"scope": "Server runtime gate only; allocation and native-client frame gates remain separate.",
494-
}
495-
(evidence_root / "dropped-item-section-candidates.gate.json").write_text(
496-
json.dumps(gate, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
497-
)
498-
summary = os.environ.get("GITHUB_STEP_SUMMARY")
499-
if summary:
500-
with open(summary, "a", encoding="utf-8", newline="\n") as stream:
501-
stream.write("### Dropped-item section-candidate server gate\n\n")
502-
stream.write(f"Result: `{'pass' if passed else 'fail'}`.\n\n")
503-
stream.write(
504-
f"droppedItemMs B/A: `{dropped['medianBRatioToA']}` "
505-
f"(95% CI `{dropped['ratioBootstrap95Ci']}`).\n\n"
506-
)
507-
stream.write(
508-
f"MSPT p95 CI upper: `{p95['ratioBootstrap95Ci'][1]}`; "
509-
f"p99 CI upper: `{p99['ratioBootstrap95Ci'][1]}`.\n"
510-
)
511-
if not gate["formalComplete"]:
512-
raise SystemExit("dropped-item gate is not a complete 12-run campaign")
513-
if not passed:
514-
failed = ", ".join(name for name, value in checks.items() if not value)
515-
raise SystemExit(f"dropped-item server gate failed: {failed}")
516-
PY
427+
python3 tools/perf/evaluate-dropped-item-gate.py \
428+
"$MANIFEST" "$EVIDENCE_ROOT" "$ITEMS" "$DROPPED_NEARBY_ITEMS" \
429+
"$EVIDENCE_ROOT/dropped-item-section-candidates.gate.json"
517430
fi
518431
519432
if [[ "$SCENARIO" == block-* ]]; then

‎common/src/main/java/com/loohp/interactionvisualizer/Commands.java‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,7 @@ private boolean handlePerformanceCommand(CommandSender sender, String[] args) {
308308
if (args.length < 4) {
309309
sender.sendMessage(Component.text(
310310
"Usage: /iv perf scene <static|motion|itemdisplay|textdisplay|dropped> " +
311-
"<count> [lifetimeTicks] [player]"));
311+
"<count> [lifetimeTicks] [player] [nearbyCount]"));
312312
return true;
313313
}
314314
boolean moving = args[2].equalsIgnoreCase("motion");
@@ -321,6 +321,11 @@ private boolean handlePerformanceCommand(CommandSender sender, String[] args) {
321321
"[InteractionVisualizer] Scene type must be static, motion, itemdisplay, textdisplay, or dropped."));
322322
return true;
323323
}
324+
if (args.length > 7 || args.length == 7 && !dropped) {
325+
sender.sendMessage(Component.text(
326+
"[InteractionVisualizer] nearbyCount is only valid for dropped scenes."));
327+
return true;
328+
}
324329
long defaultLifetime = moving ? 80L : 200L;
325330
long lifetime = defaultLifetime;
326331
String requestedPlayer = args.length >= 6 ? args[5] : null;
@@ -344,17 +349,34 @@ private boolean handlePerformanceCommand(CommandSender sender, String[] args) {
344349
return true;
345350
}
346351
int count = parseInteger(args[3], 1);
352+
int requestedNearbyCount = count;
353+
if (args.length == 7) {
354+
try {
355+
requestedNearbyCount = Integer.parseInt(args[6]);
356+
} catch (NumberFormatException ignored) {
357+
sender.sendMessage(Component.text(
358+
"[InteractionVisualizer] nearbyCount must be an integer."));
359+
return true;
360+
}
361+
if (requestedNearbyCount < 1) {
362+
sender.sendMessage(Component.text(
363+
"[InteractionVisualizer] nearbyCount must be positive."));
364+
return true;
365+
}
366+
}
347367
int spawned = dropped
348-
? PerformanceScene.spawnDroppedItems(player, count, lifetime)
368+
? PerformanceScene.spawnDroppedItems(player, count, requestedNearbyCount, lifetime)
349369
: itemDisplay || textDisplay
350370
? PerformanceScene.spawnDisplay(player, textDisplay, count, lifetime)
351371
: PerformanceScene.spawn(player, moving, count, lifetime);
352372
String sceneName = moving ? "moving" : staticItem ? "static"
353373
: itemDisplay ? "itemdisplay" : textDisplay ? "textdisplay" : "dropped";
354374
String entityLabel = staticItem || moving || dropped
355375
? " benchmark items" : " benchmark entities";
376+
String nearbyLabel = dropped && requestedNearbyCount != spawned
377+
? " (" + Math.min(spawned, requestedNearbyCount) + " nearby)" : "";
356378
sender.sendMessage(Component.text("[InteractionVisualizer] Spawned " + spawned + " "
357-
+ sceneName + entityLabel + " for " + lifetime + " ticks."));
379+
+ sceneName + entityLabel + nearbyLabel + " for " + lifetime + " ticks."));
358380
}
359381
case "clear" -> {
360382
String requestedOwner = args.length >= 3 ? args[2] : null;

0 commit comments

Comments
 (0)