Skip to content

Commit da028a5

Browse files
Arbousier1Arbousier1
committed
Separate profiled runtime evidence
1 parent ff97c99 commit da028a5

1 file changed

Lines changed: 75 additions & 39 deletions

File tree

‎.github/workflows/phase2-runtime-ab.yml‎

Lines changed: 75 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
name: Phase 2 Clean Runtime A-B
1+
name: Phase 2 Runtime A-B
22

33
on:
44
pull_request:
55
types: [opened, synchronize, reopened, labeled]
66
workflow_dispatch:
77
inputs:
88
scenario:
9-
description: "Clean runtime workload"
9+
description: "Runtime workload"
1010
required: true
1111
default: "static-steady"
1212
type: choice
@@ -33,7 +33,7 @@ on:
3333
default: "20"
3434
type: string
3535
measure_seconds:
36-
description: "Clean MSPT/TPS sampling window"
36+
description: "MSPT/TPS sampling window (diagnostic when Spark is enabled)"
3737
required: true
3838
default: "180"
3939
type: string
@@ -49,7 +49,7 @@ permissions:
4949

5050
jobs:
5151
clean-runtime-ab:
52-
name: Paper 26.1.2 clean TPS-MSPT ABBA
52+
name: Paper 26.1.2 runtime ABBA
5353
if: >-
5454
github.event_name == 'workflow_dispatch' ||
5555
github.event.action != 'labeled' ||
@@ -64,6 +64,7 @@ jobs:
6464
CAMPAIGN_SETTLE_SECONDS: ${{ github.event_name == 'workflow_dispatch' && inputs.settle_seconds || github.event.action == 'labeled' && '20' || '5' }}
6565
CAMPAIGN_MEASURE_SECONDS: ${{ github.event_name == 'workflow_dispatch' && inputs.measure_seconds || github.event.action == 'labeled' && '180' || '10' }}
6666
CAMPAIGN_SPARK_PROFILE_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.spark_profile_mode || 'none' }}
67+
CAMPAIGN_EVIDENCE_KIND: ${{ github.event_name == 'workflow_dispatch' && inputs.spark_profile_mode != 'none' && 'diagnostic' || 'clean' }}
6768

6869
steps:
6970
- name: Check out immutable test source
@@ -113,7 +114,7 @@ jobs:
113114
- name: Prepare immutable protocol client artifact
114115
run: bash tools/perf/prepare-phase2-protocol-client.sh phase2-dependencies/protocol-client
115116

116-
- name: Run restart-isolated clean ABBA campaign
117+
- name: Run restart-isolated runtime campaign
117118
env:
118119
SCENARIO: ${{ env.CAMPAIGN_SCENARIO }}
119120
RUNS: ${{ env.CAMPAIGN_RUNS }}
@@ -122,6 +123,7 @@ jobs:
122123
SETTLE_SECONDS: ${{ env.CAMPAIGN_SETTLE_SECONDS }}
123124
MEASURE_SECONDS: ${{ env.CAMPAIGN_MEASURE_SECONDS }}
124125
SPARK_PROFILE_MODE: ${{ env.CAMPAIGN_SPARK_PROFILE_MODE }}
126+
EVIDENCE_KIND: ${{ env.CAMPAIGN_EVIDENCE_KIND }}
125127
run: |
126128
set -euo pipefail
127129
[[ "$SCENARIO" == static-steady || "$SCENARIO" == block-idle || \
@@ -163,8 +165,16 @@ jobs:
163165
164166
PLUGIN_JAR=$(find build/libs -maxdepth 1 -type f -name 'InteractionVisualizer-*.jar' ! -name '*-sources.jar' ! -name '*-benchmark.jar' -print -quit)
165167
test -n "$PLUGIN_JAR"
166-
mkdir -p phase2-results/clean
167-
MANIFEST=phase2-results/clean/abba-manifest.csv
168+
if [[ "$SPARK_PROFILE_MODE" == none ]]; then
169+
[[ "$EVIDENCE_KIND" == clean ]]
170+
CAPTURE_METHOD=none
171+
else
172+
[[ "$EVIDENCE_KIND" == diagnostic ]]
173+
CAPTURE_METHOD="spark-$SPARK_PROFILE_MODE"
174+
fi
175+
EVIDENCE_ROOT="phase2-results/$EVIDENCE_KIND"
176+
mkdir -p "$EVIDENCE_ROOT"
177+
MANIFEST="$EVIDENCE_ROOT/abba-manifest.csv"
168178
printf 'Scenario,Block,Position,Variant,RunId,StackSha256,ArtifactSha256,CaptureMethod,SourcePath\n' > "$MANIFEST"
169179
ARTIFACT_SHA=$(sha256sum "$PLUGIN_JAR" | awk '{print $1}')
170180
STACK_SHA=$(
@@ -189,7 +199,7 @@ jobs:
189199
PHASE2_PLUGIN_JAR="$PLUGIN_JAR" \
190200
PHASE2_PAPER_JAR=phase2-dependencies/paper.jar \
191201
PHASE2_CLIENT_ROOT=phase2-dependencies/protocol-client \
192-
PHASE2_OUTPUT_ROOT=phase2-results/clean \
202+
PHASE2_OUTPUT_ROOT="$EVIDENCE_ROOT" \
193203
PHASE2_RUN_ID="$run_id" \
194204
PHASE2_SCENARIO="$SCENARIO" \
195205
PHASE2_VARIANT="$variant" \
@@ -202,42 +212,45 @@ jobs:
202212
PHASE2_SPARK_PROFILE_MODE="$SPARK_PROFILE_MODE" \
203213
bash tools/perf/run-phase2-runtime-once.sh
204214
205-
printf '%s,%d,%d,%s,%s,%s,%s,none,%s/iv-perf.json\n' \
206-
"$SCENARIO" "$block" "$position" "$variant" "$run_id" "$STACK_SHA" "$ARTIFACT_SHA" "$run_id" \
215+
printf '%s,%d,%d,%s,%s,%s,%s,%s,%s/iv-perf.json\n' \
216+
"$SCENARIO" "$block" "$position" "$variant" "$run_id" "$STACK_SHA" "$ARTIFACT_SHA" \
217+
"$CAPTURE_METHOD" "$run_id" \
207218
>> "$MANIFEST"
208219
done
209220
210-
minimum_seconds=$(( MEASURE_SECONDS - 2 ))
211-
incomplete=()
212-
if [[ "$RUNS" != 12 ]]; then incomplete=(-AllowIncomplete); fi
213-
for metric in msptP95 msptP99 msptMean; do
221+
if [[ "$SPARK_PROFILE_MODE" == none ]]; then
222+
minimum_seconds=$(( MEASURE_SECONDS - 2 ))
223+
incomplete=()
224+
if [[ "$RUNS" != 12 ]]; then incomplete=(-AllowIncomplete); fi
225+
for metric in msptP95 msptP99 msptMean; do
226+
pwsh -NoProfile -File tools/perf/analyze-phase2-abba.ps1 "$MANIFEST" \
227+
-Scenario "$SCENARIO" -Metric "$metric" -Direction LowerIsBetter \
228+
-MinimumSeconds "$minimum_seconds" "${incomplete[@]}" \
229+
-OutputJson "$EVIDENCE_ROOT/$metric.analysis.json" -Overwrite
230+
done
231+
# At a healthy 20 TPS cap this is only an overload/non-regression
232+
# diagnostic. MSPT remains the optimization effect-size metric.
214233
pwsh -NoProfile -File tools/perf/analyze-phase2-abba.ps1 "$MANIFEST" \
215-
-Scenario "$SCENARIO" -Metric "$metric" -Direction LowerIsBetter \
234+
-Scenario "$SCENARIO" -Metric observedTps -Direction HigherIsBetter \
216235
-MinimumSeconds "$minimum_seconds" "${incomplete[@]}" \
217-
-OutputJson "phase2-results/clean/$metric.analysis.json" -Overwrite
218-
done
219-
# At a healthy 20 TPS cap this is only an overload/non-regression
220-
# diagnostic. MSPT remains the optimization effect-size metric.
221-
pwsh -NoProfile -File tools/perf/analyze-phase2-abba.ps1 "$MANIFEST" \
222-
-Scenario "$SCENARIO" -Metric observedTps -Direction HigherIsBetter \
223-
-MinimumSeconds "$minimum_seconds" "${incomplete[@]}" \
224-
-OutputJson phase2-results/clean/observedTps.analysis.json -Overwrite
236+
-OutputJson "$EVIDENCE_ROOT/observedTps.analysis.json" -Overwrite
237+
fi
225238
226239
if [[ "$SCENARIO" == block-* ]]; then
227240
# Active/direct-write CPU time has a meaningful LowerIsBetter
228241
# direction after lifecycle/config assertions pass. An idle
229242
# candidate can legitimately perform zero checks in a short
230243
# window, so it stays diagnostic instead of entering log ratios.
231244
# Raw check counts never carry a universal performance direction.
232-
if [[ "$SCENARIO" != block-idle ]]; then
245+
if [[ "$SPARK_PROFILE_MODE" == none && "$SCENARIO" != block-idle ]]; then
233246
pwsh -NoProfile -File tools/perf/analyze-phase2-abba.ps1 "$MANIFEST" \
234247
-Scenario "$SCENARIO" -Metric blockUpdateMs -Direction LowerIsBetter \
235248
-MinimumSeconds "$minimum_seconds" "${incomplete[@]}" \
236-
-OutputJson phase2-results/clean/blockUpdateMs.analysis.json -Overwrite
249+
-OutputJson "$EVIDENCE_ROOT/blockUpdateMs.analysis.json" -Overwrite
237250
fi
238251
239-
python3 - "$MANIFEST" "$SCENARIO" "$RUNS" "$ITEMS" \
240-
phase2-results/clean/blockUpdateChecks.evidence.json <<'PY'
252+
python3 - "$MANIFEST" "$SCENARIO" "$RUNS" "$ITEMS" "$CAPTURE_METHOD" \
253+
"$EVIDENCE_ROOT/blockUpdateChecks.evidence.json" <<'PY'
241254
from pathlib import Path
242255
import csv
243256
import json
@@ -250,7 +263,8 @@ jobs:
250263
scenario = sys.argv[2]
251264
expected_runs = int(sys.argv[3])
252265
expected_items = int(sys.argv[4])
253-
output_path = Path(sys.argv[5])
266+
capture_method = sys.argv[5]
267+
output_path = Path(sys.argv[6])
254268
records = []
255269
with manifest_path.open(encoding="utf-8", newline="") as stream:
256270
rows = list(csv.DictReader(stream))
@@ -260,6 +274,10 @@ jobs:
260274
for row in rows:
261275
if row["Scenario"] != scenario:
262276
raise SystemExit(f"manifest scenario drift: {row['Scenario']} != {scenario}")
277+
if row["CaptureMethod"] != capture_method:
278+
raise SystemExit(
279+
f"manifest capture-method drift: {row['CaptureMethod']} != {capture_method}"
280+
)
263281
source_path = manifest_path.parent / row["SourcePath"]
264282
metrics = json.loads(source_path.read_text(encoding="utf-8"))
265283
run_id = row["RunId"]
@@ -341,22 +359,36 @@ jobs:
341359
),
342360
}
343361
362+
if capture_method == "none":
363+
interpretation = (
364+
"blockUpdateChecks is workload evidence, not a standalone optimization objective; "
365+
"fewer checks can mean either effective event-driven scheduling or missing work. "
366+
"For block-idle, zero candidate checks/time in a short window is valid and blockUpdateMs "
367+
"has no ratio analysis. Interpret all scenarios with machine-validated scene actions, "
368+
"blockUpdateMs, MSPT and TPS."
369+
)
370+
else:
371+
interpretation = (
372+
"Instrumented Spark workload evidence only. blockUpdateChecks can establish executed work, "
373+
"but MSPT, TPS and blockUpdateMs were collected under profiler overhead and must not be "
374+
"used as clean performance effect sizes. Use the saved profile to attribute CPU cost, then "
375+
"confirm any optimization in a separate captureMethod=none campaign."
376+
)
377+
344378
evidence = {
345379
"schemaVersion": 1,
346380
"analysisType": "diagnostic-work-evidence",
347381
"scenario": scenario,
348382
"requestedRuns": expected_runs,
349383
"expectedItems": expected_items,
350-
"formalComplete": expected_runs == 12,
384+
"captureMethod": capture_method,
385+
"performanceEvidenceReady": capture_method == "none",
386+
"formalComplete": expected_runs == 12 and capture_method == "none",
351387
"blockUpdateChecksDirection": None,
352-
"blockUpdateMsDirection": None if scenario == "block-idle" else "LowerIsBetter",
353-
"interpretation": (
354-
"blockUpdateChecks is workload evidence, not a standalone optimization objective; "
355-
"fewer checks can mean either effective event-driven scheduling or missing work. "
356-
"For block-idle, zero candidate checks/time in a short window is valid and blockUpdateMs "
357-
"has no ratio analysis. Interpret all scenarios with machine-validated scene actions, "
358-
"blockUpdateMs, MSPT and TPS."
388+
"blockUpdateMsDirection": (
389+
"LowerIsBetter" if capture_method == "none" and scenario != "block-idle" else None
359390
),
391+
"interpretation": interpretation,
360392
"activeCadenceGuard": cadence_guard,
361393
"byVariant": by_variant,
362394
"runs": records,
@@ -370,6 +402,10 @@ jobs:
370402
if step_summary:
371403
with open(step_summary, "a", encoding="utf-8", newline="\n") as stream:
372404
stream.write(f"### {scenario} block update evidence\n\n")
405+
stream.write(
406+
f"Capture method: `{capture_method}`; clean performance evidence ready: "
407+
f"`{str(capture_method == 'none').lower()}`.\n\n"
408+
)
373409
stream.write("`blockUpdateChecks` is diagnostic work evidence (no universal direction).\n\n")
374410
stream.write("| Run | Variant | Checks | Checks/tick | Update ms | ms/check |\n")
375411
stream.write("|---|---:|---:|---:|---:|---:|\n")
@@ -398,13 +434,13 @@ jobs:
398434
PY
399435
fi
400436
401-
- name: Publish clean runtime evidence
437+
- name: Publish runtime evidence
402438
if: always()
403439
uses: actions/upload-artifact@v4
404440
with:
405-
name: phase2-clean-runtime-${{ env.CAMPAIGN_SCENARIO }}-${{ github.sha }}-${{ github.run_id }}
441+
name: phase2-${{ env.CAMPAIGN_EVIDENCE_KIND }}-runtime-${{ env.CAMPAIGN_SCENARIO }}-${{ github.sha }}-${{ github.run_id }}
406442
path: |
407-
phase2-results/clean
443+
phase2-results/${{ env.CAMPAIGN_EVIDENCE_KIND }}
408444
phase2-dependencies/protocol-client/client-build-manifest.json
409445
phase2-dependencies/protocol-client/client-files.sha256
410446
phase2-dependencies/protocol-client/node-minecraft-protocol/package-lock.json

0 commit comments

Comments
 (0)