Skip to content

Commit df49b91

Browse files
giles17Copilot
andcommitted
fix: use deterministic provider-qualified keys for dotnet tests
Always prefix dotnet test keys with provider (e.g. net10.0 (ubuntu)::TestName) to ensure stable, comparable counts across runs regardless of file parse order. Also show Executed (passed+failed) instead of Total in summary table. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent f48c8b3 commit df49b91

1 file changed

Lines changed: 16 additions & 19 deletions

File tree

python/scripts/flaky_report/aggregate.py

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -238,24 +238,18 @@ def load_current_run(reports_dir: Path) -> dict[str, Any]:
238238
"results": {},
239239
}
240240

241+
# Dotnet tests always run under multiple frameworks, so we always
242+
# qualify their keys with the provider to ensure deterministic,
243+
# stable keys across runs regardless of file parse order.
244+
is_dotnet = any(d.startswith("dotnet-test-results-") for d, _ in xml_files)
245+
241246
for dir_name, xml_file in xml_files:
242247
print(f" Loading: {xml_file}")
243248
provider = _derive_provider(dir_name)
244249
tests = _parse_junit_xml(xml_file)
245250
for test in tests:
246-
# Use provider-qualified key when the same test runs under
247-
# multiple providers (e.g. dotnet net10.0 vs net472). This
248-
# prevents later results from silently overwriting earlier ones.
249251
raw_id = test["nodeid"]
250-
key = raw_id
251-
if key in combined_results and combined_results[key]["provider"] != provider:
252-
# Collision: re-key existing entry and use qualified key for new one
253-
existing = combined_results.pop(key)
254-
combined_results[f"{existing['provider']}::{raw_id}"] = existing
255-
key = f"{provider}::{raw_id}"
256-
elif f"{provider}::{raw_id}" in combined_results:
257-
# Provider-qualified key already exists (previous collision)
258-
key = f"{provider}::{raw_id}"
252+
key = f"{provider}::{raw_id}" if is_dotnet else raw_id
259253

260254
combined_results[key] = {
261255
"status": test["status"],
@@ -327,19 +321,22 @@ def generate_trend_report(runs: list[dict[str, Any]]) -> str:
327321
# --- Overall status table (most recent first) ---
328322
lines.append("## Overall Status (Last 5 Runs)")
329323
lines.append("")
330-
lines.append("| Run | Total | ✅ Passed | ❌ Failed | ⏭️ Skipped |")
331-
lines.append("|-----|-------|-----------|-----------|------------|")
324+
lines.append("| Run | Executed | ✅ Passed | ❌ Failed | ⏭️ Skipped |")
325+
lines.append("|-----|----------|-----------|-----------|------------|")
332326

333327
for run in reversed(runs):
334328
s = run.get("summary", {})
335-
total = s.get("total", 0)
329+
passed = s.get("passed", 0)
330+
failed = s.get("failed", 0)
331+
skipped = s.get("skipped", 0)
332+
executed = passed + failed
336333
label = _format_run_label(run["timestamp"])
337334
lines.append(
338335
f"| {label} "
339-
f"| {total} "
340-
f"| {s.get('passed', 0)}/{total} "
341-
f"| {s.get('failed', 0)}/{total} "
342-
f"| {s.get('skipped', 0)}/{total} |"
336+
f"| {executed} "
337+
f"| {passed} "
338+
f"| {failed} "
339+
f"| {skipped} |"
343340
)
344341

345342
for _ in range(MAX_HISTORY - len(runs)):

0 commit comments

Comments
 (0)