Skip to content

Commit d81e162

Browse files
cds-amalcds-amal
committed
feat(tests): add make trace-report for pipeline coverage analysis
Adds a trace-report target that runs TRACE=1 across all integration tests and produces a coverage summary. The report shows: - event counts with per-program attribution - invariant verification (UnevaluatedConstDiscovered should stay at 0) - sub-category breakdown with gaps classified as actionable vs structurally unreachable (e.g. FnDef/FnPtr/Closure ty_kinds are actionable gaps; Coroutine/Pat are unreachable) - per-program @Covers annotations extracted from test source files The analysis logic lives in trace-report.py (called by trace-report.sh) to keep the Makefile target minimal.
1 parent 955f8a2 commit d81e162

3 files changed

Lines changed: 232 additions & 1 deletion

File tree

‎Makefile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,12 @@ integration-test:
4343
golden:
4444
make integration-test DIFF=">"
4545

46-
format:
46+
.PHONY: trace-report
47+
trace-report: SMIR ?= cargo run -- "-Zno-codegen"
48+
trace-report:
49+
bash tests/integration/trace-report.sh $(TESTDIR) $(SMIR) | tee tests/integration/trace-report.txt
50+
51+
format:
4752
cargo fmt
4853
bash -O globstar -c 'nixfmt **/*.nix'
4954

‎tests/integration/trace-report.py‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
"""Aggregate trace event coverage and report gaps.
2+
3+
Called by trace-report.sh after TRACE=1 runs have been collected.
4+
Usage: trace-report.py <trace_dir> <test_dir>
5+
"""
6+
7+
import json
8+
import glob
9+
import os
10+
import re
11+
import sys
12+
from collections import Counter
13+
14+
trace_dir = sys.argv[1]
15+
test_dir = sys.argv[2] if len(sys.argv) > 2 else None
16+
17+
# ── Collect events ──────────────────────────────────────────────────
18+
19+
counts = {}
20+
by_program = {} # event -> { program -> count }
21+
sub_values = {} # event -> field -> Counter(value -> count)
22+
sub_programs = {} # event -> field -> value -> set(programs)
23+
24+
SUB_FIELDS = ["ty_kind", "sym_kind", "source"]
25+
26+
files = sorted(glob.glob(trace_dir + "/*.smir.trace.json"))
27+
if not files:
28+
print("No trace files found. Is TRACE=1 working?")
29+
sys.exit(1)
30+
31+
for f in files:
32+
prog = os.path.basename(f).replace(".smir.trace.json", "")
33+
for e in json.load(open(f)):
34+
ev = e["event"]
35+
counts[ev] = counts.get(ev, 0) + 1
36+
by_program.setdefault(ev, {})
37+
by_program[ev][prog] = by_program[ev].get(prog, 0) + 1
38+
for key in SUB_FIELDS:
39+
if key in e:
40+
sub_values.setdefault(ev, {}).setdefault(key, Counter())
41+
sub_values[ev][key][e[key]] += 1
42+
sub_programs.setdefault(ev, {}).setdefault(key, {})
43+
sub_programs[ev][key].setdefault(e[key], set())
44+
sub_programs[ev][key][e[key]].add(prog)
45+
46+
n = len(files)
47+
48+
# ── Event summary ───────────────────────────────────────────────────
49+
50+
print(f"Trace coverage across {n} test(s):\n")
51+
print(f" {'Count':>6} Event")
52+
print(f" {'-----':>6} -----")
53+
for k, v in sorted(counts.items(), key=lambda x: -x[1]):
54+
progs = by_program.get(k, {})
55+
np = len(progs)
56+
if np == n:
57+
detail = "all programs"
58+
elif np <= 3:
59+
detail = ", ".join(sorted(progs.keys()))
60+
else:
61+
detail = f"{np}/{n} programs"
62+
print(f" {v:6} {k} ({detail})")
63+
64+
# ── Coverage vs invariant classification ────────────────────────────
65+
66+
coverage_events = [
67+
"ItemDiscovered", "BodyWalkStarted", "BodyWalkFinished",
68+
"FunctionCallResolved", "DropGlueResolved", "ReifyFnPointerResolved",
69+
"AllocationCollected", "FnDefAsValue",
70+
"TypeCollected", "SpanResolved", "AssemblyStarted",
71+
]
72+
73+
invariant_events = [
74+
"UnevaluatedConstDiscovered",
75+
]
76+
77+
uncovered = [e for e in coverage_events if e not in counts]
78+
violated = [e for e in invariant_events if counts.get(e, 0) > 0]
79+
80+
if uncovered:
81+
print(f"\nUncovered ({len(uncovered)}):")
82+
for e in uncovered:
83+
print(f" ** {e}")
84+
85+
if violated:
86+
print(f"\nInvariant violations ({len(violated)}):")
87+
for e in violated:
88+
print(f" !! {e}: {counts[e]} occurrence(s)")
89+
90+
held = [e for e in invariant_events if counts.get(e, 0) == 0]
91+
if held:
92+
print(f"\nInvariants holding ({len(held)}):")
93+
for e in held:
94+
print(f" ok {e}")
95+
96+
if not uncovered and not violated:
97+
print("\nAll coverage events exercised, all invariants holding.")
98+
99+
# ── Sub-category breakdown ──────────────────────────────────────────
100+
101+
# Known universe of values for each (event, field) pair, classified as
102+
# either "actionable" (could be covered with a test) or "unreachable"
103+
# (structurally impossible or requires unstable/exotic features).
104+
# Each entry maps a value to a reason string (None = actionable).
105+
KNOWN_UNIVERSE = {
106+
("TypeCollected", "ty_kind"): {
107+
# actionable
108+
"Bool": None, "Char": None, "Int": None, "Uint": None, "Float": None,
109+
"Adt": None, "Str": None, "Array": None, "Slice": None,
110+
"RawPtr": None, "Ref": None, "Dynamic": None, "Never": None, "Tuple": None,
111+
"FnDef": None, "FnPtr": None, "Closure": None,
112+
"Foreign": None,
113+
# unreachable or exotic
114+
"Coroutine": "async/generators monomorphize to ADT state machines",
115+
"CoroutineWitness": "async/generators monomorphize to ADT state machines",
116+
"Pat": "unstable feature (pattern_types)",
117+
},
118+
("FunctionCallResolved", "sym_kind"): {
119+
"normal": None, "intrinsic": None,
120+
"no_op": "no-op shims only arise from drop glue, not call terminators",
121+
},
122+
("DropGlueResolved", "sym_kind"): {
123+
"no_op": None,
124+
"normal": "drop glue is always a no-op shim",
125+
"intrinsic": "drop glue is always a no-op shim",
126+
},
127+
("ReifyFnPointerResolved", "sym_kind"): {
128+
"normal": None,
129+
"intrinsic": "intrinsics cannot be coerced to fn pointers",
130+
"no_op": "no-op shims cannot be coerced to fn pointers",
131+
},
132+
("ItemDiscovered", "source"): {
133+
"mono_collect": None,
134+
"unevaluated_const": "compiler eagerly evaluates all consts on current nightly",
135+
},
136+
}
137+
138+
print("\n" + "=" * 60)
139+
print("Sub-category breakdown:")
140+
print("=" * 60)
141+
142+
for ev in sorted(sub_values):
143+
for key in sorted(sub_values[ev]):
144+
counter = sub_values[ev][key]
145+
universe_key = (ev, key)
146+
universe = KNOWN_UNIVERSE.get(universe_key)
147+
148+
print(f"\n {ev} / {key}:")
149+
for val, count in counter.most_common():
150+
progs = sub_programs[ev][key][val]
151+
np = len(progs)
152+
if np == n:
153+
detail = "all"
154+
elif np <= 3:
155+
detail = ", ".join(sorted(progs))
156+
else:
157+
detail = f"{np}/{n}"
158+
print(f" {count:6} {val} ({detail})")
159+
160+
if universe:
161+
actionable = [v for v in universe if v not in counter and universe[v] is None]
162+
unreachable = [(v, universe[v]) for v in universe if v not in counter and universe[v] is not None]
163+
if actionable:
164+
print(f" gaps (actionable):")
165+
for v in actionable:
166+
print(f" ** {v}")
167+
if unreachable:
168+
print(f" gaps (unreachable):")
169+
for v, reason in unreachable:
170+
print(f" -- {v} ({reason})")
171+
172+
# ── Per-program annotations ─────────────────────────────────────────
173+
174+
covers = {}
175+
if test_dir:
176+
for rs in sorted(glob.glob(test_dir + "/*.rs")):
177+
prog = os.path.basename(rs).replace(".rs", "")
178+
with open(rs) as f:
179+
for line in f:
180+
m = re.match(r"//[!/]\s*@covers:\s*(.+)", line)
181+
if m:
182+
covers[prog] = m.group(1).strip()
183+
break
184+
185+
if covers:
186+
print(f"\nTest programs ({len(covers)}/{n} annotated):\n")
187+
for prog in sorted(covers):
188+
events = []
189+
for ev in sorted(by_program):
190+
if prog in by_program[ev]:
191+
events.append(ev)
192+
print(f" {prog}")
193+
print(f" covers: {covers[prog]}")
194+
unique = [ev for ev in events if len(by_program[ev]) == 1]
195+
if unique:
196+
print(f" unique: {', '.join(unique)}")
197+
unannotated = sorted(set(
198+
os.path.basename(f).replace(".smir.trace.json", "")
199+
for f in files
200+
) - set(covers))
201+
if unannotated:
202+
print(f"\n Missing @covers annotation:")
203+
for p in unannotated:
204+
print(f" ** {p}")

‎tests/integration/trace-report.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
# Run TRACE=1 across integration tests and aggregate event coverage.
3+
#
4+
# Usage: trace-report.sh <testdir> <smir_cmd...>
5+
# Example: trace-report.sh tests/integration/programs cargo run -- -Zno-codegen
6+
7+
set -euo pipefail
8+
9+
testdir="$1"; shift
10+
smir=("$@")
11+
12+
tmpdir=$(mktemp -d)
13+
trap 'rm -rf "$tmpdir"' EXIT
14+
15+
for rust in "$testdir"/*.rs; do
16+
name=$(basename "$rust" .rs)
17+
echo "Tracing $name..."
18+
TRACE=1 "${smir[@]}" --out-dir "$tmpdir" "$rust" 2>/dev/null
19+
done
20+
21+
script_dir="$(cd "$(dirname "$0")" && pwd)"
22+
python3 "$script_dir/trace-report.py" "$tmpdir" "$testdir"

0 commit comments

Comments
 (0)