Skip to content

Commit 73bcd0f

Browse files
uLipeuLipe
committed
tools: emit CI coverage reports from JSONL test results
- Persist per-case JSONL via dev.py --json-dir and upload artifacts in CI - Aggregate PASS/FAIL and coverage matrix from tools/coverage_manifest.yaml
1 parent 0c53d48 commit 73bcd0f

5 files changed

Lines changed: 532 additions & 10 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,15 @@ jobs:
113113
docker tag "$IMG" ulipe-microkernel:dev
114114
115115
- name: Run unit tests
116-
run: python3 tools/dev.py tests unit
116+
run: python3 tools/dev.py tests unit --json-dir ci-results/unit
117+
118+
- name: Upload unit results
119+
if: always()
120+
uses: actions/upload-artifact@v4
121+
with:
122+
name: test-results-unit
123+
path: ci-results/unit/
124+
if-no-files-found: warn
117125

118126
# ── SDK suite (black-box QEMU, replaces legacy integ Makefile suites) ─────
119127
integ-tests:
@@ -141,10 +149,18 @@ jobs:
141149
docker tag "$IMG" ulipe-microkernel:dev
142150
143151
- name: Run SDK suite
144-
run: python3 tools/dev.py tests e2e
152+
run: python3 tools/dev.py tests e2e --json-dir ci-results/e2e-tricore
145153

146154
- name: Run arch whitebox (TriCore CSA early)
147-
run: python3 tools/dev.py tests integ --test sdk_suite/arch_whitebox/ctx_early_tricore
155+
run: python3 tools/dev.py tests integ --test sdk_suite/arch_whitebox/ctx_early_tricore --json-dir ci-results/e2e-tricore
156+
157+
- name: Upload TriCore results
158+
if: always()
159+
uses: actions/upload-artifact@v4
160+
with:
161+
name: test-results-tricore
162+
path: ci-results/e2e-tricore/
163+
if-no-files-found: warn
148164

149165
# ── RISC-V: build ─────────────────────────────────────────────────────────
150166
build-riscv:
@@ -202,7 +218,15 @@ jobs:
202218
docker tag "$IMG" ulipe-microkernel:dev
203219
204220
- name: Run SDK suite
205-
run: python3 tools/dev.py tests e2e --board boards/qemu_riscv_virt
221+
run: python3 tools/dev.py tests e2e --board boards/qemu_riscv_virt --json-dir ci-results/e2e-riscv
222+
223+
- name: Upload RISC-V results
224+
if: always()
225+
uses: actions/upload-artifact@v4
226+
with:
227+
name: test-results-riscv
228+
path: ci-results/e2e-riscv/
229+
if-no-files-found: warn
206230

207231
# ── ARM Cortex-M: build (mps2-an500 M7 + mps2-an505 M33) ──────────────────
208232
build-arm:
@@ -268,4 +292,46 @@ jobs:
268292
docker tag "$IMG" ulipe-microkernel:dev
269293
270294
- name: Run SDK suite
271-
run: python3 tools/dev.py tests e2e --board boards/${{ matrix.board }}
295+
run: python3 tools/dev.py tests e2e --board boards/${{ matrix.board }} --json-dir ci-results/e2e-arm-${{ matrix.board }}
296+
297+
- name: Upload ARM results
298+
if: always()
299+
uses: actions/upload-artifact@v4
300+
with:
301+
name: test-results-arm-${{ matrix.board }}
302+
path: ci-results/e2e-arm-${{ matrix.board }}/
303+
if-no-files-found: warn
304+
305+
# ── Aggregate markdown report (PASS/FAIL/PARTIAL + coverage matrix) ─────
306+
coverage-report:
307+
name: Test coverage report
308+
needs: [unit-tests, integ-tests, integ-tests-riscv, integ-tests-arm]
309+
if: always()
310+
runs-on: ubuntu-latest
311+
permissions:
312+
contents: read
313+
steps:
314+
- uses: actions/checkout@v4
315+
316+
- name: Download all result artifacts
317+
uses: actions/download-artifact@v4
318+
with:
319+
pattern: test-results-*
320+
path: ci-results
321+
322+
- name: Install PyYAML
323+
run: pip3 install --user pyyaml
324+
325+
- name: Generate markdown report
326+
run: |
327+
python3 tools/gen_ci_test_report.py \
328+
--manifest tools/coverage_manifest.yaml \
329+
--results-dir ci-results \
330+
--out test-report.md
331+
cat test-report.md >> "$GITHUB_STEP_SUMMARY"
332+
333+
- name: Upload test-report.md
334+
uses: actions/upload-artifact@v4
335+
with:
336+
name: test-report
337+
path: test-report.md

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,6 @@ __pycache__/
5555
# Internal personal design notes — kept locally, not tracked
5656
docs/microkernel_book_tricore.md
5757
build-riscv/
58+
59+
# SDK suite object leftovers
60+
tests/sdk_suite/**/*.o

‎tools/coverage_manifest.yaml‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# Coverage areas for CI markdown report (tools/gen_ci_test_report.py).
2+
# status per arch is derived: covered | partial | missing
3+
# - covered: listed case exists and is expected on that arch
4+
# - partial: happy-path only / unit-only / arch-limited / known gap
5+
# - missing: area not exercised on that arch
6+
7+
version: 1
8+
9+
arches:
10+
- tricore
11+
- riscv
12+
- arm
13+
14+
# Unit tests run once on host; report marks them kind=unit (not per-arch QEMU).
15+
areas:
16+
- id: thread.basic
17+
title: Thread create/yield/prio/suspend
18+
cases: [thread_unit, sdk_suite/thread_lifecycle, sdk_suite/abi_smoke]
19+
tricore: covered
20+
riscv: covered
21+
arm: covered
22+
23+
- id: sched.order
24+
title: Scheduler ordering / preempt
25+
cases: [sched_unit, sdk_suite/sched_order, sdk_suite/preempt]
26+
tricore: covered
27+
riscv: covered
28+
arm: covered
29+
30+
- id: ipc.basic
31+
title: IPC call/recv/reply
32+
cases: [ipc_unit, sdk_suite/ipc, sdk_suite/abi_smoke]
33+
tricore: covered
34+
riscv: covered
35+
arm: covered
36+
37+
- id: ipc.priority_inherit
38+
title: Priority inheritance (observable)
39+
cases: [ipc_unit, sdk_suite/ipc_pi]
40+
tricore: covered
41+
riscv: covered
42+
arm: covered
43+
note: silicon_ipc_pi + sdk_suite/ipc_pi
44+
45+
- id: mem.heap
46+
title: Per-thread heap / extend
47+
cases: [mem_unit, sdk_suite/mem_heap, sdk_suite/abi_smoke]
48+
tricore: covered
49+
riscv: covered
50+
arm: covered
51+
52+
- id: mem.isolation
53+
title: MPU isolation
54+
cases: [sdk_suite/mem_isolation]
55+
tricore: covered
56+
riscv: covered
57+
arm: partial
58+
note: ARM mem_grant livelock risk
59+
60+
- id: cap.eperm_neg
61+
title: CAP/privilege EPERM matrix
62+
cases: [sdk_suite/cap_neg]
63+
tricore: covered
64+
riscv: covered
65+
arm: covered
66+
note: silicon_cap_neg HIL PASS; sdk_suite/cap_neg
67+
68+
- id: mem.grant
69+
title: mem_grant peer access
70+
cases: [sdk_suite/mem_grant]
71+
tricore: covered
72+
riscv: covered
73+
arm: covered
74+
note: silicon_mem_grant HIL PASS; sdk_suite/mem_grant
75+
76+
- id: fault.policy
77+
title: Fault kill vs panic policy
78+
cases: [sdk_suite/fault_policy]
79+
tricore: covered
80+
riscv: covered
81+
arm: covered
82+
note: userspace MPU fault kills thread; system continues
83+
84+
- id: irq.sw
85+
title: Software IRQ bind/enable/ack
86+
cases: [sdk_suite/irq_sw, sdk_suite/abi_smoke]
87+
tricore: covered
88+
riscv: covered
89+
arm: covered
90+
91+
- id: irq.stress
92+
title: IRQ rebind/flood/out-of-order ack
93+
cases: [sdk_suite/irq_stress]
94+
tricore: covered
95+
riscv: covered
96+
arm: covered
97+
note: silicon_irq_stress HIL + sdk_suite/irq_stress
98+
99+
- id: wcet.syscall_o1
100+
title: Syscall WCET O(1) via kernel CCNT slot
101+
cases: []
102+
tricore: covered
103+
riscv: n/a
104+
arm: n/a
105+
note: silicon_wcet HIL only (ULMK_CONFIG_SYSCALL_WCET); QEMU out of scope
106+
107+
- id: proc.mgmt
108+
title: Process CREATE/DESTROY/ADD_REGION/GRANT_IRQ
109+
cases: []
110+
tricore: missing
111+
riscv: missing
112+
arm: missing
113+
note: out of scope this cycle — process model TBD; GRANT_CAP is ulmk_cap_grant
114+
115+
- id: atomic
116+
title: Atomics
117+
cases: [atomic_unit, sdk_suite/atomic_stress]
118+
tricore: covered
119+
riscv: covered
120+
arm: covered
121+
122+
- id: sleep.timer
123+
title: Board timer sleep
124+
cases: [sdk_suite/sleep_timer]
125+
tricore: covered
126+
riscv: covered
127+
arm: covered
128+
129+
- id: boot.bss
130+
title: Boot / BSS
131+
cases: [sdk_suite/boot_bss]
132+
tricore: covered
133+
riscv: covered
134+
arm: covered
135+
136+
- id: ctx.switch
137+
title: Context switch
138+
cases: [sdk_suite/ctx_switch]
139+
tricore: covered
140+
riscv: covered
141+
arm: covered
142+
143+
- id: resource.leak
144+
title: Resource leak / reclaim
145+
cases: [sdk_suite/resource_leak]
146+
tricore: covered
147+
riscv: covered
148+
arm: covered
149+
150+
- id: abi.smoke
151+
title: Public syscall smoke
152+
cases: [sdk_suite/abi_smoke]
153+
tricore: covered
154+
riscv: covered
155+
arm: covered
156+
157+
- id: arch.ctx_early_tricore
158+
title: TriCore CSA early whitebox
159+
cases: [sdk_suite/arch_whitebox/ctx_early_tricore]
160+
tricore: covered
161+
riscv: missing
162+
arm: missing
163+
note: arch-only
164+
165+
- id: ipc.destroy_waiters
166+
title: ep/notif destroy with waiters
167+
cases: [ipc_unit, sdk_suite/destroy_waiters]
168+
tricore: missing
169+
riscv: missing
170+
arm: missing
171+
note: silicon_destroy_waiters + sdk_suite/destroy_waiters
172+
173+
- id: ipc.kill_rendezvous
174+
title: Kill/exit mid IPC rendezvous
175+
cases: [sdk_suite/kill_rendezvous]
176+
tricore: missing
177+
riscv: missing
178+
arm: missing
179+
note: silicon_kill_rendezvous + sdk_suite/kill_rendezvous
180+
181+
- id: pool.exhaust
182+
title: TCB/EP/notif pool exhaustion
183+
cases: [sdk_suite/pool_exhaust]
184+
tricore: missing
185+
riscv: missing
186+
arm: missing
187+
note: silicon_pool_exhaust + sdk_suite/pool_exhaust
188+
189+
- id: ipc.recv_or_notif_race
190+
title: recv_or_notif IPC vs notif race
191+
cases: [sdk_suite/recv_or_notif_race]
192+
tricore: missing
193+
riscv: missing
194+
arm: missing
195+
note: silicon_recv_or_notif_race + sdk_suite/recv_or_notif_race

0 commit comments

Comments
 (0)