Skip to content

Commit fb5d985

Browse files
sasdftimothytrippel
sasdf
authored and
timothytrippel
committed
[cov,ci] enable coverage tests on ci
This commit introduces code coverage collection and reporting to the CI pipeline. Change-Id: Iffdf510d5cf714d6364e335fa494ea36ac4242b1 Signed-off-by: Yi-Hsuan Deng <yhdeng@google.com>
1 parent 4e272f3 commit fb5d985

9 files changed

Lines changed: 586 additions & 0 deletions

File tree

‎.github/actions/publish-bazel-test-results/action.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,16 @@ runs:
5656
report_paths: ${{ inputs.merged-results }}
5757
annotate_only: true
5858
detailed_summary: true
59+
60+
- name: Collect coverage reports
61+
id: collect_coverage
62+
shell: bash
63+
run: ci/scripts/collect-coverage-report.sh /tmp/coverage_report
64+
65+
- name: Upload coverage report
66+
if: steps.collect_coverage.outputs.coverageReport == 'ok'
67+
uses: actions/upload-artifact@v4
68+
with:
69+
name: ${{ inputs.artifact-name }}-coverage
70+
path: /tmp/coverage_report
71+
overwrite: true

‎.github/workflows/ci-coverage.yml‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
# Copyright lowRISC contributors (OpenTitan project).
2+
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
name: Coverage
6+
on:
7+
push:
8+
branches-ignore:
9+
- "backport-*"
10+
tags:
11+
- "*"
12+
13+
workflow_dispatch:
14+
inputs:
15+
branch:
16+
description: 'Branch to run the coverage tests'
17+
required: true
18+
type: string
19+
20+
permissions:
21+
contents: read
22+
# Needed for workload identity federation
23+
id-token: write
24+
25+
concurrency:
26+
group: ${{ github.workflow }}-${{ github.ref }}
27+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
28+
29+
env:
30+
VIVADO_VERSION: "2021.1"
31+
# Release tag from https://github.com/lowRISC/lowrisc-toolchains/releases
32+
TOOLCHAIN_VERSION: 20220210-1
33+
34+
jobs:
35+
chip_earlgrey_cw310:
36+
name: Earl Grey for CW310
37+
uses: ./.github/workflows/bitstream.yml
38+
secrets: inherit
39+
with:
40+
top_name: earlgrey
41+
design_suffix: cw310
42+
43+
chip_earlgrey_cw310_hyperdebug:
44+
name: Earl Grey for CW310 Hyperdebug
45+
uses: ./.github/workflows/bitstream.yml
46+
secrets: inherit
47+
with:
48+
top_name: earlgrey
49+
design_suffix: cw310_hyperdebug
50+
51+
chip_earlgrey_cw340:
52+
name: Earl Grey for CW340
53+
uses: ./.github/workflows/bitstream.yml
54+
secrets: inherit
55+
with:
56+
top_name: earlgrey
57+
design_suffix: cw340
58+
59+
# CW310 FPGA jobs.
60+
execute_test_rom_fpga_tests_cw310:
61+
name: CW310 Test ROM Tests
62+
needs: chip_earlgrey_cw310
63+
uses: ./.github/workflows/fpga.yml
64+
secrets: inherit
65+
with:
66+
job_name: execute_test_rom_fpga_tests_cw310
67+
bitstream: chip_earlgrey_cw310
68+
board: cw310
69+
interface: cw310
70+
tag_filters: "cw310_test_rom,-coverage_broken"
71+
mode: coverage
72+
73+
execute_rom_fpga_tests_cw310:
74+
name: CW310 ROM Tests
75+
needs: chip_earlgrey_cw310
76+
uses: ./.github/workflows/fpga.yml
77+
secrets: inherit
78+
with:
79+
job_name: execute_rom_fpga_tests_cw310
80+
bitstream: chip_earlgrey_cw310
81+
board: cw310
82+
interface: cw310
83+
tag_filters: "cw310_rom_with_fake_keys,cw310_rom_with_real_keys,-manuf,-coverage_broken"
84+
timeout: 90
85+
mode: coverage
86+
87+
execute_rom_ext_fpga_tests_cw310:
88+
name: CW310 ROM_EXT Tests
89+
needs: chip_earlgrey_cw310
90+
uses: ./.github/workflows/fpga.yml
91+
secrets: inherit
92+
with:
93+
job_name: execute_rom_ext_fpga_tests_cw310
94+
bitstream: chip_earlgrey_cw310
95+
board: cw310
96+
interface: cw310
97+
tag_filters: "cw310_rom_ext,-coverage_broken"
98+
mode: coverage
99+
100+
execute_sival_fpga_tests_cw310:
101+
name: CW310 SiVal Tests
102+
needs: chip_earlgrey_cw310_hyperdebug
103+
uses: ./.github/workflows/fpga.yml
104+
secrets: inherit
105+
with:
106+
job_name: execute_sival_fpga_tests_cw310
107+
bitstream: chip_earlgrey_cw310_hyperdebug
108+
board: cw310
109+
interface: hyper310
110+
tag_filters: "cw310_sival,-manuf,-coverage_broken"
111+
mode: coverage
112+
113+
execute_sival_rom_ext_fpga_tests_cw310:
114+
name: CW310 SiVal ROM_EXT Tests
115+
needs: chip_earlgrey_cw310_hyperdebug
116+
uses: ./.github/workflows/fpga.yml
117+
secrets: inherit
118+
with:
119+
job_name: execute_sival_rom_ext_fpga_tests_cw310
120+
bitstream: chip_earlgrey_cw310_hyperdebug
121+
board: cw310
122+
interface: hyper310
123+
tag_filters: "cw310_sival_rom_ext,-coverage_broken"
124+
mode: coverage
125+
126+
execute_manuf_fpga_tests_cw310:
127+
name: CW310 Manufacturing Tests
128+
needs: chip_earlgrey_cw310_hyperdebug
129+
uses: ./.github/workflows/fpga.yml
130+
secrets: inherit
131+
with:
132+
job_name: execute_manuf_fpga_tests_cw310
133+
bitstream: chip_earlgrey_cw310_hyperdebug
134+
board: cw310
135+
interface: hyper310
136+
tag_filters: "manuf,-cw340,-qemu,-coverage_broken"
137+
mode: coverage
138+
139+
execute_fpga_rom_ext_tests_hyper310:
140+
name: Hyper310 ROM_EXT Tests
141+
needs: chip_earlgrey_cw310_hyperdebug
142+
uses: ./.github/workflows/fpga.yml
143+
secrets: inherit
144+
with:
145+
job_name: execute_fpga_rom_ext_tests_hyper310
146+
bitstream: chip_earlgrey_cw310_hyperdebug
147+
board: cw310
148+
interface: hyper310
149+
tag_filters: "hyper310_rom_ext,-coverage_broken"
150+
mode: coverage
151+
152+
# CW340 FPGA jobs
153+
execute_test_rom_fpga_tests_cw340:
154+
name: CW340 Test ROM Tests
155+
needs: chip_earlgrey_cw340
156+
uses: ./.github/workflows/fpga.yml
157+
secrets: inherit
158+
with:
159+
job_name: execute_test_rom_fpga_tests_cw340
160+
bitstream: chip_earlgrey_cw340
161+
board: cw340
162+
interface: cw340
163+
tag_filters: "cw340_test_rom,-coverage_broken"
164+
mode: coverage
165+
166+
execute_rom_fpga_tests_cw340:
167+
name: CW340 ROM Tests
168+
needs: chip_earlgrey_cw340
169+
uses: ./.github/workflows/fpga.yml
170+
secrets: inherit
171+
with:
172+
job_name: execute_rom_fpga_tests_cw340
173+
bitstream: chip_earlgrey_cw340
174+
board: cw340
175+
interface: cw340
176+
tag_filters: "cw340_rom_with_fake_keys,cw340_rom_with_real_keys,-manuf,-coverage_broken"
177+
mode: coverage
178+
179+
execute_rom_ext_fpga_tests_cw340:
180+
name: CW340 ROM_EXT Tests
181+
needs: chip_earlgrey_cw340
182+
uses: ./.github/workflows/fpga.yml
183+
secrets: inherit
184+
with:
185+
job_name: execute_rom_ext_fpga_tests_cw340
186+
bitstream: chip_earlgrey_cw340
187+
board: cw340
188+
interface: cw340
189+
tag_filters: "cw340_rom_ext,-coverage_broken"
190+
mode: coverage
191+
192+
execute_sival_fpga_tests_cw340:
193+
name: CW340 SiVal Tests
194+
needs: chip_earlgrey_cw340
195+
uses: ./.github/workflows/fpga.yml
196+
secrets: inherit
197+
with:
198+
job_name: execute_sival_fpga_tests_cw340
199+
bitstream: chip_earlgrey_cw340
200+
board: cw340
201+
interface: cw340
202+
tag_filters: "cw340_sival,-manuf,-coverage_broken"
203+
mode: coverage
204+
205+
execute_sival_rom_ext_fpga_tests_cw340:
206+
name: CW340 SiVal ROM_EXT Tests
207+
needs: chip_earlgrey_cw340
208+
uses: ./.github/workflows/fpga.yml
209+
secrets: inherit
210+
with:
211+
job_name: execute_sival_rom_ext_fpga_tests_cw340
212+
bitstream: chip_earlgrey_cw340
213+
board: cw340
214+
interface: cw340
215+
tag_filters: "cw340_sival_rom_ext,-coverage_broken"
216+
mode: coverage
217+
218+
execute_manuf_fpga_tests_cw340:
219+
name: CW340 Manufacturing Tests
220+
needs: chip_earlgrey_cw340
221+
uses: ./.github/workflows/fpga.yml
222+
secrets: inherit
223+
with:
224+
job_name: execute_manuf_fpga_tests_cw340
225+
bitstream: chip_earlgrey_cw340
226+
board: cw340
227+
interface: cw340
228+
tag_filters: "manuf,-hyper310,-qemu,-coverage_broken"
229+
mode: coverage
230+
231+
coverage_quick:
232+
name: Coverage (quick)
233+
uses: ./.github/workflows/coverage-quick.yml
234+
secrets: inherit
235+
236+
publish_coverage_report:
237+
name: Publish Coverage Report
238+
needs:
239+
- execute_test_rom_fpga_tests_cw310
240+
- execute_rom_fpga_tests_cw310
241+
- execute_rom_ext_fpga_tests_cw310
242+
- execute_sival_fpga_tests_cw310
243+
- execute_sival_rom_ext_fpga_tests_cw310
244+
- execute_manuf_fpga_tests_cw310
245+
- execute_fpga_rom_ext_tests_hyper310
246+
- execute_test_rom_fpga_tests_cw340
247+
- execute_rom_fpga_tests_cw340
248+
- execute_rom_ext_fpga_tests_cw340
249+
- execute_sival_fpga_tests_cw340
250+
- execute_sival_rom_ext_fpga_tests_cw340
251+
- execute_manuf_fpga_tests_cw340
252+
- coverage_quick
253+
if: success() || failure()
254+
uses: ./.github/workflows/publish-coverage-report.yml
255+
secrets: inherit
256+
with:
257+
branch: ${{ inputs.branch || 'earlgrey_1.0.0' }}

‎.github/workflows/ci.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -765,3 +765,19 @@ jobs:
765765
with:
766766
name: qemu-override-build-logs
767767
path: qemu/build/*.log
768+
769+
coverage_quick:
770+
name: Coverage (quick)
771+
needs: quick_lint
772+
uses: ./.github/workflows/coverage-quick.yml
773+
secrets: inherit
774+
775+
publish_coverage_report:
776+
name: Publish Coverage Report
777+
needs:
778+
- coverage_quick
779+
if: success() || failure()
780+
uses: ./.github/workflows/publish-coverage-report.yml
781+
secrets: inherit
782+
with:
783+
branch: ${{ inputs.branch || 'earlgrey_1.0.0' }}

‎.github/workflows/coverage-quick.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# Copyright lowRISC contributors (OpenTitan project).
2+
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
name: Coverage (quick)
6+
on:
7+
workflow_call:
8+
9+
jobs:
10+
otbn_crypto_tests:
11+
name: OTBN Crypto Tests
12+
runs-on: ubuntu-22.04
13+
timeout-minutes: 60
14+
steps:
15+
- uses: actions/checkout@v4
16+
with:
17+
fetch-depth: 0 # Bitstream cache requires all commits.
18+
- name: Prepare environment
19+
uses: ./.github/actions/prepare-env
20+
with:
21+
service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}'
22+
- name: Execute tests
23+
run: |
24+
./bazelisk.sh coverage --config=ot_coverage --test_tag_filters=-nightly \
25+
//sw/otbn/crypto/...
26+
- name: Publish Bazel test results
27+
uses: ./.github/actions/publish-bazel-test-results
28+
if: ${{ !cancelled() }}
29+
with:
30+
artifact-name: otbn-test-results
31+
32+
sw_build_test:
33+
name: Unit Tests
34+
runs-on: ubuntu-22.04-vivado
35+
steps:
36+
- uses: actions/checkout@v4
37+
with:
38+
fetch-depth: 0 # Required for bitstream cache to work.
39+
- name: Prepare environment
40+
uses: ./.github/actions/prepare-env
41+
with:
42+
service_account_json: '${{ secrets.BAZEL_CACHE_CREDS }}'
43+
- name: Check Bazel build graph
44+
run: |
45+
# Test the graph with both an empty and filled bitstream cache.
46+
./ci/scripts/test-empty-bitstream-cache.sh
47+
./bazelisk.sh build --nobuild //...
48+
- name: Select software targets
49+
run: |
50+
target_pattern_file="$(mktemp)"
51+
echo "target_pattern_file=${target_pattern_file}" >> "$GITHUB_ENV"
52+
53+
# Start with building the whole graph.
54+
echo '//sw/...' > "$target_pattern_file"
55+
# Exclude some targets:
56+
#
57+
# 1. `//hw/...` is out of scope.
58+
# 2. `//quality/...` is tested by the lint jobs.
59+
# 3. `//sw/otbn/crypto/...` is tested by the OTBN job.
60+
# 4. `//third_party/...` which is not our code.
61+
# 5. `//sw/host/...` FIXME: rust coverage are broken on CI.
62+
printf "%s\n" \
63+
"-//hw/..." \
64+
"-//quality/..." \
65+
"-//sw/otbn/crypto/..." \
66+
"-//third_party/..." \
67+
"-//sw/host/..." \
68+
>> "$target_pattern_file"
69+
# Exclude anything that requires a bitstream splice.
70+
./bazelisk.sh cquery \
71+
--noinclude_aspects \
72+
--output=starlark \
73+
--starlark:expr='"-{}".format(target.label)' \
74+
--define DISABLE_VERILATOR_BUILD=true \
75+
-- "rdeps(//..., kind(bitstream_splice, //...))" \
76+
>> "$target_pattern_file"
77+
- name: Run software unit tests coverage
78+
run: |
79+
./bazelisk.sh coverage \
80+
--build_tests_only \
81+
--keep_going \
82+
--config=ot_coverage \
83+
--test_output=errors \
84+
--define DISABLE_VERILATOR_BUILD=true \
85+
--target_pattern_file="$target_pattern_file" \
86+
--test_tag_filters=-broken,-coverage_broken,-cw310,-verilator,-dv,-silicon,-qemu
87+
- name: Publish Bazel test results
88+
uses: ./.github/actions/publish-bazel-test-results
89+
if: ${{ !cancelled() }}
90+
with:
91+
artifact-name: sw_build_test-test-results
92+
- name: Check for unrunnable tests
93+
run: ./ci/scripts/check-unrunnable-tests.sh
94+
continue-on-error: true

0 commit comments

Comments
 (0)