Merge pull request #549 from castler/js_use_llvm_cov_for_coverage_and_justification

 use llvm cov for coverage and justification

Author: castler

.bazelrc

@@ -113,7 +113,7 @@ common --check_direct_dependencies=error # Issue an error if a direct dependency
 build --test_env="GTEST_COLOR=1"
 
 # Configuration options required for quality assurance
-import %workspace%/quality/coverage.bazelrc
+import %workspace%/quality/coverage/coverage.bazelrc
 import %workspace%/quality/sanitizer/sanitizer.bazelrc
 import %workspace%/quality/static_analysis/static_analysis.bazelrc
 
@@ -128,3 +128,6 @@ build --tool_java_runtime_version=remotejdk_21
 
 # Import AI checker custom configuration
 try-import %workspace%/.bazelrc.ai_checker
+
+# Enable user-defined configs
+try-import %workspace%/user.bazelrc

.github/workflows/coverage_report.yml

@@ -60,9 +60,6 @@ jobs:
       - name: Allow linux-sandbox
         uses: ./actions/unblock_user_namespace_for_linux_sandbox
 
-      - name: Install Perl dependencies for genhtml
-        run: sudo apt-get install -y --no-install-recommends lcov
-
       - name: Run Unit Test with Coverage for C++
         id: run-coverage
         run: |

.gitignore

@@ -20,3 +20,5 @@ rust-project.json
 
 # docs-as-code
 docs/ubproject.toml
+cpp_coverage/
+__pycache__/

BUILD

@@ -17,6 +17,8 @@ load("@rules_shell//shell:sh_binary.bzl", "sh_binary")
 load("@score_tooling//:defs.bzl", "copyright_checker")
 load("//tools/lint:linters.bzl", "use_clang_tidy_targets")
 
+exports_files(["MODULE.bazel"])
+
 compile_pip_requirements(
     name = "pip_requirements",
     src = "requirements.in",

MODULE.bazel

@@ -290,18 +290,6 @@ use_repo(
 
 # We use here a pre-compiled fully static and hermetic clang_format binary
 # and not the one provided by llvm_toolchain, because the one from llvm_toolchain is not fully hermetic (and different version for now)
-###############################################################################
-# lcov deb package (provides genhtml + lcov for coverage HTML reports)
-###############################################################################
-deb = use_repo_rule("@download_utils//download/deb:defs.bzl", "download_deb")
-
-deb(
-    name = "lcov_deb",
-    dev_dependency = True,
-    integrity = "sha256-Ip14IkKavqBtkQ7mh6AXzr/6YyHpvSAZ0veMmw1+N80=",
-    urls = ["https://archive.ubuntu.com/ubuntu/pool/universe/l/lcov/lcov_2.0-4ubuntu2_all.deb"],
-)
-
 download_file = use_repo_rule("@download_utils//download/file:defs.bzl", "download_file")
 
 download_file(

quality/coverage.bazelrc

@@ -16,6 +16,5 @@ load("@rules_shell//shell:sh_binary.bzl", "sh_binary")
 sh_binary(
     name = "generate_coverage_html",
     srcs = ["generate_coverage_html.sh"],
-    data = ["@lcov_deb//:srcs"],
     visibility = ["//visibility:private"],
 )

quality/coverage/BUILD

@@ -0,0 +1,207 @@
+# Coverage Infrastructure
+
+This directory contains the tooling to generate, post-process, and report C++ code coverage for the Score Communication project using LLVM's source-based coverage instrumentation (`llvm-cov`).
+
+## Overview
+
+```
+quality/coverage/
+├── README.md                       ← You are here
+├── BUILD                           ← Bazel target for generate_coverage_html
+├── coverage.bazelrc                ← Bazel coverage configuration flags
+├── coverage_justifications.yaml    ← Central justification database
+├── generate_coverage_html.sh       ← Orchestrator script (entry point)
+└── llvm_cov/                       ← Python tools for coverage processing
+    ├── README.md                   ← Detailed tool documentation
+    ├── BUILD                       ← Bazel targets for Python tools
+    ├── merger.py                   ← Per-test coverage output generator
+    ├── reporter.py                 ← Final combined report generator
+    ├── justify.py                  ← Justification resolver
+    └── effective_coverage.py       ← HTML post-processor & effective coverage calculator
+```
+
+## Requirements
+
+The coverage pipeline was built to satisfy the following requirements:
+
+### REQ-COV-001: Native llvm-cov HTML Reports
+
+Coverage reports **must** be generated directly by `llvm-cov show` using LLVM's source-based coverage (`--experimental_use_llvm_covmap`). No intermediate LCOV-to-HTML conversion (genhtml) is used. This provides accurate source-level coverage including branch and expansion views.
+
+### REQ-COV-002: Instrumentation Filtering
+
+Only project source code under `//score/message_passing` and `//score/mw/com` shall be instrumented and reported. Tests, benchmarks, and external/third-party code must be excluded from the report.
+
+> **Note:** `--experimental_use_llvm_covmap` causes Bazel to instrument ALL targets regardless of `--instrumentation_filter`. Actual source filtering is enforced by `--ignore-filename-regex` in the merger and reporter. See `coverage.bazelrc` for details.
+
+### REQ-COV-003: Coverage Justification Infrastructure
+
+A YAML-based justification system must allow developers to "argue" non-covered lines and branches to achieve 100% effective coverage. Justified lines must:
+- Be tracked in a central YAML file with unique IDs, categories, and rationale
+- Optionally be referenced from code via `COV_JUSTIFIED` markers
+- Appear visually distinct (yellow/orange) in the HTML report
+- Be reflected in both per-file and total coverage percentages
+
+### REQ-COV-004: Effective Coverage Calculation
+
+The system must calculate and display:
+- **Raw coverage**: actual lines/branches hit ÷ total instrumented lines/branches
+- **Effective coverage**: (hit + justified) ÷ total
+
+Both line and branch effective coverage must be shown in the summary, per-file index table, and totals row.
+
+### REQ-COV-005: Stale Justification Detection
+
+Justifications for lines/branches that are actually covered by tests must be detected and reported as stale warnings, enabling cleanup.
+
+### REQ-COV-006: Template Instantiation Handling
+
+For C++ templates with multiple instantiations, a line or branch is considered "covered" if ANY instantiation covers it (consistent with llvm-cov semantics). This prevents inflated totals from repeated template expansions.
+
+### REQ-COV-007: Threshold Enforcement
+
+The pipeline must support a configurable effective coverage threshold (default: 100%) and emit a warning when coverage falls below it.
+
+## Quick Start
+
+### 1. Run Coverage Collection
+
+```bash
+# Full project
+bazel coverage //...
+
+# Specific target
+bazel coverage //score/message_passing:client_connection_test_linux
+```
+
+### 2. Generate the HTML Report
+
+```bash
+bazel run //quality/coverage:generate_coverage_html
+```
+
+This extracts the HTML report to `cpp_coverage/`, runs justification processing, and prints the coverage summary. Open the report:
+
+```bash
+xdg-open cpp_coverage/index.html
+```
+
+### 3. Create an Archive (CI)
+
+```bash
+bazel run //quality/coverage:generate_coverage_html -- --archive coverage-report
+```
+
+Creates `coverage-report.zip` containing the HTML report, LCOV data, and JUnit XML test results.
+
+## Pipeline Architecture
+
+The coverage pipeline has two phases:
+
+### Phase 1: Bazel Coverage Collection
+
+Configured by `coverage.bazelrc`, Bazel runs tests with coverage instrumentation enabled:
+
+```
+bazel coverage //...
+    │
+    ├── Per-test: merger.py (--coverage_output_generator)
+    │   • Receives .profraw files from test execution
+    │   • Merges into .profdata via llvm-profdata
+    │   • Packages profdata + metadata into a zip
+    │
+    └── Final: reporter.py (--coverage_report_generator)
+        • Merges all per-test profdata into one
+        • Runs llvm-cov show → HTML report
+        • Runs llvm-cov export → LCOV data
+        • Runs llvm-cov report → text summary
+        • Packages everything into _coverage_report.dat (zip)
+```
+
+### Phase 2: Report Extraction & Justification
+
+```
+bazel run //quality/coverage:generate_coverage_html
+    │
+    └── generate_coverage_html.sh
+        ├── Extract HTML from _coverage_report.dat → cpp_coverage/
+        ├── justify.py: YAML + code markers → manifest.json
+        ├── effective_coverage.py: Post-process HTML + calculate effective %
+        └── Print summary + threshold check
+```
+
+## Configuration
+
+### coverage.bazelrc
+
+Key settings:
+
+| Flag | Purpose |
+|------|---------|
+| `--experimental_use_llvm_covmap` | Use LLVM source-based coverage (not gcov) |
+| `--instrumentation_filter` | Documents intended scope (not enforced by Bazel with covmap) |
+| `--coverage_output_generator` | Points to `merger.py` for per-test processing |
+| `--coverage_report_generator` | Points to `reporter.py` for final aggregation |
+| `--test_env=LLVM_PROFILE_CONTINUOUS_MODE=1` | Enables profiling of abnormal terminations |
+| `-mllvm -runtime-counter-relocation` | Required for continuous-mode profiling with LLVM |
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `COVERAGE_THRESHOLD` | `100` | Minimum effective line coverage % (warning if below) |
+
+## Coverage Justifications
+
+See [`coverage_justifications.yaml`](coverage_justifications.yaml) for the justification database and [`llvm_cov/README.md`](llvm_cov/README.md) for detailed documentation of the justification tools.
+
+### Adding a Justification
+
+1. **Via YAML** — add an entry to `coverage_justifications.yaml`:
+
+```yaml
+justifications:
+  - id: my-unique-id              # kebab-case, must be unique
+    category: defensive_programming  # or: tool_false_positive, platform_specific, other
+    reason: >
+      Explanation of why these lines cannot be covered by tests.
+    locations:
+      - file: score/mw/com/impl/some_file.cpp
+        line_start: 42
+        line_end: 45
+```
+
+2. **Via code markers** — reference the ID from source (no `locations` needed in YAML):
+
+```cpp
+unreachable_code();  // COV_JUSTIFIED my-unique-id
+
+// COV_JUSTIFIED_START my-unique-id
+defensive_block();
+more_defensive_code();
+// COV_JUSTIFIED_STOP
+```
+
+Both methods can be combined. A justification covers both the line and any branches on that line.
+
+We strongly suggest though to use the in-code marker where possible, as this better supports refactorings and avoids
+better that justifications get outdated.
+
+### Justification Categories
+
+| Category | Use Case |
+|----------|----------|
+| `defensive_programming` | Unreachable code kept as safety guard (e.g., default case in exhaustive switch) |
+| `tool_false_positive` | Coverage tool incorrectly marks line as uncovered |
+| `platform_specific` | Code path only reachable on platforms not under test |
+| `other` | Any other valid reason |
+
+### Visual Indicators in HTML Report
+
+| Color | Meaning |
+|-------|---------|
+| **Green** | Covered by tests |
+| **Red** | Not covered (needs tests or justification) |
+| **Yellow/Orange** | Justified — not covered but argued with rationale |
+
+The index page shows a banner with overall effective coverage and updates per-file percentages in the table to reflect justifications.

quality/coverage/README.md

@@ -0,0 +1,50 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+# NOTE: --experimental_use_llvm_covmap (required for llvm-cov) causes Bazel to instrument
+# ALL targets regardless of --instrumentation_filter. The actual source filtering happens
+# in the merger/reporter via --ignore-filename-regex. The instrumentation_filter is kept
+# for documentation purposes and in case this Bazel limitation is fixed in the future.
+coverage --nocache_test_results
+coverage --cxxopt=-O0
+coverage --instrumentation_filter="^//score/message_passing[/:],^//score/mw/com/(impl|gateway|dependability|design|example|mocking|doc)"
+coverage --experimental_generate_llvm_lcov
+coverage --experimental_use_llvm_covmap
+coverage --combined_report=lcov
+coverage --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux
+coverage --extra_toolchains=@ferrocene_x86_64_unknown_linux_gnu_llvm//:rust_ferrocene_toolchain
+
+# Use llvm-cov directly for HTML report generation instead of genhtml/lcov.
+# The custom merger (per-test) receives profraw files and produces a zip with profdata + HTML.
+# The custom reporter (final) merges all profdata and generates the combined HTML report.
+coverage --coverage_output_generator=//quality/coverage/llvm_cov:merger
+coverage --coverage_report_generator=//quality/coverage/llvm_cov:reporter_wrapper
+coverage --experimental_fetch_all_coverage_outputs
+
+# Override GENERATE_LLVM_LCOV to keep raw profraw files instead of converting to LCOV.
+# The custom merger handles profraw→profdata→HTML directly.
+coverage --test_env=GENERATE_LLVM_LCOV=0
+# Suppress the default gcov path since we use llvm-cov directly.
+coverage --test_env=COVERAGE_GCOV_PATH=/usr/bin/true
+
+# These compile time options are required to cover abnormal termination cases. In GCC one can use `__gcc_dump()`, but this does not work with LLVM
+# LLVM provided these compile-time options in combination with a specific profile setting which is enabled in bazel via `LLVM_PROFILE_CONTINUOUS_MODE`
+coverage --test_env=LLVM_PROFILE_CONTINUOUS_MODE=1
+coverage --cxxopt -mllvm
+coverage --cxxopt -runtime-counter-relocation
+
+# By default Bazel creates for each library a *.so for its tests. If there is a header that is used by multiple *.so files
+# and these *.so files have different instrumentation (production code has, test has not) a conflict arises which one to use when calculating coverage.
+# Then the first object is used, and it is pure luck if it contains the correct data or not. Even worse, small changes can change the order and then lead
+# to big coverage gaps. All these problems do not arise if no dynamic libs are used. Thus, we rather take bigger build times and binaries in advance for correct data.
+coverage --dynamic_mode=off