|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# ******************************************************************************* |
| 3 | +# Copyright (c) 2026 Contributors to the Eclipse Foundation |
| 4 | +# |
| 5 | +# See the NOTICE file(s) distributed with this work for additional |
| 6 | +# information regarding copyright ownership. |
| 7 | +# |
| 8 | +# This program and the accompanying materials are made available under the |
| 9 | +# terms of the Apache License Version 2.0 which is available at |
| 10 | +# https://www.apache.org/licenses/LICENSE-2.0 |
| 11 | +# |
| 12 | +# SPDX-License-Identifier: Apache-2.0 |
| 13 | +# ******************************************************************************* |
| 14 | +""" |
| 15 | +Aggregate Stage 1 and Stage 2 quality reports into a single consolidated report. |
| 16 | +
|
| 17 | +Implements the upstream aggregation step of DR-008 Option 4: reads per-module |
| 18 | +unit_test_summary.md and coverage_summary.md from downloaded stage2-report-* |
| 19 | +artifacts and combines them with the Stage 1 integration result. |
| 20 | +
|
| 21 | +Usage: |
| 22 | + python3 scripts/aggregate_quality_report.py \\ |
| 23 | + --stage1-result success \\ |
| 24 | + --stage2-result success \\ |
| 25 | + --stage2-dir _stage2_reports/ \\ |
| 26 | + >> "$GITHUB_STEP_SUMMARY" |
| 27 | +
|
| 28 | +The --stage2-dir is expected to contain subdirectories named |
| 29 | +stage2-report-<module>, each holding unit_test_summary.md and |
| 30 | +coverage_summary.md produced by quality_runners.py. |
| 31 | +""" |
| 32 | + |
| 33 | +import argparse |
| 34 | +import json |
| 35 | +import sys |
| 36 | +from pathlib import Path |
| 37 | + |
| 38 | +_STATUS_MAP = { |
| 39 | + "success": "✅ Success", |
| 40 | + "failure": "❌ Failure", |
| 41 | + "cancelled": "⚪ Cancelled", |
| 42 | + "skipped": "⚪ Skipped", |
| 43 | + "": "⚪ Unknown", |
| 44 | +} |
| 45 | + |
| 46 | + |
| 47 | +def _format_status(result: str) -> str: |
| 48 | + return _STATUS_MAP.get(result.lower().strip(), "⚪ Unknown") |
| 49 | + |
| 50 | + |
| 51 | +def _extract_table_data_rows(md_path: Path) -> list[str]: |
| 52 | + """Return the data rows of the first markdown table found in md_path. |
| 53 | +
|
| 54 | + Skips the title line (starts with #), the header row, and the separator |
| 55 | + row (contains ---), then collects all remaining pipe-delimited lines. |
| 56 | + """ |
| 57 | + if not md_path.exists(): |
| 58 | + return [] |
| 59 | + |
| 60 | + lines = md_path.read_text(encoding="utf-8").splitlines() |
| 61 | + data_rows: list[str] = [] |
| 62 | + header_seen = False |
| 63 | + separator_seen = False |
| 64 | + |
| 65 | + for line in lines: |
| 66 | + stripped = line.strip() |
| 67 | + if not stripped.startswith("|"): |
| 68 | + continue |
| 69 | + if not header_seen: |
| 70 | + header_seen = True |
| 71 | + continue |
| 72 | + if not separator_seen: |
| 73 | + separator_seen = True |
| 74 | + continue |
| 75 | + if stripped: |
| 76 | + data_rows.append(stripped) |
| 77 | + |
| 78 | + return data_rows |
| 79 | + |
| 80 | + |
| 81 | +def _excluded_test_targets(known_good_path: Path) -> list[tuple[str, list[str]]]: |
| 82 | + """Return [(module, [excluded targets])] for target_sw modules in known_good.json. |
| 83 | +
|
| 84 | + These targets are excluded from Stage 2 (e.g. they depend on dev_dependency-only |
| 85 | + deps) and so are absent from the consolidated report — they are still covered by |
| 86 | + each module's own CI. Surfacing them keeps the report honest about completeness. |
| 87 | + """ |
| 88 | + if not known_good_path.exists(): |
| 89 | + return [] |
| 90 | + |
| 91 | + data = json.loads(known_good_path.read_text(encoding="utf-8")) |
| 92 | + target_sw = data.get("modules", {}).get("target_sw", {}) |
| 93 | + |
| 94 | + excluded: list[tuple[str, list[str]]] = [] |
| 95 | + for name in sorted(target_sw): |
| 96 | + targets = target_sw[name].get("metadata", {}).get("exclude_test_targets", []) |
| 97 | + if targets: |
| 98 | + excluded.append((name, targets)) |
| 99 | + return excluded |
| 100 | + |
| 101 | + |
| 102 | +def main() -> int: |
| 103 | + parser = argparse.ArgumentParser( |
| 104 | + description="Aggregate Stage 1 + Stage 2 quality reports (DR-008 Option 4).", |
| 105 | + formatter_class=argparse.RawDescriptionHelpFormatter, |
| 106 | + epilog=( |
| 107 | + "Examples:\n" |
| 108 | + " python3 scripts/aggregate_quality_report.py \\\n" |
| 109 | + " --stage1-result success \\\n" |
| 110 | + " --stage2-result failure \\\n" |
| 111 | + " --stage2-dir _stage2_reports/ \\\n" |
| 112 | + " >> $GITHUB_STEP_SUMMARY\n" |
| 113 | + ), |
| 114 | + ) |
| 115 | + parser.add_argument( |
| 116 | + "--stage1-result", |
| 117 | + default="", |
| 118 | + help="GitHub Actions result of the stage1_integration job (success/failure/cancelled/skipped).", |
| 119 | + ) |
| 120 | + parser.add_argument( |
| 121 | + "--stage2-result", |
| 122 | + default="", |
| 123 | + help="GitHub Actions result of the stage2_module_validation job.", |
| 124 | + ) |
| 125 | + parser.add_argument( |
| 126 | + "--stage2-dir", |
| 127 | + type=Path, |
| 128 | + default=Path("_stage2_reports"), |
| 129 | + help="Directory containing downloaded stage2-report-* artifact subdirectories.", |
| 130 | + ) |
| 131 | + parser.add_argument( |
| 132 | + "--known-good-path", |
| 133 | + type=Path, |
| 134 | + default=Path("known_good.json"), |
| 135 | + help="Path to known_good.json (used to list test targets excluded from Stage 2).", |
| 136 | + ) |
| 137 | + args = parser.parse_args() |
| 138 | + |
| 139 | + out = sys.stdout |
| 140 | + |
| 141 | + out.write("# S-CORE Quality Report — DR-008 Option 4\n\n") |
| 142 | + |
| 143 | + # ------------------------------------------------------------------ |
| 144 | + # Stage 1 summary |
| 145 | + # ------------------------------------------------------------------ |
| 146 | + out.write("## Stage 1 — Integration Results\n\n") |
| 147 | + out.write("| Check | Status |\n") |
| 148 | + out.write("|-------|--------|\n") |
| 149 | + out.write(f"| Platform Build + Feature Integration Tests (linux-x86_64) | {_format_status(args.stage1_result)} |\n") |
| 150 | + out.write("\n") |
| 151 | + |
| 152 | + # ------------------------------------------------------------------ |
| 153 | + # Stage 2 summary — read per-module reports |
| 154 | + # ------------------------------------------------------------------ |
| 155 | + out.write("## Stage 2 — Module Validation Results\n\n") |
| 156 | + |
| 157 | + stage2_dir: Path = args.stage2_dir |
| 158 | + ut_rows: list[str] = [] |
| 159 | + cov_rows: list[str] = [] |
| 160 | + |
| 161 | + if stage2_dir.exists(): |
| 162 | + for artifact_dir in sorted(stage2_dir.iterdir()): |
| 163 | + if not artifact_dir.is_dir(): |
| 164 | + continue |
| 165 | + if not artifact_dir.name.startswith("stage2-report-"): |
| 166 | + continue |
| 167 | + ut_rows.extend(_extract_table_data_rows(artifact_dir / "unit_test_summary.md")) |
| 168 | + cov_rows.extend(_extract_table_data_rows(artifact_dir / "coverage_summary.md")) |
| 169 | + else: |
| 170 | + out.write(f"*Stage 2 reports directory not found: `{stage2_dir}`*\n\n") |
| 171 | + |
| 172 | + if ut_rows: |
| 173 | + out.write("### Unit Test Summary\n\n") |
| 174 | + out.write("| module | passed | failed | skipped | total |\n") |
| 175 | + out.write("|--------|--------|--------|---------|-------|\n") |
| 176 | + for row in ut_rows: |
| 177 | + out.write(f"{row}\n") |
| 178 | + out.write("\n") |
| 179 | + else: |
| 180 | + out.write("*No Stage 2 unit test reports found.*\n\n") |
| 181 | + |
| 182 | + if cov_rows: |
| 183 | + out.write("### Coverage Summary\n\n") |
| 184 | + out.write("| module | lines | functions | branches |\n") |
| 185 | + out.write("|--------|-------|-----------|----------|\n") |
| 186 | + for row in cov_rows: |
| 187 | + out.write(f"{row}\n") |
| 188 | + out.write("\n") |
| 189 | + |
| 190 | + # ------------------------------------------------------------------ |
| 191 | + # Excluded test targets — completeness disclosure (DR-008 Q4) |
| 192 | + # ------------------------------------------------------------------ |
| 193 | + excluded = _excluded_test_targets(args.known_good_path) |
| 194 | + if excluded: |
| 195 | + out.write("### Test Targets Excluded from Stage 2\n\n") |
| 196 | + out.write( |
| 197 | + "These targets are excluded from central Stage 2 execution (typically because " |
| 198 | + "they depend on `dev_dependency`-only deps that are not visible from the resolved " |
| 199 | + "graph). They are still validated by each module's own CI.\n\n" |
| 200 | + ) |
| 201 | + out.write("| module | excluded test target |\n") |
| 202 | + out.write("|--------|----------------------|\n") |
| 203 | + for module_name, targets in excluded: |
| 204 | + for target in targets: |
| 205 | + out.write(f"| {module_name} | `{target}` |\n") |
| 206 | + out.write("\n") |
| 207 | + |
| 208 | + # ------------------------------------------------------------------ |
| 209 | + # Overall status |
| 210 | + # ------------------------------------------------------------------ |
| 211 | + out.write("## Overall Status\n\n") |
| 212 | + stage1_ok = args.stage1_result == "success" |
| 213 | + stage2_ok = args.stage2_result in ("success", "skipped") |
| 214 | + |
| 215 | + if stage1_ok and stage2_ok: |
| 216 | + out.write("✅ All quality checks passed.\n") |
| 217 | + else: |
| 218 | + out.write("❌ One or more quality checks failed — see details above.\n\n") |
| 219 | + out.write("| Stage | Result |\n") |
| 220 | + out.write("|-------|--------|\n") |
| 221 | + out.write(f"| Stage 1 (integration) | {_format_status(args.stage1_result)} |\n") |
| 222 | + out.write(f"| Stage 2 (module validation) | {_format_status(args.stage2_result)} |\n") |
| 223 | + |
| 224 | + return 0 if (stage1_ok and stage2_ok) else 1 |
| 225 | + |
| 226 | + |
| 227 | +if __name__ == "__main__": |
| 228 | + sys.exit(main()) |
0 commit comments