coverage report workflow

- Added script .github/scripts/check_pyrefly_coverage.py to check for
regressions in type annotations by comparing baseline coverage report to
newly generated coverage report.

- Added actions workflow to automatically call type coverage regression
test.

- Added unit tests for coverage report script.

- Checked in baseline report to track current type annotation state.

Author: NiklasMueller2001

.github/scripts/check_pyrefly_coverage.py

@@ -0,0 +1,79 @@
+from argparse import ArgumentParser
+from difflib import unified_diff
+from pathlib import Path
+import pprint
+import json
+import sys
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--baseline_report_path",
+        type=str,
+        required=False,
+        default=".pyrefly-baseline-report.json",
+    )
+    parser.add_argument(
+        "--current_report_path",
+        type=str,
+        required=False,
+        default="pyrefly-current-report.json",
+    )
+    args = parser.parse_args()
+    baseline = json.loads(Path(args.baseline_report_path).read_text())
+    current = json.loads(Path(args.current_report_path).read_text())
+
+    baseline_reports = {}
+    current_reports = {}
+
+    for report in baseline["module_reports"]:
+        baseline_reports[report["name"]] = report
+
+    for report in current["module_reports"]:
+        current_reports[report["name"]] = report
+
+    failures = []
+
+    for module_name, current_module_report in current_reports.items():
+        baseline_module_report = baseline_reports.get(module_name)
+
+        # File does not exist in baseline yet
+        if baseline_module_report is None:
+            completeness = current_module_report["coverage"]
+
+            if completeness < 100:
+                failures.append(
+                    f"New file {module_name} is only " f"{completeness:.1f}% annotated"
+                )
+            continue
+
+        old_n_untpyed = baseline_module_report["n_untyped"]
+        new_n_untpyed = current_module_report["n_untyped"]
+
+        if new_n_untpyed > old_n_untpyed:
+            dict1_lines = pprint.pformat(
+                baseline_module_report, sort_dicts=True
+            ).splitlines()
+            dict2_lines = pprint.pformat(
+                current_module_report, sort_dicts=True
+            ).splitlines()
+
+            diff = unified_diff(
+                dict1_lines, dict2_lines, fromfile="dict1", tofile="dict2", lineterm=""
+            )
+
+            failures.append(
+                f"{module_name}: Untyped count increased "
+                f"from {old_n_untpyed} to {new_n_untpyed}\n"
+                f"\n{'\n'.join(diff)}"
+            )
+
+    if failures:
+        print("Pyrefly coverage regression detected:")
+
+        for failure in failures:
+            print(f"- {failure}\n\n")
+
+        sys.exit(1)
+
+    print("No pyrefly coverage regressions detected.")

.github/workflows/pyrefly_type_coverage.yaml

@@ -0,0 +1,33 @@
+name: Pyrefly Coverage Check
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  check-type-coverage:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyrefly
+
+      - name: Generate current Pyrefly coverage report
+        run: |
+          pyrefly coverage report > pyrefly-current-report.json
+
+      - name: Compare against baseline
+        run: |
+          python .github/scripts/check_pyrefly_coverage.py