Extend logic to use generation time data too

Author: pawelrutkaq

.github/workflows/test_and_docs.yml

@@ -30,6 +30,8 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: ${{ github.ref_name != 'main' && !startsWith(github.ref_name, 'release/') }}
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 jobs:
   test_and_docs:
     runs-on: ubuntu-22.04

scripts/known_good/known_good_to_workspace_metadata.py

@@ -38,9 +38,9 @@ def main():
     try:
         known_good = load_known_good(Path(args.known_good))
     except FileNotFoundError as e:
-        raise SystemExit(f"ERROR: {e}") from e
+        raise SystemExit(f"ERROR: {e}")
     except ValueError as e:
-        raise SystemExit(f"ERROR: {e}") from e
+        raise SystemExit(f"ERROR: {e}")
 
     modules = list(known_good.modules.values())
 

scripts/known_good/models/__init__.py

@@ -12,6 +12,6 @@
 # *******************************************************************************
 """Models for score reference integration tools."""
 
-from .module import Metadata, Module
+from .module import Module, Metadata
 
 __all__ = ["Module", "Metadata"]

scripts/known_good/models/known_good.py

@@ -18,7 +18,7 @@
 import json
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, Dict
 
 from .module import Module
 
@@ -30,34 +30,34 @@ class KnownGood:
     Module structure: modules = {"group1": {"module1": Module}, "group2": {"module2": Module}}
     """
 
-    modules: dict[str, dict[str, Module]]
+    modules: Dict[str, Dict[str, Module]]
     timestamp: str
 
     @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> KnownGood:
+    def from_dict(cls, data: Dict[str, Any]) -> KnownGood:
         """Create a KnownGood instance from a dictionary.
 
         Expected structure:
         {"modules": {"group1": {"score_baselibs": {...}}, "group2": {"score_logging": {...}}}}
 
         Args:
-                data: dictionary containing known_good.json data
+                data: Dictionary containing known_good.json data
 
         Returns:
                 KnownGood instance
         """
         modules_dict = data.get("modules", {})
         timestamp = data.get("timestamp", "")
 
-        parsed_modules: dict[str, dict[str, Module]] = {}
+        parsed_modules: Dict[str, Dict[str, Module]] = {}
         for group_name, group_modules in modules_dict.items():
             if isinstance(group_modules, dict):
                 modules_list = Module.parse_modules(group_modules)
                 parsed_modules[group_name] = {m.name: m for m in modules_list}
 
         return cls(modules=parsed_modules, timestamp=timestamp)
 
-    def to_dict(self) -> dict[str, Any]:
+    def to_dict(self) -> Dict[str, Any]:
         """Convert KnownGood instance to dictionary for JSON output.
 
         Returns:
@@ -70,7 +70,7 @@ def to_dict(self) -> dict[str, Any]:
 
         return {"modules": modules_output, "timestamp": self.timestamp}
 
-    def write(self, output_path: Path, *, dry_run: bool = False) -> None:
+    def write(self, output_path: Path, dry_run: bool = False) -> None:
         """Write known_good data to file or print for dry-run.
 
         Args:
@@ -89,7 +89,8 @@ def write(self, output_path: Path, *, dry_run: bool = False) -> None:
             print(output_json, end="")
             print("---- END UPDATED JSON ----")
         else:
-            output_path.write_text(output_json, encoding="utf-8")
+            with open(output_path, "w", encoding="utf-8") as f:
+                f.write(output_json)
             print(f"Successfully wrote updated known_good.json to {output_path}")
 
 
@@ -103,21 +104,22 @@ def load_known_good(path: Path) -> KnownGood:
             KnownGood instance with parsed modules
     """
 
-    try:
-        text = path.read_text(encoding="utf-8")
-        data = json.loads(text)
-    except json.JSONDecodeError as e:
-        lines = text.splitlines()
-        line = lines[e.lineno - 1] if 0 <= e.lineno - 1 < len(lines) else ""
-        pointer = " " * (e.colno - 1) + "^"
-
-        hint = ""
-        if "Expecting value" in e.msg:
-            hint = "Possible causes: trailing comma, missing value, or extra comma."
-
-        raise ValueError(
-            f"Invalid JSON at line {e.lineno}, column {e.colno}\n{line}\n{pointer}\n{e.msg}. {hint}"
-        ) from None
+    with open(path, "r", encoding="utf-8") as f:
+        text = f.read()
+        try:
+            data = json.loads(text)
+        except json.JSONDecodeError as e:
+            lines = text.splitlines()
+            line = lines[e.lineno - 1] if 0 <= e.lineno - 1 < len(lines) else ""
+            pointer = " " * (e.colno - 1) + "^"
+
+            hint = ""
+            if "Expecting value" in e.msg:
+                hint = "Possible causes: trailing comma, missing value, or extra comma."
+
+            raise ValueError(
+                f"Invalid JSON at line {e.lineno}, column {e.colno}\n{line}\n{pointer}\n{e.msg}. {hint}"
+            ) from None
 
     if not isinstance(data, dict) or not isinstance(data.get("modules"), dict):
         raise ValueError(f"Invalid known_good.json at {path} (expected object with 'modules' dict)")

scripts/known_good/models/module.py

@@ -16,7 +16,7 @@
 
 import logging
 from dataclasses import dataclass, field
-from typing import Any
+from typing import Any, Dict, List
 from urllib.parse import urlparse
 
 
@@ -37,7 +37,7 @@ class Metadata:
     langs: list[str] = field(default_factory=lambda: ["cpp", "rust"])
 
     @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> Metadata:
+    def from_dict(cls, data: Dict[str, Any]) -> Metadata:
         """Create a Metadata instance from a dictionary.
 
         Args:
@@ -53,7 +53,7 @@ def from_dict(cls, data: dict[str, Any]) -> Metadata:
             langs=data.get("langs", ["cpp", "rust"]),
         )
 
-    def to_dict(self) -> dict[str, Any]:
+    def to_dict(self) -> Dict[str, Any]:
         """Convert Metadata instance to dictionary representation.
 
         Returns:
@@ -79,30 +79,30 @@ class Module:
     pin_version: bool = False
 
     @classmethod
-    def from_dict(cls, name: str, module_data: dict[str, Any]) -> Module:
+    def from_dict(cls, name: str, module_data: Dict[str, Any]) -> Module:
         """Create a Module instance from a dictionary representation.
 
         Args:
-            name: The module name
-            module_data: Dictionary containing module configuration with keys:
-                - repo (str): Repository URL
-                - hash or commit (str): Commit hash
-                - version (str, optional): Module version (when present, hash is ignored)
-                - bazel_patches (list[str], optional): List of patch files for Bazel
-                - metadata (dict, optional): Metadata configuration
-                    Example: {
-                            "code_root_path": "path/to/code/root",
-                            "extra_test_config": [""],
-                            "exclude_test_targets": [""],
-                            "langs": ["cpp", "rust"]
-                    }
-                    If not present, uses default Metadata values.
-                - branch (str, optional): Git branch name (default: main)
-                - pin_version (bool, optional): If true, module hash is not updated
-                    to latest HEAD by update scripts (default: false)
+                name: The module name
+                module_data: Dictionary containing module configuration with keys:
+                        - repo (str): Repository URL
+                        - hash or commit (str): Commit hash
+                        - version (str, optional): Module version (when present, hash is ignored)
+                        - bazel_patches (list[str], optional): List of patch files for Bazel
+                        - metadata (dict, optional): Metadata configuration
+                                Example: {
+                                        "code_root_path": "path/to/code/root",
+                                        "extra_test_config": [""],
+                                        "exclude_test_targets": [""],
+                                        "langs": ["cpp", "rust"]
+                                }
+                                If not present, uses default Metadata values.
+                        - branch (str, optional): Git branch name (default: main)
+                        - pin_version (bool, optional): If true, module hash is not updated
+                                            to latest HEAD by update scripts (default: false)
 
         Returns:
-            Module instance
+                Module instance
         """
         repo = module_data.get("repo", "")
         # Support both 'hash' and 'commit' keys
@@ -119,14 +119,17 @@ def from_dict(cls, name: str, module_data: dict[str, Any]) -> Module:
 
         # Parse metadata - if not present or is None/empty dict, use defaults
         metadata_data = module_data.get("metadata")
-        metadata = Metadata.from_dict(metadata_data) if metadata_data is not None else Metadata()
-
-        # Enable once we are able to remove '*' in known_good.json
-        # if any("*" in target for target in metadata.exclude_test_targets):
-        #     raise Exception(
-        #         f"Module {name} has wildcard '*' in exclude_test_targets, which is not allowed. "
-        #         "Please specify explicit test targets to exclude or remove the key if no exclusions are needed."
-        #     )
+        if metadata_data is not None:
+            metadata = Metadata.from_dict(metadata_data)
+            # Enable once we are able to remove '*' in known_good.json
+            # if any("*" in target for target in metadata.exclude_test_targets):
+            #     raise Exception(
+            #         f"Module {name} has wildcard '*' in exclude_test_targets, which is not allowed. "
+            #         "Please specify explicit test targets to exclude or remove the key if no exclusions are needed."
+            #     )
+        else:
+            # If metadata key is missing, create with defaults
+            metadata = Metadata()
 
         branch = module_data.get("branch", "main")
         pin_version = module_data.get("pin_version", False)
@@ -143,7 +146,7 @@ def from_dict(cls, name: str, module_data: dict[str, Any]) -> Module:
         )
 
     @classmethod
-    def parse_modules(cls, modules_dict: dict[str, Any]) -> list[Module]:
+    def parse_modules(cls, modules_dict: Dict[str, Any]) -> List[Module]:
         """Parse modules dictionary into Module dataclass instances.
 
         Args:
@@ -184,13 +187,13 @@ def owner_repo(self) -> str:
 
         return f"{parts[0]}/{parts[1]}"
 
-    def to_dict(self) -> dict[str, Any]:
+    def to_dict(self) -> Dict[str, Any]:
         """Convert Module instance to dictionary representation for JSON output.
 
         Returns:
                 Dictionary with module configuration
         """
-        result: dict[str, Any] = {"repo": self.repo}
+        result: Dict[str, Any] = {"repo": self.repo}
         if self.version:
             result["version"] = self.version
         else:

scripts/known_good/override_known_good_repo.py

@@ -26,10 +26,12 @@
 """
 
 import argparse
-import datetime as dt
-import logging
+import os
 import re
+import datetime as dt
 from pathlib import Path
+from typing import Dict, List
+import logging
 
 from models import Module
 from models.known_good import KnownGood, load_known_good
@@ -38,7 +40,7 @@
 logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
 
 
-def parse_and_apply_overrides(modules: dict[str, Module], repo_overrides: list[str]) -> int:
+def parse_and_apply_overrides(modules: Dict[str, Module], repo_overrides: List[str]) -> int:
     """
     Parse repo override arguments and apply them to modules.
 
@@ -130,7 +132,7 @@ def parse_and_apply_overrides(modules: dict[str, Module], repo_overrides: list[s
     return overrides_applied
 
 
-def apply_overrides(known_good: KnownGood, repo_overrides: list[str]) -> KnownGood:
+def apply_overrides(known_good: KnownGood, repo_overrides: List[str]) -> KnownGood:
     """Apply repository commit overrides to the known_good data.
 
     Args:
@@ -207,25 +209,25 @@ def main() -> None:
     if args.verbose:
         logging.getLogger().setLevel(logging.DEBUG)
 
-    known_path = Path(args.known).resolve()
-    output_path = Path(args.output).resolve()
+    known_path = os.path.abspath(args.known)
+    output_path = os.path.abspath(args.output)
 
     # Load, update, and output
     logging.info(f"Loading {known_path}")
     try:
         known_good = load_known_good(known_path)
     except FileNotFoundError as e:
-        raise SystemExit(f"ERROR: {e}") from e
+        raise SystemExit(f"ERROR: {e}")
     except ValueError as e:
-        raise SystemExit(f"ERROR: {e}") from e
+        raise SystemExit(f"ERROR: {e}")
 
     if not args.module_overrides:
         parser.error("at least one --module-override is required")
 
     overrides = args.module_overrides
 
     updated_known_good = apply_overrides(known_good, overrides)
-    updated_known_good.write(Path(output_path), dry_run=args.dry_run)
+    updated_known_good.write(Path(output_path), args.dry_run)
 
 
 if __name__ == "__main__":