feat: include update_hardware_registry command on crontab (#1916)

* Added an option to read an index of registries on
     update_hardware_registry
   * Added option to read urls as update_hardware_registry entries
   * Add to dashboard cronjob list

    Closes #1899

Signed-off-by: Alan Peixinho <alan.peixinho@profusion.mobi>

Author: alanpeixinho

backend/kernelCI/settings.py

@@ -150,6 +150,14 @@ def get_json_env_var(name, default):
 }
 """Maps monitoring_id to the relative_path that will be appended to the base healthcheck URL."""
 
+DEFAULT_HARDWARE_REGISTRY_INDEX_URL = (
+    "https://raw.githubusercontent.com/kernelci/kernelci-pipeline/main/"
+    "config/hardware_registry/index.yaml"
+)
+HARDWARE_REGISTRY_INDEX_URL = os.environ.get(
+    "HARDWARE_REGISTRY_INDEX_URL", DEFAULT_HARDWARE_REGISTRY_INDEX_URL
+)
+
 SKIP_CRONJOBS = is_boolean_or_string_true(os.environ.get("SKIP_CRONJOBS", False))
 if SKIP_CRONJOBS:
     CRONJOBS = []
@@ -236,6 +244,14 @@ def get_json_env_var(name, default):
                 "--yes",
             ],
         ),
+        (
+            "0 3 * * *",
+            "django.core.management.call_command",
+            [
+                "update_hardware_registry",
+                f"--index={HARDWARE_REGISTRY_INDEX_URL}",
+            ],
+        ),
     ]
 
 # Email settings for SMTP backend

backend/kernelCI_app/management/commands/update_hardware_registry.py

@@ -4,10 +4,12 @@
 
 from pathlib import Path
 from typing import Any
+from urllib.parse import urljoin, urlparse
 
 import yaml
 from django.core.management.base import BaseCommand, CommandError
 
+import requests
 from kernelCI_app.models import (
     HardwareRegistryPlatform,
     HardwareRegistryPlatformVendor,
@@ -34,57 +36,105 @@ def get_update_fields(model):
 )
 
 
+def _is_url(value: str) -> bool:
+    parsed = urlparse(value)
+    return parsed.scheme in ("http", "https")
+
+
 class Command(BaseCommand):
     help = (
         "Parse a KernelCI hardware registry YAML file (same shape as "
-        "kernelci-pipeline config/hardware_registry/*.yaml) and print its data."
+        "kernelci-pipeline config/hardware_registry/*.yaml) and print its data. "
+        "Accepts a local file path or a URL."
     )
 
     def add_arguments(self, parser):
         parser.add_argument(
             "registry_file",
+            nargs="?",
+            type=str,
+            help="Path or URL to a single hardware registry YAML file.",
+        )
+        parser.add_argument(
+            "--index",
             type=str,
-            help="Path to the hardware registry YAML file.",
+            help=(
+                "Path or URL to an index YAML listing registry files "
+                "(e.g. hardware_registry/index.yaml). Entries in the "
+                "'registries' key are resolved relative to the index location."
+            ),
         )
 
-    def load_yaml(self, path: str) -> dict:
+    def _fetch_url(self, url: str) -> str:
+        try:
+            response = requests.get(url, timeout=30)
+            response.raise_for_status()
+            return response.text
+        except requests.RequestException as exc:
+            raise CommandError(f"Failed to fetch {url}: {exc}") from exc
+
+    def _read_file(self, path: Path) -> str:
         if not path.is_file():
             raise CommandError(f"Registry file not found: {path}")
         try:
-            raw = path.read_text(encoding="utf-8")
+            return path.read_text(encoding="utf-8")
         except OSError as exc:
             raise CommandError(f"Cannot read registry file: {exc}") from exc
 
+    def load_yaml(self, source: str) -> dict:
+        if _is_url(source):
+            raw = self._fetch_url(source)
+        else:
+            raw = self._read_file(Path(source).expanduser().resolve())
+
         try:
-            data = yaml.safe_load(raw)
-            return data
+            return yaml.safe_load(raw)
         except yaml.YAMLError as exc:
             raise CommandError(f"Invalid YAML: {exc}") from exc
 
-    def handle(self, *args: Any, registry_file: str, **options: Any) -> None:
-        path = Path(registry_file).expanduser().resolve()
-        data = self.load_yaml(path)
+    def _resolve_relative(self, base: str, relative: str) -> str:
+        """Resolve a relative filename against the base index location."""
+        if _is_url(base):
+            return urljoin(base, relative)
+        return str(Path(base).expanduser().resolve().parent / relative)
+
+    def _resolve_index(self, index_source: str) -> list[str]:
+        """Load an index YAML and return resolved paths/URLs for each registry."""
+        data = self.load_yaml(index_source)
+        if not isinstance(data, dict) or "registries" not in data:
+            raise CommandError(
+                "Index YAML must contain a 'registries' key with a list of filenames."
+            )
+        entries = data["registries"]
+        if not isinstance(entries, list) or not entries:
+            raise CommandError("Index 'registries' must be a non-empty list.")
+        return [self._resolve_relative(index_source, entry) for entry in entries]
+
+    def _process_registry(self, source: str) -> None:
+        """Load and persist a single registry YAML."""
+        data = self.load_yaml(source)
 
         if data is None:
-            raise CommandError("Registry file is empty or parses to null.")
+            raise CommandError(f"Registry is empty or parses to null: {source}")
         if not isinstance(data, dict):
             raise CommandError(
-                f"Expected a YAML mapping at root, got {type(data).__name__}."
+                f"Expected a YAML mapping at root, got {type(data).__name__}: {source}"
             )
 
         missing = [k for k in EXPECTED_SECTIONS if k not in data]
         if missing:
             self.stderr.write(
                 self.style.WARNING(
-                    f"Missing typical hardware-registry sections : {', '.join(missing)}"
+                    f"Missing typical hardware-registry sections in {source}: "
+                    f"{', '.join(missing)}"
                 )
             )
 
-        silicon_vendors = data["silicon_vendors"] or []
-        platform_vendors = data["platform_vendors"] or []
-        processors = data["processors"] or []
-        system_modules = data["system_modules"] or []
-        platforms = data["platforms"] or []
+        silicon_vendors = data["silicon_vendors"] or {}
+        platform_vendors = data["platform_vendors"] or {}
+        processors = data["processors"] or {}
+        system_modules = data["system_modules"] or {}
+        platforms = data["platforms"] or {}
 
         HardwareRegistrySiliconVendor.objects.bulk_create(
             [HardwareRegistrySiliconVendor(**sv) for sv in silicon_vendors.values()],
@@ -120,3 +170,22 @@ def handle(self, *args: Any, registry_file: str, **options: Any) -> None:
             unique_fields=["id"],
             update_fields=get_update_fields(HardwareRegistryPlatform),
         )
+
+        self.stdout.write(self.style.SUCCESS(f"Processed: {source}"))
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        registry_file = options.get("registry_file")
+        index = options.get("index")
+
+        if not registry_file and not index:
+            raise CommandError("Provide either a registry_file or --index.")
+        if registry_file and index:
+            raise CommandError("Provide either a registry_file or --index, not both.")
+
+        if index:
+            sources = self._resolve_index(index)
+        else:
+            sources = [registry_file]
+
+        for source in sources:
+            self._process_registry(source)