refactor: unversion tree and hardware listing flows (#1923)

* Promote the v2 tree and hardware listing implementations to unversioned routes and APIs
  * Remove legacy v1 endpoints, pages, feature flags.
  * Remove orphaned checkout-cache code.

 Closes #1917

Signed-off-by: Alan Peixinho <alan.peixinho@profusion.mobi>

Author: alanpeixinho

backend/kernelCI/settings.py

@@ -167,9 +167,6 @@ def get_json_env_var(name, default):
         "CRONTAB_COMMAND_SUFFIX", ">> /proc/1/fd/1 2>&1"
     )
     CRONJOBS = [
-        # not using a monitoring_id in the first task since it should
-        # be removed once the denormalization is set in stone
-        ("0 * * * *", "kernelCI_app.tasks.update_checkout_cache"),
         (
             "59 * * * *",
             "django.core.management.call_command",

backend/kernelCI_app/management/commands/seed_test_data.py

@@ -9,15 +9,19 @@
 
 from kernelCI_app.constants.general import UNKNOWN_STRING
 from kernelCI_app.helpers.system import get_running_instance
+from kernelCI_app.management.commands.helpers.aggregation_helpers import simplify_status
 from kernelCI_app.management.commands.helpers.process_pending_helpers import (
     accumulate_rollup_entry,
     extract_path_group,
 )
 from kernelCI_app.models import (
     Builds,
     Checkouts,
+    HardwareStatus,
     Incidents,
     Issues,
+    LatestCheckout,
+    SimplifiedStatusChoices,
     StatusChoices,
     Tests,
     TreeTestsRollup,
@@ -31,6 +35,13 @@
 )
 from kernelCI_app.tests.factories.mocks import Build, Issue, Test
 
+STATUS_FIELD_BY_SIMPLIFIED_STATUS = {
+    SimplifiedStatusChoices.PASS: "pass",
+    SimplifiedStatusChoices.FAIL: "failed",
+    SimplifiedStatusChoices.INCONCLUSIVE: "inc",
+    None: "inc",
+}
+
 
 class Command(BaseCommand):
     help = "Seed test database with realistic data for integration tests"
@@ -74,6 +85,8 @@ def handle(self, *args, **options):
             issues = self.create_issues(count=options["issues"])
             incidents = self.create_incidents(issues=issues, builds=builds, tests=tests)
             rollup_rows = self.create_tests_rollup(tests=tests, incidents=incidents)
+            latest_checkouts = self.create_latest_checkouts(checkouts=checkouts)
+            hardware_rows = self.create_hardware_status(tests=tests)
 
         self.stdout.write(
             self.style.SUCCESS(
@@ -84,6 +97,8 @@ def handle(self, *args, **options):
                 f"- {len(issues)} issues\n"
                 f"- {len(incidents)} incidents\n"
                 f"- {len(rollup_rows)} tree_tests_rollup rows\n"
+                f"- {len(latest_checkouts)} latest_checkout rows\n"
+                f"- {len(hardware_rows)} hardware_status rows\n"
             )
         )
 
@@ -112,6 +127,8 @@ def _validate_clear_operation(self, *, skip_confirmation: bool) -> None:
 
     def clear_data(self) -> None:
         """Clear existing test data."""
+        HardwareStatus.objects.all().delete()
+        LatestCheckout.objects.all().delete()
         TreeTestsRollup.objects.all().delete()
         Incidents.objects.all().delete()
         Issues.objects.all().delete()
@@ -335,3 +352,90 @@ def create_tests_rollup(
 
         created = TreeTestsRollup.objects.bulk_create(rollup_objects)
         return created
+
+    def create_latest_checkouts(
+        self, *, checkouts: list[Checkouts]
+    ) -> list[LatestCheckout]:
+        """Create latest_checkout rows used by listing queries."""
+        self.stdout.write("Creating latest_checkout rows...")
+
+        latest_by_tree: dict[tuple, Checkouts] = {}
+        for checkout in checkouts:
+            key = (
+                checkout.origin,
+                checkout.tree_name,
+                checkout.git_repository_url,
+                checkout.git_repository_branch,
+            )
+            current = latest_by_tree.get(key)
+            if current is None or checkout.start_time > current.start_time:
+                latest_by_tree[key] = checkout
+
+        latest_checkouts = [
+            LatestCheckout(
+                checkout_id=checkout.id,
+                start_time=checkout.start_time,
+                origin=checkout.origin,
+                tree_name=checkout.tree_name,
+                git_repository_url=checkout.git_repository_url,
+                git_repository_branch=checkout.git_repository_branch,
+            )
+            for checkout in latest_by_tree.values()
+        ]
+
+        return LatestCheckout.objects.bulk_create(latest_checkouts)
+
+    def create_hardware_status(self, *, tests: list[Tests]) -> list[HardwareStatus]:
+        """Aggregate seeded tests into hardware_status rows."""
+        self.stdout.write("Creating hardware_status aggregations...")
+
+        hardware_data: dict[tuple, dict] = {}
+        counted_builds_by_key: dict[tuple, set[str]] = {}
+
+        for test in tests:
+            platform = (
+                test.environment_misc.get("platform") if test.environment_misc else None
+            )
+            if platform is None:
+                continue
+
+            checkout = test.build.checkout
+            key = (test.origin, platform, checkout.id)
+            if key not in hardware_data:
+                hardware_data[key] = {
+                    "checkout_id": checkout.id,
+                    "test_origin": test.origin,
+                    "platform": platform,
+                    "compatibles": test.environment_compatible,
+                    "start_time": checkout.start_time,
+                    "build_pass": 0,
+                    "build_failed": 0,
+                    "build_inc": 0,
+                    "boot_pass": 0,
+                    "boot_failed": 0,
+                    "boot_inc": 0,
+                    "test_pass": 0,
+                    "test_failed": 0,
+                    "test_inc": 0,
+                }
+                counted_builds_by_key[key] = set()
+
+            record = hardware_data[key]
+            status_type = STATUS_FIELD_BY_SIMPLIFIED_STATUS[
+                simplify_status(test.status)
+            ]
+
+            test_prefix = "boot" if (test.path or "").startswith("boot") else "test"
+            record[f"{test_prefix}_{status_type}"] += 1
+
+            if test.build_id not in counted_builds_by_key[
+                key
+            ] and not test.build_id.startswith("maestro:dummy_"):
+                counted_builds_by_key[key].add(test.build_id)
+                build_status_type = STATUS_FIELD_BY_SIMPLIFIED_STATUS[
+                    simplify_status(test.build.status)
+                ]
+                record[f"build_{build_status_type}"] += 1
+
+        hardware_rows = [HardwareStatus(**data) for data in hardware_data.values()]
+        return HardwareStatus.objects.bulk_create(hardware_rows)

backend/kernelCI_app/queries/hardware.py

@@ -109,98 +109,6 @@ def _get_hardware_listing_count_clauses() -> str:
     return build_count_clause + boot_count_clause + test_count_clause
 
 
-def get_hardware_listing_data(
-    *,
-    start_date: datetime,
-    end_date: datetime,
-    origin: str,
-    commits_list: Optional[list[str]] = None,
-) -> list[dict]:
-    """
-    Retrieves the listing of platform, compatibles, and
-    the status counts of builds, boots and tests
-    for the latest checkout of every tree.
-    The selected checkouts and tests are limited to the start_date/end_date interval.
-    When commits_list is set, tree heads are not used: checkouts qualify if
-    git_commit_hash is in the token list or git_commit_tags overlaps it (comma-
-    separated request values can be full SHAs or tag strings stored on checkouts).
-    Still scoped by the test start_time and origin filters below.
-    """
-
-    count_clauses = _get_hardware_listing_count_clauses()
-
-    params = {
-        "start_date": start_date,
-        "end_date": end_date,
-        "origin": origin,
-    }
-
-    if commits_list:
-        params["commits_list"] = commits_list
-        checkout_ids_select = """
-            SELECT C.id
-            FROM checkouts C
-            WHERE C.git_commit_hash = ANY(%(commits_list)s)
-               OR (
-                   C.git_commit_tags IS NOT NULL
-                   AND C.git_commit_tags && %(commits_list)s::text[]
-               )
-            """
-    else:
-        checkout_ids_select = _get_hardware_tree_heads_clause(id_only=True)
-
-    selected_checkouts_cte = f"""
-            selected_checkouts AS (
-                {checkout_ids_select}
-            ),
-        """
-
-    # The grouping by platform and compatibles is possible because a platform
-    # can dictate the array of compatibles, meaning that if the array of compatibles
-    # is different, then the platform should/must be different as well.
-    #
-    # It is possible that a platform appears for tests from origin A which are related
-    # to checkouts from origin B. It's for those cases that the origin filter is applied
-    # to the tests, not checkouts. There are no platforms being tested by multiple origins yet.
-    query = f"""
-        WITH
-            {selected_checkouts_cte}
-            relevant_tests AS (
-                SELECT
-                    "tests"."environment_compatible" AS hardware,
-                    "tests"."environment_misc" ->> 'platform' AS platform,
-                    "tests"."status",
-                    "tests"."path",
-                    "tests"."id",
-                    b.id AS build_id,
-                    b.status AS build_status
-                FROM
-                    tests
-                    INNER JOIN builds b ON tests.build_id = b.id
-                    JOIN selected_checkouts AC ON b.checkout_id = AC.id
-                WHERE
-                    "tests"."environment_misc" ->> 'platform' IS NOT NULL
-                    AND "tests"."origin" = %(origin)s
-                    AND "tests"."start_time" >= %(start_date)s
-                    AND "tests"."start_time" <= %(end_date)s
-            )
-        -- From the raw rows, selects platform, hardware, and performs the status grouping.
-        SELECT
-            platform,
-            hardware,
-            {count_clauses}
-        FROM
-            relevant_tests
-        GROUP BY
-            platform,
-            hardware
-    """
-
-    with connection.cursor() as cursor:
-        cursor.execute(query, params)
-        return cursor.fetchall()
-
-
 def get_hardware_selectors(origin: str) -> list[dict]:
     cache_key = "hardwareSelectors"
     cache_params = {"origin": origin}

backend/kernelCI_app/queries/notifications.py

@@ -368,7 +368,7 @@ def kcidb_last_test_without_issue(issue, incident):
     return kcidb_execute_query(query, params)
 
 
-# Similar to get_tree_listing_data, but at the same time it has to be different.
+# Similar to the tree listing summary query, but with notification-specific filters.
 # Only the "with", "join" and "where" clauses change
 def get_checkout_summary_data(
     *,