fix(hardware): use hardware_status window for listing and details trees

backend/kernelCI_app/queries/hardware.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Optional, TypedDict
 
-from django.db import connection
+from django.db import connection, connections
 
 from kernelCI_app.cache import get_query_cache, set_query_cache
 from kernelCI_app.helpers.database import dict_fetchall
@@ -12,44 +12,38 @@
 from kernelCI_app.typeModels.hardwareDetails import CommitHead, Tree
 
 
-def _get_hardware_tree_heads_clause(*, id_only: bool) -> str:
-    """Returns the tree_heads for the hardware queries,
-    where the checkout is not filtered by origin.
+def _get_hardware_trees_from_status_query(*, fields: str) -> str:
+    """Latest checkout per tree in the window for this hardware via hardware_status.
 
-    This is done because tests from a origin can be
-    related to checkouts from another origin."""
-    if id_only is True:
-        fields = "C.id"
-    else:
-        fields = """C.id,
-                    C.origin,
-                    C.tree_name,
-                    C.start_time,
-                    C.git_repository_branch,
-                    C.git_repository_url,
-                    C.git_commit_name,
-                    C.git_commit_hash,
-                    C.git_commit_tags"""
-
-    return f"""SELECT DISTINCT
-                    ON (
-                        C.tree_name,
-                        C.git_repository_branch,
-                        C.git_repository_url,
-                        C.origin
-                    )
-                    {fields}
-                FROM
-                    checkouts C
-                WHERE
-                    C.start_time >= %(start_date)s
-                    AND C.start_time <= %(end_date)s
-                ORDER BY
-                    C.tree_name ASC,
-                    C.git_repository_branch ASC,
-                    C.git_repository_url ASC,
-                    C.origin ASC,
-                    C.start_time DESC"""
+    Checkout origin is not filtered: tests from one origin can be related to
+    checkouts from another origin.
+    """
+    return f"""
+        SELECT DISTINCT ON (
+            C.tree_name,
+            C.git_repository_branch,
+            C.git_repository_url,
+            C.origin
+        )
+            {fields}
+        FROM
+            hardware_status HS
+            INNER JOIN checkouts C ON C.id = HS.checkout_id
+        WHERE
+            HS.test_origin = %(origin)s
+            AND (
+                HS.platform = %(hardware)s
+                OR HS.compatibles @> ARRAY[%(hardware)s]::TEXT[]
+            )
+            AND HS.start_time >= %(start_date)s
+            AND HS.start_time <= %(end_date)s
+        ORDER BY
+            C.tree_name ASC,
+            C.git_repository_branch ASC,
+            C.git_repository_url ASC,
+            C.origin ASC,
+            C.start_time DESC
+    """
 
 
 # TODO: unify with get_tree_listing_count_clause
@@ -370,16 +364,10 @@ def get_hardware_listing_data_from_status_table(
             SUM(test_inc) AS test_null
         FROM
             hardware_status
-        INNER JOIN
-            latest_checkout
-            ON
-                hardware_status.checkout_id = latest_checkout.checkout_id
-            AND
-                latest_checkout.start_time >= %(start_date)s
-            AND
-                latest_checkout.start_time <= %(end_date)s
         WHERE
             hardware_status.test_origin = %(origin)s
+            AND hardware_status.start_time >= %(start_date)s
+            AND hardware_status.start_time <= %(end_date)s
         GROUP BY
             platform,
             compatibles
@@ -762,11 +750,7 @@ def get_hardware_trees_head_commits(
     start_datetime: datetime,
     end_datetime: datetime,
 ) -> list[tuple[str, str]]:
-
-    # similar to the get_hardware_trees_data, except we limit the information
-    # being brought to the commit hash
-
-    cache_key = "hardwareTreesHeadCommits"
+    cache_key = "hardwareTreesHeadCommitsFromStatus"
 
     cache_params = {
         "hardware": hardware_id,
@@ -780,54 +764,20 @@ def get_hardware_trees_head_commits(
     if trees:
         return trees
 
-    tree_head_clause = _get_hardware_tree_heads_clause(id_only=False)
-
-    # We need a subquery because if we filter by any hardware, it will get the
-    # last head that has that hardware, but not the real head of the trees
-    query = f"""
-    WITH
-        -- Selects the data of the latest checkout of all trees in the given period
-        tree_heads AS (
-            {tree_head_clause}
-        )
-    SELECT DISTINCT
-        ON (
-            TH.tree_name,
-            TH.git_repository_branch,
-            TH.git_repository_url,
-            TH.git_commit_hash
-        ) TH.tree_name,
-        TH.git_commit_hash
-    FROM
-        tests
-        INNER JOIN builds ON tests.build_id = builds.id
-        INNER JOIN tree_heads TH ON builds.checkout_id = TH.id
-    WHERE
-        (
-            (
-                tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[]
-                OR tests.environment_misc ->> 'platform' = %(hardware)s
-            )
-            AND tests.origin = %(origin)s
-            AND TH.start_time >= %(start_date)s
-            AND TH.start_time <= %(end_date)s
-        )
-    ORDER BY
-        TH.tree_name ASC,
-        TH.git_repository_branch ASC,
-        TH.git_repository_url ASC,
-        TH.git_commit_hash ASC,
-        TH.start_time DESC
-    """
+    query = _get_hardware_trees_from_status_query(
+        fields="""
+            C.tree_name,
+            C.git_commit_hash
+        """
+    )
 
     params = {
         "hardware": hardware_id,
         "origin": origin,
         "start_date": start_datetime,
         "end_date": end_datetime,
     }
-    trees = []
-    with connection.cursor() as cursor:
+    with connections["default"].cursor() as cursor:
         cursor.execute(query, params)
         tree_records = dict_fetchall(cursor)
         trees = [
@@ -846,7 +796,7 @@ def get_hardware_trees_data(
     start_datetime: datetime,
     end_datetime: datetime,
 ) -> list[Tree]:
-    cache_key = "hardwareDetailsTreeData"
+    cache_key = "hardwareDetailsTreeDataFromStatus"
 
     params = {
         "hardware": hardware_id,
@@ -857,52 +807,19 @@ def get_hardware_trees_data(
 
     trees: list[Tree] = get_query_cache(cache_key, params)
 
-    tree_head_clause = _get_hardware_tree_heads_clause(id_only=False)
-
     if not trees:
-        # We need a subquery because if we filter by any hardware, it will get the
-        # last head that has that hardware, but not the real head of the trees
-        query = f"""
-        WITH
-            -- Selects the data of the latest checkout of all trees in the given period
-            tree_heads AS (
-                {tree_head_clause}
-            )
-        SELECT DISTINCT
-            ON (
-                TH.tree_name,
-                TH.git_repository_branch,
-                TH.git_repository_url,
-                TH.git_commit_hash
-            ) TH.tree_name,
-            TH.origin,
-            TH.git_repository_branch,
-            TH.git_repository_url,
-            TH.git_commit_name,
-            TH.git_commit_hash,
-            TH.git_commit_tags
-        FROM
-            tests
-            INNER JOIN builds ON tests.build_id = builds.id
-            INNER JOIN tree_heads TH ON builds.checkout_id = TH.id
-        WHERE
-            (
-                (
-                    tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[]
-                    OR tests.environment_misc ->> 'platform' = %(hardware)s
-                )
-                AND tests.origin = %(origin)s
-                AND TH.start_time >= %(start_date)s
-                AND TH.start_time <= %(end_date)s
-            )
-        ORDER BY
-            TH.tree_name ASC,
-            TH.git_repository_branch ASC,
-            TH.git_repository_url ASC,
-            TH.git_commit_hash ASC,
-            TH.start_time DESC
-        """
-        with connection.cursor() as cursor:
+        query = _get_hardware_trees_from_status_query(
+            fields="""
+                C.tree_name,
+                C.origin,
+                C.git_repository_branch,
+                C.git_repository_url,
+                C.git_commit_name,
+                C.git_commit_hash,
+                C.git_commit_tags
+            """
+        )
+        with connections["default"].cursor() as cursor:
             cursor.execute(query, params)
             tree_records = dict_fetchall(cursor)
 

backend/kernelCI_app/tests/factories/mocks/fixtures/build_data.py

@@ -144,6 +144,20 @@
         "status": "FAIL",
         "config_name": "defconfig",
     },
+    "older_checkout_build": {
+        "checkout_id": "older_checkout_with_tests",
+        "origin": "maestro",
+        "architecture": "arm64",
+        "status": "PASS",
+        "config_name": "defconfig",
+    },
+    "latest_checkout_build": {
+        "checkout_id": "latest_checkout_without_tests",
+        "origin": "maestro",
+        "architecture": "arm64",
+        "status": "PASS",
+        "config_name": "defconfig",
+    },
     # TODO: Add builds to test STATUS NULL + add integration tests for this case
 }
 

backend/kernelCI_app/tests/factories/mocks/fixtures/tree_data.py

@@ -167,4 +167,23 @@
         ),  # failed_tests_build
         "hardware_platform": None,
     },
+    # Older-checkout scenario (#1264): the older checkout ran "older-checkout-board",
+    # but the latest checkout (latest_checkout_without_tests) did not. Details must
+    # still resolve the older checkout as the tree head.
+    "older_checkout_with_tests": {
+        "origin": "maestro",
+        "git_url": "https://git.kernel.org/pub/scm/linux/kernel/git/older-checkout/linux.git",
+        "git_branch": "master",
+        "tree_name": "older_checkout_mainline",
+        "start_time": datetime.fromtimestamp(1741356000, timezone.utc),
+        "hardware_platform": "older-checkout-board",
+    },
+    "latest_checkout_without_tests": {
+        "origin": "maestro",
+        "git_url": "https://git.kernel.org/pub/scm/linux/kernel/git/older-checkout/linux.git",
+        "git_branch": "master",
+        "tree_name": "older_checkout_mainline",
+        "start_time": datetime.fromtimestamp(1741399200, timezone.utc),
+        "hardware_platform": None,
+    },
 }

backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py

@@ -63,6 +63,18 @@
     ),
 }
 
+# Older-checkout board (#1264): tested only on an older checkout, not on the latest one.
+OLDER_CHECKOUT_HARDWARE = {
+    "id": "older-checkout-board",
+    "body": HardwareDetailsPostBody(
+        origin="maestro",
+        startTimestampInSeconds=1741300000,
+        endTimestampInSeconds=1741420000,
+        selectedCommits={},
+        filter={},
+    ),
+}
+
 
 client = HardwareClient()
 
@@ -202,6 +214,18 @@ def test_no_filters(base_hardware, status_code, has_error_body):
         )
 
 
+def test_older_checkout_board_resolves_older_checkout():
+    """#1264 regression: a board tested only on an older checkout (not the latest
+    one) must still resolve that older checkout as the tree head."""
+    response, content = request_data(OLDER_CHECKOUT_HARDWARE)
+    assert_status_code(response=response, status_code=HTTPStatus.OK)
+    assert "error" not in content
+
+    head_hashes = {tree["head_git_commit_hash"] for tree in content["common"]["trees"]}
+    assert "older_checkout_with_tests" in head_hashes
+    assert "latest_checkout_without_tests" not in head_hashes
+
+
 def test_filter_test_status(test_status_input):
     """
     Tests for the status filter for both boots and tests

backend/kernelCI_app/tests/integrationTests/metrics_test.py

@@ -56,14 +56,14 @@ def _ok_content(query: dict[str, str] | None = None) -> dict:
 metrics_expected_counts = {
     "n_trees": 5,
     "n_checkouts": 21,
-    "n_builds": 12,
+    "n_builds": 14,
     "n_tests": 14,
     "n_issues": 7,
     "n_incidents": 7,
     "prev_n_trees": 6,
     "prev_n_checkouts": 20,
     "prev_n_builds": 7,
-    "prev_n_tests": 7,
+    "prev_n_tests": 9,
 }
 
 

backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py

@@ -75,9 +75,9 @@ def test_get_hardware_trees_data_from_cache(self, mock_get_cache):
     @patch("kernelCI_app.queries.hardware.get_query_cache")
     @patch("kernelCI_app.queries.hardware.set_query_cache")
     @patch("kernelCI_app.queries.hardware.dict_fetchall")
-    @patch("kernelCI_app.queries.hardware.connection")
+    @patch("kernelCI_app.queries.hardware.connections")
     def test_get_hardware_trees_data_from_database(
-        self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache
+        self, mock_connections, mock_dict_fetchall, mock_set_cache, mock_get_cache
     ):
         tree_records = [
             {
@@ -92,7 +92,7 @@ def test_get_hardware_trees_data_from_database(
         ]
         mock_get_cache.return_value = None
         mock_dict_fetchall.return_value = tree_records
-        setup_mock_cursor(mock_connection)
+        setup_mock_cursor(mock_connections.__getitem__.return_value)
 
         result = get_hardware_trees_data(
             hardware_id="hardware",
@@ -104,6 +104,7 @@ def test_get_hardware_trees_data_from_database(
         assert len(result) == 1
         assert result[0].tree_name == "mainline"
         mock_set_cache.assert_called_once()
+        mock_connections.__getitem__.assert_called_with("default")
 
 
 class TestGenerateQueryParams:

backend/kernelCI_app/views/originsView.py

@@ -22,8 +22,11 @@
     "arm",
     "broonie",
     "linaro",
+    "linaro_pull_labs",
     "maestro",
     "microsoft",
+    "pullab_cloud_aws_arm64",
+    "pull_labs_aws_ec2",
     "redhat",
     "riscv",
     "syzbot",