fix(hardware): resolve details trees from hardware_status in window

Stop selecting absolute tree tips for hardware details. Use hardware_status
to pick the latest checkout per tree that actually ran the board, so older
checkouts with tests still appear.

Closes #1264

Signed-off-by: Alan Peixinho <alan.peixinho@profusion.mobi>

Author: alanpeixinho

backend/kernelCI_app/queries/hardware.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Optional, TypedDict
 
-from django.db import connection
+from django.db import connection, connections
 
 from kernelCI_app.cache import get_query_cache, set_query_cache
 from kernelCI_app.helpers.database import dict_fetchall
@@ -12,44 +12,38 @@
 from kernelCI_app.typeModels.hardwareDetails import CommitHead, Tree
 
 
-def _get_hardware_tree_heads_clause(*, id_only: bool) -> str:
-    """Returns the tree_heads for the hardware queries,
-    where the checkout is not filtered by origin.
+def _get_hardware_trees_from_status_query(*, fields: str) -> str:
+    """Latest checkout per tree in the window for this hardware via hardware_status.
 
-    This is done because tests from a origin can be
-    related to checkouts from another origin."""
-    if id_only is True:
-        fields = "C.id"
-    else:
-        fields = """C.id,
-                    C.origin,
-                    C.tree_name,
-                    C.start_time,
-                    C.git_repository_branch,
-                    C.git_repository_url,
-                    C.git_commit_name,
-                    C.git_commit_hash,
-                    C.git_commit_tags"""
-
-    return f"""SELECT DISTINCT
-                    ON (
-                        C.tree_name,
-                        C.git_repository_branch,
-                        C.git_repository_url,
-                        C.origin
-                    )
-                    {fields}
-                FROM
-                    checkouts C
-                WHERE
-                    C.start_time >= %(start_date)s
-                    AND C.start_time <= %(end_date)s
-                ORDER BY
-                    C.tree_name ASC,
-                    C.git_repository_branch ASC,
-                    C.git_repository_url ASC,
-                    C.origin ASC,
-                    C.start_time DESC"""
+    Checkout origin is not filtered: tests from one origin can be related to
+    checkouts from another origin.
+    """
+    return f"""
+        SELECT DISTINCT ON (
+            C.tree_name,
+            C.git_repository_branch,
+            C.git_repository_url,
+            C.origin
+        )
+            {fields}
+        FROM
+            hardware_status HS
+            INNER JOIN checkouts C ON C.id = HS.checkout_id
+        WHERE
+            HS.test_origin = %(origin)s
+            AND (
+                HS.platform = %(hardware)s
+                OR HS.compatibles @> ARRAY[%(hardware)s]::TEXT[]
+            )
+            AND HS.start_time >= %(start_date)s
+            AND HS.start_time <= %(end_date)s
+        ORDER BY
+            C.tree_name ASC,
+            C.git_repository_branch ASC,
+            C.git_repository_url ASC,
+            C.origin ASC,
+            C.start_time DESC
+    """
 
 
 # TODO: unify with get_tree_listing_count_clause
@@ -756,11 +750,7 @@ def get_hardware_trees_head_commits(
     start_datetime: datetime,
     end_datetime: datetime,
 ) -> list[tuple[str, str]]:
-
-    # similar to the get_hardware_trees_data, except we limit the information
-    # being brought to the commit hash
-
-    cache_key = "hardwareTreesHeadCommits"
+    cache_key = "hardwareTreesHeadCommitsFromStatus"
 
     cache_params = {
         "hardware": hardware_id,
@@ -774,54 +764,20 @@ def get_hardware_trees_head_commits(
     if trees:
         return trees
 
-    tree_head_clause = _get_hardware_tree_heads_clause(id_only=False)
-
-    # We need a subquery because if we filter by any hardware, it will get the
-    # last head that has that hardware, but not the real head of the trees
-    query = f"""
-    WITH
-        -- Selects the data of the latest checkout of all trees in the given period
-        tree_heads AS (
-            {tree_head_clause}
-        )
-    SELECT DISTINCT
-        ON (
-            TH.tree_name,
-            TH.git_repository_branch,
-            TH.git_repository_url,
-            TH.git_commit_hash
-        ) TH.tree_name,
-        TH.git_commit_hash
-    FROM
-        tests
-        INNER JOIN builds ON tests.build_id = builds.id
-        INNER JOIN tree_heads TH ON builds.checkout_id = TH.id
-    WHERE
-        (
-            (
-                tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[]
-                OR tests.environment_misc ->> 'platform' = %(hardware)s
-            )
-            AND tests.origin = %(origin)s
-            AND TH.start_time >= %(start_date)s
-            AND TH.start_time <= %(end_date)s
-        )
-    ORDER BY
-        TH.tree_name ASC,
-        TH.git_repository_branch ASC,
-        TH.git_repository_url ASC,
-        TH.git_commit_hash ASC,
-        TH.start_time DESC
-    """
+    query = _get_hardware_trees_from_status_query(
+        fields="""
+            C.tree_name,
+            C.git_commit_hash
+        """
+    )
 
     params = {
         "hardware": hardware_id,
         "origin": origin,
         "start_date": start_datetime,
         "end_date": end_datetime,
     }
-    trees = []
-    with connection.cursor() as cursor:
+    with connections["default"].cursor() as cursor:
         cursor.execute(query, params)
         tree_records = dict_fetchall(cursor)
         trees = [
@@ -840,7 +796,7 @@ def get_hardware_trees_data(
     start_datetime: datetime,
     end_datetime: datetime,
 ) -> list[Tree]:
-    cache_key = "hardwareDetailsTreeData"
+    cache_key = "hardwareDetailsTreeDataFromStatus"
 
     params = {
         "hardware": hardware_id,
@@ -851,52 +807,19 @@ def get_hardware_trees_data(
 
     trees: list[Tree] = get_query_cache(cache_key, params)
 
-    tree_head_clause = _get_hardware_tree_heads_clause(id_only=False)
-
     if not trees:
-        # We need a subquery because if we filter by any hardware, it will get the
-        # last head that has that hardware, but not the real head of the trees
-        query = f"""
-        WITH
-            -- Selects the data of the latest checkout of all trees in the given period
-            tree_heads AS (
-                {tree_head_clause}
-            )
-        SELECT DISTINCT
-            ON (
-                TH.tree_name,
-                TH.git_repository_branch,
-                TH.git_repository_url,
-                TH.git_commit_hash
-            ) TH.tree_name,
-            TH.origin,
-            TH.git_repository_branch,
-            TH.git_repository_url,
-            TH.git_commit_name,
-            TH.git_commit_hash,
-            TH.git_commit_tags
-        FROM
-            tests
-            INNER JOIN builds ON tests.build_id = builds.id
-            INNER JOIN tree_heads TH ON builds.checkout_id = TH.id
-        WHERE
-            (
-                (
-                    tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[]
-                    OR tests.environment_misc ->> 'platform' = %(hardware)s
-                )
-                AND tests.origin = %(origin)s
-                AND TH.start_time >= %(start_date)s
-                AND TH.start_time <= %(end_date)s
-            )
-        ORDER BY
-            TH.tree_name ASC,
-            TH.git_repository_branch ASC,
-            TH.git_repository_url ASC,
-            TH.git_commit_hash ASC,
-            TH.start_time DESC
-        """
-        with connection.cursor() as cursor:
+        query = _get_hardware_trees_from_status_query(
+            fields="""
+                C.tree_name,
+                C.origin,
+                C.git_repository_branch,
+                C.git_repository_url,
+                C.git_commit_name,
+                C.git_commit_hash,
+                C.git_commit_tags
+            """
+        )
+        with connections["default"].cursor() as cursor:
             cursor.execute(query, params)
             tree_records = dict_fetchall(cursor)
 

backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py

@@ -75,9 +75,9 @@ def test_get_hardware_trees_data_from_cache(self, mock_get_cache):
     @patch("kernelCI_app.queries.hardware.get_query_cache")
     @patch("kernelCI_app.queries.hardware.set_query_cache")
     @patch("kernelCI_app.queries.hardware.dict_fetchall")
-    @patch("kernelCI_app.queries.hardware.connection")
+    @patch("kernelCI_app.queries.hardware.connections")
     def test_get_hardware_trees_data_from_database(
-        self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache
+        self, mock_connections, mock_dict_fetchall, mock_set_cache, mock_get_cache
     ):
         tree_records = [
             {
@@ -92,7 +92,7 @@ def test_get_hardware_trees_data_from_database(
         ]
         mock_get_cache.return_value = None
         mock_dict_fetchall.return_value = tree_records
-        setup_mock_cursor(mock_connection)
+        setup_mock_cursor(mock_connections.__getitem__.return_value)
 
         result = get_hardware_trees_data(
             hardware_id="hardware",
@@ -104,6 +104,7 @@ def test_get_hardware_trees_data_from_database(
         assert len(result) == 1
         assert result[0].tree_name == "mainline"
         mock_set_cache.assert_called_once()
+        mock_connections.__getitem__.assert_called_with("default")
 
 
 class TestGenerateQueryParams: