fix(hardware): list boards in time window and load test origins from hardware_status

* Non longer limiting hardware listing to latest checkout.
* Hardcoded TEST_ORIGINS also omitted origins from the Hardware dropdown.
* Dynamic origins list from hardware_status.

Closes #1983

Signed-off-by: Alan Peixinho <alan.peixinho@profusion.mobi>

Author: alanpeixinho

backend/kernelCI_app/queries/hardware.py

@@ -5,13 +5,41 @@
 
 from kernelCI_app.cache import get_query_cache, set_query_cache
 from kernelCI_app.helpers.database import dict_fetchall
+from kernelCI_app.queries.checkout import ORIGINS_CACHE_TIMEOUT
 from kernelCI_app.queries.duration import (
     get_boot_test_duration_clause,
     get_build_duration_clause,
 )
 from kernelCI_app.typeModels.hardwareDetails import CommitHead, Tree
 
 
+def get_test_origins(interval_in_days: int) -> list[str]:
+    """Distinct test origins from hardware_status in the given interval."""
+    cache_key = f"test_origins_hardware_status_{interval_in_days}"
+    cached = get_query_cache(key=cache_key)
+    if cached is not None:
+        return cached
+
+    query = """
+        SELECT DISTINCT test_origin AS origin
+        FROM hardware_status
+        WHERE start_time >= CURRENT_DATE - INTERVAL '%(interval_in_days)s days'
+        ORDER BY origin
+    """
+
+    with connection.cursor() as cursor:
+        cursor.execute(query, {"interval_in_days": interval_in_days})
+        origins = [row[0] for row in cursor.fetchall() if row[0]]
+
+    if origins:
+        set_query_cache(
+            key=cache_key,
+            rows=origins,
+            timeout=ORIGINS_CACHE_TIMEOUT,
+        )
+    return origins
+
+
 def _get_hardware_tree_heads_clause(*, id_only: bool) -> str:
     """Returns the tree_heads for the hardware queries,
     where the checkout is not filtered by origin.
@@ -370,16 +398,10 @@ def get_hardware_listing_data_from_status_table(
             SUM(test_inc) AS test_null
         FROM
             hardware_status
-        INNER JOIN
-            latest_checkout
-            ON
-                hardware_status.checkout_id = latest_checkout.checkout_id
-            AND
-                latest_checkout.start_time >= %(start_date)s
-            AND
-                latest_checkout.start_time <= %(end_date)s
         WHERE
             hardware_status.test_origin = %(origin)s
+            AND hardware_status.start_time >= %(start_date)s
+            AND hardware_status.start_time <= %(end_date)s
         GROUP BY
             platform,
             compatibles

backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py

@@ -1,11 +1,13 @@
 from datetime import datetime
 from unittest.mock import patch
 
+from kernelCI_app.queries.checkout import ORIGINS_CACHE_TIMEOUT
 from kernelCI_app.queries.hardware import (
     _generate_query_params,
     get_hardware_commit_history,
     get_hardware_details_data,
     get_hardware_trees_data,
+    get_test_origins,
     query_records,
 )
 from kernelCI_app.tests.unitTests.queries.conftest import (
@@ -18,6 +20,56 @@
 END_DATE = datetime(2025, 11, 12)
 
 
+class TestGetTestOrigins:
+    @patch("kernelCI_app.queries.hardware.get_query_cache")
+    def test_get_test_origins_from_cache(self, mock_get_cache):
+        cached_data = ["linaro_pull_labs", "maestro"]
+        mock_get_cache.return_value = cached_data
+
+        result = get_test_origins(7)
+
+        assert result == cached_data
+
+    @patch("kernelCI_app.queries.hardware.get_query_cache")
+    @patch("kernelCI_app.queries.hardware.set_query_cache")
+    @patch("kernelCI_app.queries.hardware.connection")
+    def test_get_test_origins_from_database(
+        self, mock_connection, mock_set_cache, mock_get_cache
+    ):
+        mock_get_cache.return_value = None
+        mock_cursor = setup_mock_cursor(mock_connection)
+        mock_cursor.fetchall.return_value = [
+            ("linaro_pull_labs",),
+            ("maestro",),
+            (None,),
+        ]
+
+        result = get_test_origins(7)
+
+        assert result == ["linaro_pull_labs", "maestro"]
+        mock_cursor.execute.assert_called_once()
+        mock_set_cache.assert_called_once_with(
+            key="test_origins_hardware_status_7",
+            rows=["linaro_pull_labs", "maestro"],
+            timeout=ORIGINS_CACHE_TIMEOUT,
+        )
+
+    @patch("kernelCI_app.queries.hardware.get_query_cache")
+    @patch("kernelCI_app.queries.hardware.set_query_cache")
+    @patch("kernelCI_app.queries.hardware.connection")
+    def test_get_test_origins_empty_result_does_not_cache(
+        self, mock_connection, mock_set_cache, mock_get_cache
+    ):
+        mock_get_cache.return_value = None
+        mock_cursor = setup_mock_cursor(mock_connection)
+        mock_cursor.fetchall.return_value = []
+
+        result = get_test_origins(7)
+
+        assert result == []
+        mock_set_cache.assert_not_called()
+
+
 class TestGetHardwareDetailsData:
     @patch("kernelCI_app.queries.hardware.get_query_cache")
     def test_get_hardware_details_data_from_cache(self, mock_get_cache):

backend/kernelCI_app/tests/unitTests/views/originsView_test.py

@@ -6,7 +6,6 @@
 
 from kernelCI_app.constants.localization import ClientStrings
 from kernelCI_app.views.originsView import (
-    TEST_ORIGINS,
     OriginsView,
     separate_origin_records,
 )
@@ -51,12 +50,14 @@ def setUp(self):
         self.view = OriginsView()
         self.url = "/origins"
 
+    @patch("kernelCI_app.views.originsView.get_test_origins")
     @patch("kernelCI_app.views.originsView.get_origins")
-    def test_get_origins_success(self, mock_get_origins):
+    def test_get_origins_success(self, mock_get_origins, mock_get_test_origins):
         mock_get_origins.return_value = [
             {"origin": "origin1", "table": "checkouts"},
             {"origin": "origin2", "table": "checkouts"},
         ]
+        mock_get_test_origins.return_value = ["maestro", "linaro_pull_labs"]
         request = self.factory.get(self.url)
         response = self.view.get(request)
 
@@ -65,18 +66,21 @@ def test_get_origins_success(self, mock_get_origins):
             response.data,
             {
                 "checkout_origins": ["origin1", "origin2"],
-                "test_origins": TEST_ORIGINS,
+                "test_origins": ["maestro", "linaro_pull_labs"],
             },
         )
+        mock_get_test_origins.assert_called_once()
 
+    @patch("kernelCI_app.views.originsView.get_test_origins")
     @patch("kernelCI_app.views.originsView.get_origins")
-    def test_get_origins_not_found(self, mock_get_origins):
+    def test_get_origins_not_found(self, mock_get_origins, mock_get_test_origins):
         mock_get_origins.return_value = []
         request = self.factory.get(self.url)
         response = self.view.get(request)
 
         self.assertEqual(response.status_code, 400)
         self.assertEqual(response.data, {"error": ClientStrings.NO_ORIGIN_FOUND})
+        mock_get_test_origins.assert_not_called()
 
     @patch("kernelCI_app.views.originsView.OriginsQueryParameters.model_validate")
     def test_get_origins_query_validation_error(self, mock_validate):
@@ -98,14 +102,16 @@ def test_get_origins_query_validation_error(self, mock_validate):
         self.assertEqual(response.status_code, 500)
         self.assertIsNotNone(response.data)
 
+    @patch("kernelCI_app.views.originsView.get_test_origins")
     @patch("kernelCI_app.views.originsView.get_origins")
     @patch("kernelCI_app.views.originsView.OriginsResponse")
     def test_get_origins_response_validation_error(
-        self, mock_response_class, mock_get_origins
+        self, mock_response_class, mock_get_origins, mock_get_test_origins
     ):
         mock_get_origins.return_value = [
             {"origin": "origin1", "table": "checkouts"},
         ]
+        mock_get_test_origins.return_value = ["maestro"]
 
         mock_error = ValidationError.from_exception_data(
             "test_error",

backend/kernelCI_app/views/originsView.py

@@ -9,27 +9,9 @@
 from kernelCI_app.helpers.errorHandling import create_api_error_response
 from kernelCI_app.helpers.logger import log_message
 from kernelCI_app.queries.checkout import get_origins
+from kernelCI_app.queries.hardware import get_test_origins
 from kernelCI_app.typeModels.origins import OriginsQueryParameters, OriginsResponse
 
-# For now we are hardcoding test origins since fetching them dynamically in a query is taking
-# around 8 minutes to finish. That time will result in a timeout every request, so this is a
-# temporary solution. Another option would be to create a cron job that feeds a TestOriginsCache
-# sqlite table, from which this endpoint would fetch.
-#
-# TODO: replace with a dynamic approach
-TEST_ORIGINS = [
-    "0dayci",
-    "arm",
-    "broonie",
-    "linaro",
-    "maestro",
-    "microsoft",
-    "redhat",
-    "riscv",
-    "syzbot",
-    "ti",
-]
-
 
 def separate_origin_records(*, records: list[dict[str, str]]) -> set[str]:
     """Iterates over the records for origins and returns a set for each table defined by those records
@@ -85,11 +67,14 @@ def get(self, request) -> Response:
             )
 
         self.checkout_origins = separate_origin_records(records=origin_records)
+        test_origins = get_test_origins(
+            interval_in_days=query_parameters.interval_in_days
+        )
 
         try:
             valid_response = OriginsResponse(
                 checkout_origins=sorted(self.checkout_origins),
-                test_origins=TEST_ORIGINS,
+                test_origins=test_origins,
             )
         except ValidationError as e:
             return Response(e.json(), HTTPStatus.INTERNAL_SERVER_ERROR)