From 8e04ba58039334f054baba22888a79bd76e0e1ea Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Tue, 14 Jul 2026 15:38:52 -0300 Subject: [PATCH 1/2] fix(hardware): list boards in time window and load test origins from hardware_status * Non longer limiting hardware listing to latest checkout. * Hardcoded TEST_ORIGINS also omitted origins from the Hardware dropdown. * Expand hardcoded test origins list. Closes #1983 Signed-off-by: Alan Peixinho --- backend/kernelCI_app/queries/hardware.py | 10 ++-------- backend/kernelCI_app/views/originsView.py | 3 +++ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index 9bf249b1b..374536638 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -370,16 +370,10 @@ def get_hardware_listing_data_from_status_table( SUM(test_inc) AS test_null FROM hardware_status - INNER JOIN - latest_checkout - ON - hardware_status.checkout_id = latest_checkout.checkout_id - AND - latest_checkout.start_time >= %(start_date)s - AND - latest_checkout.start_time <= %(end_date)s WHERE hardware_status.test_origin = %(origin)s + AND hardware_status.start_time >= %(start_date)s + AND hardware_status.start_time <= %(end_date)s GROUP BY platform, compatibles diff --git a/backend/kernelCI_app/views/originsView.py b/backend/kernelCI_app/views/originsView.py index e5ed1821d..913e402c3 100644 --- a/backend/kernelCI_app/views/originsView.py +++ b/backend/kernelCI_app/views/originsView.py @@ -22,8 +22,11 @@ "arm", "broonie", "linaro", + "linaro_pull_labs", "maestro", "microsoft", + "pullab_cloud_aws_arm64", + "pull_labs_aws_ec2", "redhat", "riscv", "syzbot", From 6e00234596540ae5333b6daa74da2d340e5a7bf5 Mon Sep 17 00:00:00 2001 From: Alan Peixinho Date: Wed, 15 Jul 2026 18:56:28 -0300 Subject: [PATCH 2/2] fix(hardware): resolve details trees from hardware_status in window Stop selecting absolute tree tips for hardware details. Use hardware_status to pick the latest checkout per tree that actually ran the board, so older checkouts with tests still appear. Closes #1264 Signed-off-by: Alan Peixinho --- backend/kernelCI_app/queries/hardware.py | 183 +++++------------- .../factories/mocks/fixtures/build_data.py | 14 ++ .../factories/mocks/fixtures/tree_data.py | 19 ++ .../hardwareDetailsSummary_test.py | 24 +++ .../tests/integrationTests/metrics_test.py | 4 +- .../queries/hardware_queries_test.py | 7 +- 6 files changed, 116 insertions(+), 135 deletions(-) diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index 374536638..bda416e24 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional, TypedDict -from django.db import connection +from django.db import connection, connections from kernelCI_app.cache import get_query_cache, set_query_cache from kernelCI_app.helpers.database import dict_fetchall @@ -12,44 +12,38 @@ from kernelCI_app.typeModels.hardwareDetails import CommitHead, Tree -def _get_hardware_tree_heads_clause(*, id_only: bool) -> str: - """Returns the tree_heads for the hardware queries, - where the checkout is not filtered by origin. +def _get_hardware_trees_from_status_query(*, fields: str) -> str: + """Latest checkout per tree in the window for this hardware via hardware_status. - This is done because tests from a origin can be - related to checkouts from another origin.""" - if id_only is True: - fields = "C.id" - else: - fields = """C.id, - C.origin, - C.tree_name, - C.start_time, - C.git_repository_branch, - C.git_repository_url, - C.git_commit_name, - C.git_commit_hash, - C.git_commit_tags""" - - return f"""SELECT DISTINCT - ON ( - C.tree_name, - C.git_repository_branch, - C.git_repository_url, - C.origin - ) - {fields} - FROM - checkouts C - WHERE - C.start_time >= %(start_date)s - AND C.start_time <= %(end_date)s - ORDER BY - C.tree_name ASC, - C.git_repository_branch ASC, - C.git_repository_url ASC, - C.origin ASC, - C.start_time DESC""" + Checkout origin is not filtered: tests from one origin can be related to + checkouts from another origin. + """ + return f""" + SELECT DISTINCT ON ( + C.tree_name, + C.git_repository_branch, + C.git_repository_url, + C.origin + ) + {fields} + FROM + hardware_status HS + INNER JOIN checkouts C ON C.id = HS.checkout_id + WHERE + HS.test_origin = %(origin)s + AND ( + HS.platform = %(hardware)s + OR HS.compatibles @> ARRAY[%(hardware)s]::TEXT[] + ) + AND HS.start_time >= %(start_date)s + AND HS.start_time <= %(end_date)s + ORDER BY + C.tree_name ASC, + C.git_repository_branch ASC, + C.git_repository_url ASC, + C.origin ASC, + C.start_time DESC + """ # TODO: unify with get_tree_listing_count_clause @@ -756,11 +750,7 @@ def get_hardware_trees_head_commits( start_datetime: datetime, end_datetime: datetime, ) -> list[tuple[str, str]]: - - # similar to the get_hardware_trees_data, except we limit the information - # being brought to the commit hash - - cache_key = "hardwareTreesHeadCommits" + cache_key = "hardwareTreesHeadCommitsFromStatus" cache_params = { "hardware": hardware_id, @@ -774,45 +764,12 @@ def get_hardware_trees_head_commits( if trees: return trees - tree_head_clause = _get_hardware_tree_heads_clause(id_only=False) - - # We need a subquery because if we filter by any hardware, it will get the - # last head that has that hardware, but not the real head of the trees - query = f""" - WITH - -- Selects the data of the latest checkout of all trees in the given period - tree_heads AS ( - {tree_head_clause} - ) - SELECT DISTINCT - ON ( - TH.tree_name, - TH.git_repository_branch, - TH.git_repository_url, - TH.git_commit_hash - ) TH.tree_name, - TH.git_commit_hash - FROM - tests - INNER JOIN builds ON tests.build_id = builds.id - INNER JOIN tree_heads TH ON builds.checkout_id = TH.id - WHERE - ( - ( - tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[] - OR tests.environment_misc ->> 'platform' = %(hardware)s - ) - AND tests.origin = %(origin)s - AND TH.start_time >= %(start_date)s - AND TH.start_time <= %(end_date)s - ) - ORDER BY - TH.tree_name ASC, - TH.git_repository_branch ASC, - TH.git_repository_url ASC, - TH.git_commit_hash ASC, - TH.start_time DESC - """ + query = _get_hardware_trees_from_status_query( + fields=""" + C.tree_name, + C.git_commit_hash + """ + ) params = { "hardware": hardware_id, @@ -820,8 +777,7 @@ def get_hardware_trees_head_commits( "start_date": start_datetime, "end_date": end_datetime, } - trees = [] - with connection.cursor() as cursor: + with connections["default"].cursor() as cursor: cursor.execute(query, params) tree_records = dict_fetchall(cursor) trees = [ @@ -840,7 +796,7 @@ def get_hardware_trees_data( start_datetime: datetime, end_datetime: datetime, ) -> list[Tree]: - cache_key = "hardwareDetailsTreeData" + cache_key = "hardwareDetailsTreeDataFromStatus" params = { "hardware": hardware_id, @@ -851,52 +807,19 @@ def get_hardware_trees_data( trees: list[Tree] = get_query_cache(cache_key, params) - tree_head_clause = _get_hardware_tree_heads_clause(id_only=False) - if not trees: - # We need a subquery because if we filter by any hardware, it will get the - # last head that has that hardware, but not the real head of the trees - query = f""" - WITH - -- Selects the data of the latest checkout of all trees in the given period - tree_heads AS ( - {tree_head_clause} - ) - SELECT DISTINCT - ON ( - TH.tree_name, - TH.git_repository_branch, - TH.git_repository_url, - TH.git_commit_hash - ) TH.tree_name, - TH.origin, - TH.git_repository_branch, - TH.git_repository_url, - TH.git_commit_name, - TH.git_commit_hash, - TH.git_commit_tags - FROM - tests - INNER JOIN builds ON tests.build_id = builds.id - INNER JOIN tree_heads TH ON builds.checkout_id = TH.id - WHERE - ( - ( - tests.environment_compatible @> ARRAY[%(hardware)s]::TEXT[] - OR tests.environment_misc ->> 'platform' = %(hardware)s - ) - AND tests.origin = %(origin)s - AND TH.start_time >= %(start_date)s - AND TH.start_time <= %(end_date)s - ) - ORDER BY - TH.tree_name ASC, - TH.git_repository_branch ASC, - TH.git_repository_url ASC, - TH.git_commit_hash ASC, - TH.start_time DESC - """ - with connection.cursor() as cursor: + query = _get_hardware_trees_from_status_query( + fields=""" + C.tree_name, + C.origin, + C.git_repository_branch, + C.git_repository_url, + C.git_commit_name, + C.git_commit_hash, + C.git_commit_tags + """ + ) + with connections["default"].cursor() as cursor: cursor.execute(query, params) tree_records = dict_fetchall(cursor) diff --git a/backend/kernelCI_app/tests/factories/mocks/fixtures/build_data.py b/backend/kernelCI_app/tests/factories/mocks/fixtures/build_data.py index 25d1a470d..67dc0009b 100644 --- a/backend/kernelCI_app/tests/factories/mocks/fixtures/build_data.py +++ b/backend/kernelCI_app/tests/factories/mocks/fixtures/build_data.py @@ -144,6 +144,20 @@ "status": "FAIL", "config_name": "defconfig", }, + "older_checkout_build": { + "checkout_id": "older_checkout_with_tests", + "origin": "maestro", + "architecture": "arm64", + "status": "PASS", + "config_name": "defconfig", + }, + "latest_checkout_build": { + "checkout_id": "latest_checkout_without_tests", + "origin": "maestro", + "architecture": "arm64", + "status": "PASS", + "config_name": "defconfig", + }, # TODO: Add builds to test STATUS NULL + add integration tests for this case } diff --git a/backend/kernelCI_app/tests/factories/mocks/fixtures/tree_data.py b/backend/kernelCI_app/tests/factories/mocks/fixtures/tree_data.py index 83f1d4234..6c95f28df 100644 --- a/backend/kernelCI_app/tests/factories/mocks/fixtures/tree_data.py +++ b/backend/kernelCI_app/tests/factories/mocks/fixtures/tree_data.py @@ -167,4 +167,23 @@ ), # failed_tests_build "hardware_platform": None, }, + # Older-checkout scenario (#1264): the older checkout ran "older-checkout-board", + # but the latest checkout (latest_checkout_without_tests) did not. Details must + # still resolve the older checkout as the tree head. + "older_checkout_with_tests": { + "origin": "maestro", + "git_url": "https://git.kernel.org/pub/scm/linux/kernel/git/older-checkout/linux.git", + "git_branch": "master", + "tree_name": "older_checkout_mainline", + "start_time": datetime.fromtimestamp(1741356000, timezone.utc), + "hardware_platform": "older-checkout-board", + }, + "latest_checkout_without_tests": { + "origin": "maestro", + "git_url": "https://git.kernel.org/pub/scm/linux/kernel/git/older-checkout/linux.git", + "git_branch": "master", + "tree_name": "older_checkout_mainline", + "start_time": datetime.fromtimestamp(1741399200, timezone.utc), + "hardware_platform": None, + }, } diff --git a/backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py b/backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py index bbe226498..577442fc5 100644 --- a/backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py +++ b/backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py @@ -63,6 +63,18 @@ ), } +# Older-checkout board (#1264): tested only on an older checkout, not on the latest one. +OLDER_CHECKOUT_HARDWARE = { + "id": "older-checkout-board", + "body": HardwareDetailsPostBody( + origin="maestro", + startTimestampInSeconds=1741300000, + endTimestampInSeconds=1741420000, + selectedCommits={}, + filter={}, + ), +} + client = HardwareClient() @@ -202,6 +214,18 @@ def test_no_filters(base_hardware, status_code, has_error_body): ) +def test_older_checkout_board_resolves_older_checkout(): + """#1264 regression: a board tested only on an older checkout (not the latest + one) must still resolve that older checkout as the tree head.""" + response, content = request_data(OLDER_CHECKOUT_HARDWARE) + assert_status_code(response=response, status_code=HTTPStatus.OK) + assert "error" not in content + + head_hashes = {tree["head_git_commit_hash"] for tree in content["common"]["trees"]} + assert "older_checkout_with_tests" in head_hashes + assert "latest_checkout_without_tests" not in head_hashes + + def test_filter_test_status(test_status_input): """ Tests for the status filter for both boots and tests diff --git a/backend/kernelCI_app/tests/integrationTests/metrics_test.py b/backend/kernelCI_app/tests/integrationTests/metrics_test.py index ec9defb28..ea39c6201 100644 --- a/backend/kernelCI_app/tests/integrationTests/metrics_test.py +++ b/backend/kernelCI_app/tests/integrationTests/metrics_test.py @@ -56,14 +56,14 @@ def _ok_content(query: dict[str, str] | None = None) -> dict: metrics_expected_counts = { "n_trees": 5, "n_checkouts": 21, - "n_builds": 12, + "n_builds": 14, "n_tests": 14, "n_issues": 7, "n_incidents": 7, "prev_n_trees": 6, "prev_n_checkouts": 20, "prev_n_builds": 7, - "prev_n_tests": 7, + "prev_n_tests": 9, } diff --git a/backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py b/backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py index 6330bf223..f5640eedf 100644 --- a/backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py +++ b/backend/kernelCI_app/tests/unitTests/queries/hardware_queries_test.py @@ -75,9 +75,9 @@ def test_get_hardware_trees_data_from_cache(self, mock_get_cache): @patch("kernelCI_app.queries.hardware.get_query_cache") @patch("kernelCI_app.queries.hardware.set_query_cache") @patch("kernelCI_app.queries.hardware.dict_fetchall") - @patch("kernelCI_app.queries.hardware.connection") + @patch("kernelCI_app.queries.hardware.connections") def test_get_hardware_trees_data_from_database( - self, mock_connection, mock_dict_fetchall, mock_set_cache, mock_get_cache + self, mock_connections, mock_dict_fetchall, mock_set_cache, mock_get_cache ): tree_records = [ { @@ -92,7 +92,7 @@ def test_get_hardware_trees_data_from_database( ] mock_get_cache.return_value = None mock_dict_fetchall.return_value = tree_records - setup_mock_cursor(mock_connection) + setup_mock_cursor(mock_connections.__getitem__.return_value) result = get_hardware_trees_data( hardware_id="hardware", @@ -104,6 +104,7 @@ def test_get_hardware_trees_data_from_database( assert len(result) == 1 assert result[0].tree_name == "mainline" mock_set_cache.assert_called_once() + mock_connections.__getitem__.assert_called_with("default") class TestGenerateQueryParams: