feat/compare ab boots tests

backend/kernelCI_app/helpers/treeCompare.py

@@ -0,0 +1,154 @@
+from typing import Literal, Optional
+
+from kernelCI_app.constants.general import UNKNOWN_STRING
+from kernelCI_app.helpers.filters import FilterParams, is_filtered_out
+from kernelCI_app.helpers.issueExtras import parse_issue
+from kernelCI_app.typeModels.treeDetails import (
+    GroupedCompareStatus,
+    TreeCompareTest,
+)
+
+CompareKey = tuple[str, str, str]
+STATUS_RANK: dict[GroupedCompareStatus, int] = {
+    "PASS": 0,
+    "INCONCLUSIVE": 1,
+    "FAIL": 2,
+}
+
+
+def group_raw_status(status: Optional[str]) -> GroupedCompareStatus:
+    if status == "PASS":
+        return "PASS"
+    if status == "FAIL":
+        return "FAIL"
+    return "INCONCLUSIVE"
+
+
+def worst_status(
+    current: Optional[GroupedCompareStatus],
+    candidate: GroupedCompareStatus,
+) -> GroupedCompareStatus:
+    if current is None:
+        return candidate
+    if STATUS_RANK[candidate] > STATUS_RANK[current]:
+        return candidate
+    return current
+
+
+def is_compare_row_filtered_out(
+    *,
+    filters: FilterParams,
+    data_type: Literal["boots", "tests"],
+    path: str,
+    status: Optional[str],
+    config: str,
+    lab: str,
+    compiler: str,
+    architecture: str,
+    platform: str,
+    origin: Optional[str],
+    compatibles: set[str],
+    known_issues: set[tuple[str, Optional[int]]],
+) -> bool:
+    tab: Literal["boot", "test"] = "boot" if data_type == "boots" else "test"
+    status_filter = (
+        filters.filterBootStatus if tab == "boot" else filters.filterTestStatus
+    )
+    path_filter = filters.filterBootPath if tab == "boot" else filters.filterTestPath
+    origin_filter = (
+        filters.filter_boot_origin if tab == "boot" else filters.filter_test_origin
+    )
+    issue_filter = filters.filterIssues[tab]
+
+    return bool(
+        is_filtered_out(status or "NULL", status_filter)
+        or (path_filter and path_filter not in path)
+        or is_filtered_out(platform, filters.filterPlatforms[tab])
+        or is_filtered_out(origin or UNKNOWN_STRING, origin_filter)
+        or is_filtered_out(compiler, filters.filterCompiler)
+        or is_filtered_out(config, filters.filterConfigs)
+        or is_filtered_out(lab, filters.filter_labs)
+        or is_filtered_out(architecture, filters.filterArchitecture)
+        or (filters.filterHardware and filters.filterHardware.isdisjoint(compatibles))
+        or (issue_filter and not known_issues.issubset(issue_filter))
+    )
+
+
+def collapse_side_statuses(
+    *,
+    rows: list[dict],
+    commit_hash: str,
+    filters: FilterParams,
+    data_type: Literal["boots", "tests"],
+) -> dict[CompareKey, GroupedCompareStatus]:
+    """Filter one commit's rows and collapse to worst status per identity key.
+
+    Duration filters are applied in SQL. Issue matching follows the
+    hardwareDetailsSummary subset pattern on aggregated known_issues.
+    """
+    result: dict[CompareKey, GroupedCompareStatus] = {}
+
+    for instance in rows:
+        if instance["git_commit_hash"] != commit_hash:
+            continue
+
+        path = instance["path"] or UNKNOWN_STRING
+        config = instance["config_name"] or UNKNOWN_STRING
+        platform = instance["platform"] or UNKNOWN_STRING
+        lab = instance["lab"] or UNKNOWN_STRING
+        status = instance["status"]
+        compatibles = set(instance["environment_compatible"] or [])
+        (compiler, architecture) = [
+            (val or UNKNOWN_STRING).strip(" []'")
+            for val in (instance["compiler_arch"] or [None, None])
+        ]
+        known_issues = {
+            parse_issue(issue) for issue in (instance["known_issues"] or [])
+        }
+
+        if is_compare_row_filtered_out(
+            filters=filters,
+            data_type=data_type,
+            path=path,
+            status=status,
+            config=config,
+            lab=lab,
+            compiler=compiler,
+            architecture=architecture,
+            platform=platform,
+            origin=instance["origin"],
+            compatibles=compatibles,
+            known_issues=known_issues,
+        ):
+            continue
+
+        key: CompareKey = (path, config, platform)
+        result[key] = worst_status(result.get(key), group_raw_status(status))
+
+    return result
+
+
+def build_compare_rows(
+    status_a: dict[CompareKey, GroupedCompareStatus],
+    status_b: dict[CompareKey, GroupedCompareStatus],
+) -> list[TreeCompareTest]:
+    """Return rows where grouped status differs, including one-sided nulls."""
+    keys = set(status_a) | set(status_b)
+    rows: list[TreeCompareTest] = []
+
+    for path, config_name, platform in sorted(keys):
+        a = status_a.get((path, config_name, platform))
+        b = status_b.get((path, config_name, platform))
+        if a == b:
+            continue
+        rows.append(
+            TreeCompareTest(
+                path=path,
+                config_name=config_name,
+                platform=platform,
+                status_a=a,
+                status_b=b,
+            )
+        )
+
+    return rows

backend/kernelCI_app/queries/tree.py

@@ -977,6 +977,110 @@ def get_tree_commit_history_hashes_aggregated(
         return rows
 
 
+def get_tree_compare_data(
+    *,
+    data_type: Literal["boots", "tests"],
+    origin: str,
+    git_branch: str,
+    tree_name: str,
+    commit_hashes: list[str],
+    boots_duration: tuple[Optional[int], Optional[int]] = (None, None),
+    tests_duration: tuple[Optional[int], Optional[int]] = (None, None),
+) -> list[dict]:
+    """Fetch aggregated boot/test rows for commit comparison.
+
+    Groups by identity dims (path, config_name, platform) plus filter dims
+    (status, compiler, arch, lab, origin, issues). Duration filters apply in SQL;
+    remaining filters apply in Python.
+    """
+    if not commit_hashes:
+        return []
+
+    boot_duration_min, boot_duration_max = boots_duration
+    test_duration_min, test_duration_max = tests_duration
+
+    params = {
+        "commit_hashes": commit_hashes,
+        "origin_param": origin,
+        "git_branch_param": git_branch,
+        "tree_name": tree_name,
+        "git_url_param": None,
+        "boot_duration_min": boot_duration_min,
+        "boot_duration_max": boot_duration_max,
+        "test_duration_min": test_duration_min,
+        "test_duration_max": test_duration_max,
+    }
+
+    cache_key = "treeCompareData"
+    cache_params = {
+        **params,
+        "data_type": data_type,
+        "commit_hashes": tuple(sorted(commit_hashes)),
+    }
+    rows = get_query_cache(cache_key, cache_params)
+    if rows is not None:
+        return rows
+
+    checkout_clauses = create_checkouts_where_clauses(
+        git_url=None, git_branch=git_branch, tree_name=tree_name
+    )
+    git_branch_clause = checkout_clauses.get("git_branch_clause")
+    tree_name_clause = checkout_clauses.get("tree_name_clause")
+    tree_name_full_clause = "\nAND " + tree_name_clause if tree_name_clause else ""
+    git_branch_full_clause = "\nAND " + git_branch_clause if git_branch_clause else ""
+
+    if data_type == "boots":
+        path_filter = "AND (tests.path = 'boot' OR tests.path LIKE 'boot.%%')"
+        duration_clause = get_boot_test_duration_clause(boots_duration, (None, None))
+    else:
+        path_filter = "AND tests.path <> 'boot' AND tests.path NOT LIKE 'boot.%%'"
+        duration_clause = get_boot_test_duration_clause((None, None), tests_duration)
+
+    query = f"""
+        SELECT
+            COUNT(DISTINCT tests.id) AS count,
+            c.git_commit_hash,
+            tests.path,
+            tests.status AS status,
+            builds.config_name,
+            tests.environment_misc->>'platform' AS platform,
+            tests.environment_compatible,
+            array[builds.compiler, builds.architecture] AS compiler_arch,
+            tests.misc->>'runtime' AS lab,
+            tests.origin,
+            ARRAY_AGG(DISTINCT ic.issue_id || ',' || ic.issue_version::text)
+                AS known_issues
+        FROM checkouts c
+        INNER JOIN builds ON c.id = builds.checkout_id
+        INNER JOIN tests ON tests.build_id = builds.id
+            {path_filter}
+        LEFT JOIN incidents ic ON tests.id = ic.test_id
+        WHERE
+            c.git_commit_hash = ANY(%(commit_hashes)s)
+            AND c.origin = %(origin_param)s
+            {git_branch_full_clause}
+            {tree_name_full_clause}
+            {duration_clause}
+        GROUP BY
+            c.git_commit_hash,
+            tests.path,
+            tests.status,
+            builds.config_name,
+            platform,
+            tests.environment_compatible,
+            builds.compiler,
+            builds.architecture,
+            lab,
+            tests.origin
+    """
+
+    with connection.cursor() as cursor:
+        cursor.execute(query, params)
+        rows = dict_fetchall(cursor)
+        set_query_cache(key=cache_key, params=cache_params, rows=rows)
+        return rows
+
+
 def get_tree_commit_history(
     *,
     commit_hash: str,