fix: optimize TreeCommitHistory endpoint and add tree commits list view (#1877)

* Improve query performance for commit history
    * Include platform filter on query to speedup plot on hardware page
    * Add TreeCommitsListView and type models
    * Update frontend API and CommitNavigationGraph
    * Include tree_name and git_url on frontend requests

Author: alanpeixinho

backend/kernelCI_app/constants/localization.py

@@ -111,6 +111,10 @@ class DocStrings:
     TREE_LATEST_COMMIT_HASH_DESCRIPTION = "Commit hash to retrieve tree information"
     TREE_LATEST_ORIGIN_DESCRIPTION = "Origin filter to retrieve tree information"
 
+    TREE_LIST_COMMIT_HASH_DESCRIPTION = (
+        "Comma-separated list of commit hashes to get history for"
+    )
+
     TREE_QUERY_ORIGIN_DESCRIPTION = "Origin of the tree"
     TREE_QUERY_GIT_URL_DESCRIPTION = "Git repository URL of the tree"
 

backend/kernelCI_app/helpers/database.py

@@ -6,3 +6,15 @@ def dict_fetchall(cursor) -> list[dict]:
     """
     columns = [col[0] for col in cursor.description]
     return [dict(zip(columns, row, strict=False)) for row in cursor.fetchall()]
+
+
+def debug_query(cursor, query, params) -> tuple[str, str]:
+    sql = cursor.mogrify(query, params)
+    profile = "\n".join(
+        row for row, *_ in cursor.execute(f"EXPLAIN (ANALYZE, BUFFERS) {query}", params)
+    )
+    return (sql, profile)
+
+
+def print_debug_query(cursor, query, params):
+    print("{}\n{}\n".format(*debug_query(cursor, query, params)))

backend/kernelCI_app/queries/duration.py

@@ -0,0 +1,43 @@
+from typing import Optional
+
+
+def get_build_duration_clause(
+    builds_duration: tuple[Optional[int], Optional[int]],
+) -> str:
+    clause = ""
+    duration_min, duration_max = builds_duration
+    if duration_min is not None:
+        clause += "AND builds.duration >= %(build_duration_min)s\n"
+    if duration_max is not None:
+        clause += "AND builds.duration <= %(build_duration_max)s\n"
+    return clause
+
+
+def get_boot_test_duration_clause(
+    boots_duration: tuple[Optional[int], Optional[int]],
+    tests_duration: tuple[Optional[int], Optional[int]],
+) -> str:
+    clause = ""
+    duration_min, duration_max = tests_duration
+    if duration_min is not None:
+        clause += (
+            "AND ((tests.path like 'boot.%%' or tests.path = 'boot') "
+            "OR tests.duration >= %(test_duration_min)s)\n"
+        )
+    if duration_max is not None:
+        clause += (
+            "AND ((tests.path like 'boot.%%' or tests.path = 'boot') "
+            "OR tests.duration <= %(test_duration_max)s)\n"
+        )
+    duration_min, duration_max = boots_duration
+    if duration_min is not None:
+        clause += (
+            "AND (NOT (tests.path like 'boot.%%' or tests.path = 'boot') "
+            "OR tests.duration >= %(boot_duration_min)s)\n"
+        )
+    if duration_max is not None:
+        clause += (
+            "AND (NOT (tests.path like 'boot.%%' or tests.path = 'boot') "
+            "OR tests.duration <= %(boot_duration_max)s)\n"
+        )
+    return clause

backend/kernelCI_app/queries/hardware.py

@@ -5,6 +5,10 @@
 
 from kernelCI_app.cache import get_query_cache, set_query_cache
 from kernelCI_app.helpers.database import dict_fetchall
+from kernelCI_app.queries.duration import (
+    get_boot_test_duration_clause,
+    get_build_duration_clause,
+)
 from kernelCI_app.typeModels.hardwareDetails import CommitHead, Tree
 
 
@@ -340,56 +344,6 @@ def get_hardware_details_data(
     return records
 
 
-def _get_build_duration_clause(
-    builds_duration: tuple[Optional[int], Optional[int]],
-) -> str:
-    clause = ""
-
-    # builds
-    duration_min, duration_max = builds_duration
-    if duration_min:
-        clause += "AND builds.duration >= %(build_duration_min)s\n"
-    if duration_max:
-        clause += "AND builds.duration <= %(build_duration_max)s\n"
-
-    return clause
-
-
-def _get_boot_test_duration_clause(
-    boots_duration: tuple[Optional[int], Optional[int]],
-    tests_duration: tuple[Optional[int], Optional[int]],
-) -> str:
-    clause = ""
-
-    # tests
-    duration_min, duration_max = tests_duration
-    if duration_min:
-        clause += (
-            "AND ((tests.path like 'boot.%%' or tests.path = 'boot') "
-            "OR tests.duration >= %(test_duration_min)s)\n"
-        )
-    if duration_max:
-        clause += (
-            "AND ((tests.path like 'boot.%%' or tests.path = 'boot') "
-            "OR tests.duration <= %(test_duration_max)s)\n"
-        )
-
-    # boots
-    duration_min, duration_max = boots_duration
-    if duration_min:
-        clause += (
-            "AND (NOT (tests.path like 'boot.%%' or tests.path = 'boot') "
-            "OR tests.duration >= %(boot_duration_min)s)\n"
-        )
-    if duration_max:
-        clause += (
-            "AND (NOT (tests.path like 'boot.%%' or tests.path = 'boot') "
-            "OR tests.duration <= %(boot_duration_max)s)\n"
-        )
-
-    return clause
-
-
 def get_hardware_details_summary(
     *,
     hardware_id: str,
@@ -427,8 +381,8 @@ def get_hardware_details_summary(
     if query_rows is not None:
         return query_rows
 
-    builds_duration_clause = _get_build_duration_clause(builds_duration)
-    boots_tests_duration_clause = _get_boot_test_duration_clause(
+    builds_duration_clause = get_build_duration_clause(builds_duration)
+    boots_tests_duration_clause = get_boot_test_duration_clause(
         boots_duration, tests_duration
     )