feat: dynamic metrics page (#1920)

* feat: Include page for interactive notification metrics

Signed-off-by: Alan Peixinho <alan.peixinho@profusion.mobi>

* feat(metrics): expose new build issues by origin

Add a parallel query that lists build issues whose first-ever build
incident falls in the selected period, and return them as
new_issues_by_origin on GET /api/metrics/.

Signed-off-by: Gustavo Flores <gustavo.flores@profusion.mobi>

* feat(metrics): expand build regressions with new issue list

Add per-origin expandable rows in Build Regressions that show new
issue comments, incident counts, and links to issue details.

Signed-off-by: Gustavo Flores <gustavo.flores@profusion.mobi>

* feat: cache metrics query work

---------

Signed-off-by: Alan Peixinho <alan.peixinho@profusion.mobi>
Signed-off-by: Gustavo Flores <gustavo.flores@profusion.mobi>
Co-authored-by: Alan Peixinho <alan.peixinho@profusion.mobi>

Author: Gustavo Flores

backend/kernelCI_app/queries/notifications.py

@@ -5,6 +5,7 @@
 from django.db import connection, connections
 from pydantic import ValidationError
 
+from kernelCI_app.cache import get_query_cache, set_query_cache
 from kernelCI_app.helpers.database import dict_fetchall
 from kernelCI_app.helpers.logger import out
 from kernelCI_app.queries.tree import get_tree_listing_query
@@ -644,22 +645,32 @@ def get_issues_summary_data(*, checkout_ids: list[str]) -> list[dict]:
         return dict_fetchall(cursor=cursor)
 
 
-def query_fetchone_work(*, query: str, params: dict[str, Any]):
+def query_fetchone_work(*, cache_key: str, query: str, params: dict[str, Any]):
+    rows = get_query_cache(key=cache_key, params=params)
+    if rows is not None:
+        return rows
     try:
         with connections["default"].cursor() as cursor:
             cursor.execute(query, params)
-            return cursor.fetchone()
+            rows = cursor.fetchone()
     finally:
         connections["default"].close()
+    set_query_cache(key=cache_key, params=params, rows=rows)
+    return rows
 
 
-def query_fetchall_work(*, query: str, params: dict[str, Any]):
+def query_fetchall_work(*, cache_key: str, query: str, params: dict[str, Any]):
+    rows = get_query_cache(key=cache_key, params=params)
+    if rows is not None:
+        return rows
     try:
         with connections["default"].cursor() as cursor:
             cursor.execute(query, params)
-            return cursor.fetchall()
+            rows = cursor.fetchall()
     finally:
         connections["default"].close()
+    set_query_cache(key=cache_key, params=params, rows=rows)
+    return rows
 
 
 def get_metrics_data(
@@ -790,6 +801,42 @@ def get_metrics_data(
     WHERE r.ranked <= 3 AND n.total_incidents > 0
     """
 
+    new_build_issues_query = """
+    WITH time_rank AS (
+        SELECT
+            _timestamp,
+            origin,
+            issue_id,
+            ROW_NUMBER() OVER (PARTITION BY issue_id ORDER BY _timestamp) AS rn
+        FROM incidents
+        WHERE build_id IS NOT NULL
+    ),
+    new_issues AS (
+        SELECT issue_id, origin
+        FROM time_rank
+        WHERE rn = 1
+            AND _timestamp BETWEEN
+                NOW() - INTERVAL %(start_days_ago)s
+                AND NOW() - INTERVAL %(end_days_ago)s
+    )
+    SELECT
+        inc.origin,
+        inc.issue_id,
+        inc.issue_version,
+        i.comment,
+        COUNT(inc.*) AS total
+    FROM incidents inc
+    JOIN issues i ON inc.issue_id = i.id AND inc.issue_version = i.version
+    JOIN new_issues ni ON inc.issue_id = ni.issue_id AND inc.origin = ni.origin
+    WHERE
+        inc.build_id IS NOT NULL
+        AND inc._timestamp BETWEEN
+            NOW() - INTERVAL %(start_days_ago)s
+            AND NOW() - INTERVAL %(end_days_ago)s
+    GROUP BY inc.origin, inc.issue_id, inc.issue_version, i.comment
+    ORDER BY inc.origin, total DESC
+    """
+
     lab_summary_query = """
     -- get count of tests of each lab and how many builds are related to those tests
     SELECT
@@ -806,32 +853,55 @@ def get_metrics_data(
     GROUP BY lab
     """
 
-    with ThreadPoolExecutor(max_workers=5) as executor:
+    with ThreadPoolExecutor(max_workers=6) as executor:
         total_objects_result = executor.submit(
-            query_fetchone_work, query=total_objects_query, params=params
+            query_fetchone_work,
+            cache_key="metricsTotalObjects",
+            query=total_objects_query,
+            params=params,
         )
         prev_total_objects_result = executor.submit(
-            query_fetchone_work, query=total_objects_query, params=prev_params
+            query_fetchone_work,
+            cache_key="metricsTotalObjects",
+            query=total_objects_query,
+            params=prev_params,
         )
         build_incidents_result = executor.submit(
-            query_fetchall_work, query=build_incidents_query, params=params
+            query_fetchall_work,
+            cache_key="metricsBuildIncidents",
+            query=build_incidents_query,
+            params=params,
+        )
+        new_build_issues_result = executor.submit(
+            query_fetchall_work,
+            cache_key="metricsNewBuildIssues",
+            query=new_build_issues_query,
+            params=params,
         )
         lab_summary_results = executor.submit(
-            query_fetchall_work, query=lab_summary_query, params=params
+            query_fetchall_work,
+            cache_key="metricsLabSummary",
+            query=lab_summary_query,
+            params=params,
         )
         prev_lab_summary_results = executor.submit(
-            query_fetchall_work, query=lab_summary_query, params=prev_params
+            query_fetchall_work,
+            cache_key="metricsLabSummary",
+            query=lab_summary_query,
+            params=prev_params,
         )
 
     total_objects_result = total_objects_result.result()
     prev_total_objects_result = prev_total_objects_result.result()
     build_incidents_result = build_incidents_result.result()
+    new_build_issues_result = new_build_issues_result.result()
     lab_summary_results = lab_summary_results.result()
     prev_lab_summary_results = prev_lab_summary_results.result()
 
     try:
         build_incidents_by_origin: dict[str, BuildIncidentsCount] = {}
         top_issues_by_origin: dict[str, dict[tuple[str, int], TopIssue]] = {}
+        new_issues_by_origin: dict[str, dict[tuple[str, int], TopIssue]] = {}
         for row in build_incidents_result:
             origin = row[0]
             issue_id = row[4]
@@ -851,6 +921,19 @@ def get_metrics_data(
                 total_incidents=row[7],
             )
 
+        for row in new_build_issues_result:
+            origin = row[0]
+            issue_id = row[1]
+            issue_version = row[2]
+            if new_issues_by_origin.get(origin) is None:
+                new_issues_by_origin[origin] = {}
+            new_issues_by_origin[origin][(issue_id, issue_version)] = TopIssue(
+                id=issue_id,
+                version=issue_version,
+                comment=row[3],
+                total_incidents=row[4],
+            )
+
         data = MetricsReportData(
             n_trees=total_objects_result[0],
             n_checkouts=total_objects_result[1],
@@ -860,6 +943,7 @@ def get_metrics_data(
             n_incidents=total_objects_result[5],
             build_incidents_by_origin=build_incidents_by_origin,
             top_issues_by_origin=top_issues_by_origin,
+            new_issues_by_origin=new_issues_by_origin,
             lab_maps={
                 row[0]: LabMetricsData(
                     builds=row[1],

backend/kernelCI_app/tests/integrationTests/metrics_test.py

@@ -21,6 +21,7 @@
     "n_incidents",
     "build_incidents_by_origin",
     "top_issues_by_origin",
+    "new_issues_by_origin",
     "lab_maps",
     "prev_n_trees",
     "prev_n_checkouts",

backend/kernelCI_app/tests/unitTests/commands/metrics_notifications_test.py

@@ -62,6 +62,16 @@ def make_metrics_data(**overrides) -> MetricsReportData:
                 ),
             },
         },
+        new_issues_by_origin={
+            "maestro": {
+                ("issue-new", 1): TopIssue(
+                    id="issue-new",
+                    version=1,
+                    comment="New regression issue",
+                    total_incidents=10,
+                ),
+            },
+        },
         lab_maps={
             "lava-collabora": LabMetricsData(
                 builds=0, boots=50000, tests=450000, origin="maestro"
@@ -242,6 +252,7 @@ def test_render_receives_fields(
             n_incidents=75,
             build_incidents_by_origin=mock.ANY,
             top_issues_by_origin=mock.ANY,
+            new_issues_by_origin=mock.ANY,
             lab_maps=mock.ANY,
             prev_n_trees=100,
             prev_n_checkouts=1000,

backend/kernelCI_app/typeModels/metrics.py

@@ -34,6 +34,7 @@ class MetricsResponse(BaseModel):
     n_incidents: int
     build_incidents_by_origin: dict[str, BuildIncidentsCount]
     top_issues_by_origin: dict[str, list[TopIssue]]
+    new_issues_by_origin: dict[str, list[TopIssue]]
     lab_maps: dict[str, LabMetricsData]
     prev_n_trees: int
     prev_n_checkouts: int
@@ -55,6 +56,10 @@ def metrics_report_data_to_response(data: MetricsReportData) -> MetricsResponse:
             origin: list(issues.values())
             for origin, issues in data.top_issues_by_origin.items()
         },
+        new_issues_by_origin={
+            origin: list(issues.values())
+            for origin, issues in data.new_issues_by_origin.items()
+        },
         lab_maps=data.lab_maps,
         prev_n_trees=data.prev_n_trees,
         prev_n_checkouts=data.prev_n_checkouts,

backend/kernelCI_app/typeModels/metrics_notifications.py

@@ -34,6 +34,8 @@ class MetricsReportData(BaseModel):
     build_incidents_by_origin: dict[str, BuildIncidentsCount]
     # top_issues = origin -> (issue_id, version) -> TopIssue
     top_issues_by_origin: dict[str, dict[tuple[str, int], TopIssue]]
+    # new_issues = origin -> (issue_id, version) -> TopIssue (first build incident in period)
+    new_issues_by_origin: dict[str, dict[tuple[str, int], TopIssue]]
     lab_maps: dict[str, LabMetricsData]
     # Previous interval (for comparison)
     prev_n_trees: int

dashboard/src/api/metrics.ts

@@ -0,0 +1,39 @@
+import { useQuery, type UseQueryResult } from '@tanstack/react-query';
+
+import type { MetricsResponse } from '@/types/metrics';
+
+import { RequestData } from './commonRequest';
+
+type FetchMetricsParams = {
+  startDaysAgo: number;
+  endDaysAgo: number;
+};
+
+export const fetchMetrics = async ({
+  startDaysAgo,
+  endDaysAgo,
+}: FetchMetricsParams): Promise<MetricsResponse> => {
+  const data = await RequestData.get<MetricsResponse>('/api/metrics/', {
+    params: {
+      start_days_ago: startDaysAgo,
+      end_days_ago: endDaysAgo,
+    },
+  });
+
+  return data;
+};
+
+export const useMetrics = ({
+  intervalInDays,
+}: {
+  intervalInDays: number;
+}): UseQueryResult<MetricsResponse> => {
+  return useQuery({
+    queryKey: ['metrics', intervalInDays],
+    queryFn: () =>
+      fetchMetrics({
+        startDaysAgo: intervalInDays,
+        endDaysAgo: 0,
+      }),
+  });
+};

dashboard/src/components/SideMenu/menuItems.tsx

@@ -1,6 +1,6 @@
 import type { JSX } from 'react';
 
-import { MdOutlineMonitorHeart } from 'react-icons/md';
+import { MdOutlineBarChart, MdOutlineMonitorHeart } from 'react-icons/md';
 import { RxRadiobutton } from 'react-icons/rx';
 import { ImTree } from 'react-icons/im';
 import { HiOutlineDocumentSearch } from 'react-icons/hi';
@@ -30,6 +30,7 @@ export type LinkStringItems = {
 const TreeIcon = <ImTree className="size-5" />;
 const MonitorHeartIcon = <MdOutlineMonitorHeart className="size-5" />;
 const RadioButtonIcon = <RxRadiobutton className="size-5" />;
+const MetricsIcon = <MdOutlineBarChart className="size-5" />;
 const DocumentSearchIcon = <HiOutlineDocumentSearch />;
 
 export const routeItems: RouteMenuItems[] = [
@@ -51,6 +52,12 @@ export const routeItems: RouteMenuItems[] = [
     icon: RadioButtonIcon,
     selected: false,
   },
+  {
+    navigateTo: '/metrics',
+    idIntl: 'routes.metricsMonitor',
+    icon: MetricsIcon,
+    selected: false,
+  },
 ];
 
 export const linkItems: LinkMenuItems[] = [

dashboard/src/components/TopBar/TopBar.tsx

@@ -108,6 +108,8 @@ const TitleName = ({ basePath }: { basePath: string }): JSX.Element => {
       return <FormattedMessage id="routes.testDetails" />;
     case 'issue':
       return <FormattedMessage id="routes.issueDetails" />;
+    case 'metrics':
+      return <FormattedMessage id="routes.metricsMonitor" />;
     default:
       return <FormattedMessage id="routes.unknown" />;
   }

dashboard/src/locales/messages/index.ts

@@ -291,6 +291,7 @@ export const messages = {
     'routes.hardwareNewMonitor': 'Hardware New',
     'routes.issueDetails': 'Issue',
     'routes.issueMonitor': 'Issues',
+    'routes.metricsMonitor': 'Metrics',
     'routes.testDetails': 'Test',
     'routes.treeMonitor': 'Trees',
     'routes.unknown': 'Unknown',