feat: Change queries to use lab column information

    * For analysis queries we are going for lab column information
      first, and coalescing to json misc information.
    *  All COALESCE expressions are marked with TODO comments for removal
      after the lab_id backfill is complete.

    Closes #1948

Signed-off-by: Alan Peixinho <alan.peixinho@profusion.mobi>

Author: alanpeixinho

backend/kernelCI_app/helpers/hardwareDetails.py

@@ -290,8 +290,6 @@ def handle_test_history(
     full_environment_misc: bool = False,
 ) -> None:
     create_record_test_platform(record=record)
-    record_misc = sanitize_dict(record.get("misc"))
-
     environment_misc_dict = get_environment_misc_value(
         full_environment_misc=full_environment_misc,
         parsed_environment_misc=record.get("parsed_environment_misc"),
@@ -313,11 +311,7 @@ def handle_test_history(
         environment_misc=environment_misc,
         tree_name=record["build__checkout__tree_name"],
         git_repository_branch=record["build__checkout__git_repository_branch"],
-        lab=(
-            record_misc.get("runtime", UNKNOWN_STRING)
-            if record_misc
-            else UNKNOWN_STRING
-        ),
+        lab=record.get("lab") or UNKNOWN_STRING,
     )
 
     task.append(test_history_item)
@@ -380,16 +374,14 @@ def handle_test_summary(
         getattr(task.origins[origin], status) + 1,
     )
 
-    misc = sanitize_dict(record.get("misc")) or {}
-    lab = misc.get("runtime", UNKNOWN_STRING)
-    if lab:
-        if task.labs.get(lab) is None:
-            task.labs[lab] = StatusCount()
-        setattr(
-            task.labs[lab],
-            status,
-            getattr(task.labs[lab], status) + 1,
-        )
+    lab = record.get("lab") or UNKNOWN_STRING
+    if task.labs.get(lab) is None:
+        task.labs[lab] = StatusCount()
+    setattr(
+        task.labs[lab],
+        status,
+        getattr(task.labs[lab], status) + 1,
+    )
 
 
 def handle_build_history(
@@ -458,18 +450,16 @@ def handle_build_summary(
             getattr(builds_summary.origins[origin], status_key) + 1,
         )
 
-    misc = sanitize_dict(build.misc) or {}
-    lab = misc.get("lab", UNKNOWN_STRING)
-    if lab:
-        build_lab_summary = builds_summary.labs.get(lab)
-        if not build_lab_summary:
-            build_lab_summary = StatusCount()
-            builds_summary.labs[lab] = build_lab_summary
-        setattr(
-            builds_summary.labs[lab],
-            status_key,
-            getattr(builds_summary.labs[lab], status_key) + 1,
-        )
+    lab = record.get("build_lab") or UNKNOWN_STRING
+    build_lab_summary = builds_summary.labs.get(lab)
+    if not build_lab_summary:
+        build_lab_summary = StatusCount()
+        builds_summary.labs[lab] = build_lab_summary
+    setattr(
+        builds_summary.labs[lab],
+        status_key,
+        getattr(builds_summary.labs[lab], status_key) + 1,
+    )
 
     process_issue(record=record, task_issues_dict=issue_dict, issue_from="build")
 
@@ -579,8 +569,7 @@ def decide_if_is_full_record_filtered_out(
     if not is_current_tree_selected:
         return True
 
-    misc = sanitize_dict(record.get("misc")) or {}
-    lab = misc.get("runtime", UNKNOWN_STRING)
+    lab = record.get("lab") or UNKNOWN_STRING
 
     is_record_filtered_out_result = instance.filters.is_record_filtered_out(
         hardwares=record["environment_compatible"],
@@ -730,10 +719,8 @@ def process_filters(*, instance, record: Dict) -> None:
 
         instance.unfiltered_origins["build"].add(record["build__origin"])
 
-        build_misc = sanitize_dict(record.get("build__misc")) or {}
-        build_lab = build_misc.get("lab")
-        if build_lab:
-            instance.unfiltered_labs["build"].add(build_lab)
+        if record.get("build_lab"):
+            instance.unfiltered_labs["build"].add(record["build_lab"])
 
     if record["id"] is not None:
         if is_boot(record["path"]):
@@ -771,10 +758,8 @@ def process_filters(*, instance, record: Dict) -> None:
         platform_set.add(test_platform)
         origin_set.add(record["test_origin"])
 
-        test_misc = sanitize_dict(record.get("misc")) or {}
-        test_lab = test_misc.get("runtime")
-        if test_lab:
-            instance.unfiltered_labs[flag_tab].add(test_lab)
+        if record.get("lab"):
+            instance.unfiltered_labs[flag_tab].add(record["lab"])
 
 
 def is_record_tree_selected(*, record, tree: Tree, is_all_selected: bool) -> bool:

backend/kernelCI_app/helpers/treeDetails.py

@@ -25,7 +25,6 @@
     create_issue_typed,
     extract_error_message,
     is_boot,
-    sanitize_dict,
 )
 
 
@@ -87,42 +86,41 @@ def get_current_row_data(
         "test_number_value": current_row[10],
         "test_misc": current_row[11],
         "test_environment_compatible": current_row[tmp_test_env_comp_key],
-        "build_id": current_row[13],
-        "build_origin": current_row[14],
-        "build_comment": current_row[15],
-        "build_start_time": current_row[16],
-        "build_duration": current_row[17],
-        "build_architecture": current_row[18],
-        "build_command": current_row[19],
-        "build_compiler": current_row[20],
-        "build_config_name": current_row[21],
-        "build_config_url": current_row[22],
-        "build_log_url": current_row[23],
-        "build_status": current_row[24],
-        "build_misc": current_row[25],
-        "checkout_id": current_row[26],
-        "checkout_git_repository_url": current_row[27],
-        "checkout_git_repository_branch": current_row[28],
-        "checkout_git_commit_tags": current_row[29],
-        "checkout_origin": current_row[30],
-        "incident_id": current_row[31],
-        "incident_test_id": current_row[32],
-        "incident_present": current_row[33],
-        "issue_id": current_row[34],
-        "issue_version": current_row[35],
-        "issue_comment": current_row[36],
-        "issue_report_url": current_row[37],
+        "test_lab": current_row[13],
+        "build_id": current_row[14],
+        "build_origin": current_row[15],
+        "build_comment": current_row[16],
+        "build_start_time": current_row[17],
+        "build_duration": current_row[18],
+        "build_architecture": current_row[19],
+        "build_command": current_row[20],
+        "build_compiler": current_row[21],
+        "build_config_name": current_row[22],
+        "build_config_url": current_row[23],
+        "build_log_url": current_row[24],
+        "build_status": current_row[25],
+        "build_misc": current_row[26],
+        "build_lab": current_row[27],
+        "checkout_id": current_row[28],
+        "checkout_git_repository_url": current_row[29],
+        "checkout_git_repository_branch": current_row[30],
+        "checkout_git_commit_tags": current_row[31],
+        "checkout_origin": current_row[32],
+        "incident_id": current_row[33],
+        "incident_test_id": current_row[34],
+        "incident_present": current_row[35],
+        "issue_id": current_row[36],
+        "issue_version": current_row[37],
+        "issue_comment": current_row[38],
+        "issue_report_url": current_row[39],
     }
 
     parsed_environment_misc = handle_misc(
         misc_value_or_default(current_row_data["test_environment_misc"])
     )
     current_row_data["test_platform"] = parsed_environment_misc.get("platform")
     current_row_data["parsed_environment_misc"] = parsed_environment_misc
-    test_misc = sanitize_dict(current_row_data["test_misc"])
-    test_runtime_lab = UNKNOWN_STRING
-    if test_misc is not None:
-        test_runtime_lab = test_misc.get("runtime", UNKNOWN_STRING)
+    test_runtime_lab = current_row_data["test_lab"] or UNKNOWN_STRING
 
     if current_row_data["test_status"] is None:
         current_row_data["test_status"] = NULL_STATUS
@@ -495,8 +493,9 @@ def process_filters(instance, row_data: dict, skip_build_filters: bool = False)
         instance.global_architectures.add(row_data["build_architecture"])
         instance.global_compilers.add(row_data["build_compiler"])
         instance.unfiltered_origins["build"].add(row_data["build_origin"])
-        if (build_misc := row_data["build_misc"]) is not None:
-            instance.unfiltered_labs["build"].add(build_misc.get("lab", UNKNOWN_STRING))
+        instance.unfiltered_labs["build"].add(
+            row_data.get("build_lab") or UNKNOWN_STRING
+        )
 
         build_issue_id, build_issue_version, is_build_issue = (
             should_increment_build_issue(

backend/kernelCI_app/management/commands/notifications.py

@@ -706,7 +706,7 @@ def generate_hardware_summary_report(
             continue
         hardware_id = environment_misc.get("platform")
         raw["job_id"] = environment_misc.get("job_id")
-        raw["runtime"] = misc.get("runtime")
+        raw["runtime"] = raw.get("lab") or misc.get("runtime")
         origin = raw.get("test_origin")
         key = (hardware_id, origin)
         tree = raw.get("tree_name")

backend/kernelCI_app/migrations/0019_labs_builds_lab_tests_lab.py

@@ -0,0 +1,43 @@
+# Generated by Django 5.2.11 on 2026-06-29 20:25
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("kernelCI_app", "0018_hardwareregistryplatformvendor_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Labs",
+            fields=[
+                ("id", models.AutoField(primary_key=True, serialize=False)),
+                ("name", models.TextField(unique=True)),
+            ],
+            options={
+                "db_table": "labs",
+            },
+        ),
+        migrations.AddField(
+            model_name="builds",
+            name="lab",
+            field=models.ForeignKey(
+                db_constraint=False,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to="kernelCI_app.labs",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tests",
+            name="lab",
+            field=models.ForeignKey(
+                db_constraint=False,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to="kernelCI_app.labs",
+            ),
+        ),
+    ]

backend/kernelCI_app/queries/build.py

@@ -1,6 +1,8 @@
 from typing import Optional
 
+from django.db.models import TextField
 from django.db.models.expressions import F
+from django.db.models.functions import Cast, Coalesce
 from querybuilder.query import Query
 
 from kernelCI_app.models import Builds, Tests
@@ -51,7 +53,13 @@ def get_build_details(build_id: str) -> Optional[list[dict]]:
 def get_build_tests(build_id: str) -> Optional[list[dict]]:
     result = (
         Tests.objects.filter(build_id=build_id)
-        .annotate(lab=F("misc__runtime"))
+        # TODO remove misc__runtime fallback after lab backfill
+        .annotate(
+            lab_name=Coalesce(
+                F("lab__name"),
+                Cast(F("misc__runtime"), output_field=TextField()),
+            )
+        )
         .values(
             "id",
             "duration",
@@ -61,7 +69,12 @@ def get_build_tests(build_id: str) -> Optional[list[dict]]:
             "environment_compatible",
             "environment_misc",
             "build__status",
-            "lab",
+            "lab_name",
         )
     )
-    return list(result)
+    tests = []
+    for test in result:
+        test["lab"] = test.pop("lab_name")
+        tests.append(test)
+
+    return tests

backend/kernelCI_app/queries/hardware.py

@@ -255,6 +255,7 @@ def get_hardware_listing_data_bulk(
                 FROM
                     tests
                     INNER JOIN builds b ON tests.build_id = b.id
+                    LEFT JOIN labs tl ON tests.lab_id = tl.id
                 WHERE
                     "tests"."environment_misc" ->> 'platform' IS NOT NULL
                     AND "tests"."start_time" >= %(start_date)s
@@ -267,7 +268,8 @@ def get_hardware_listing_data_bulk(
                         OR tests.environment_misc ->> 'platform' = key_list.hardware_id
                     )
                     AND tests.origin = key_list.origin
-                    AND tests.misc ->> 'runtime' = key_list.lab_name
+                    -- TODO remove misc->>'runtime' fallback after lab backfill
+                    AND COALESCE(tl.name, tests.misc ->> 'runtime') = key_list.lab_name
                     )
             )
         SELECT
@@ -478,7 +480,8 @@ def get_hardware_details_summary(
                      AS known_issues,
                  array[builds.compiler, builds.architecture] AS compiler_arch,
                  builds.config_name,
-                 builds.misc->>'lab' AS lab,
+                 -- TODO remove misc->>'lab' fallback after lab backfill
+                 COALESCE(bl.name, builds.misc->>'lab') AS lab,
                  tests.environment_misc->>'platform' AS platform,
                  tests.environment_compatible,
                  checkouts.origin,
@@ -497,6 +500,7 @@ def get_hardware_details_summary(
                 tests.build_id = builds.id
             INNER JOIN checkouts ON
                 builds.checkout_id = checkouts.id
+            LEFT JOIN labs bl ON builds.lab_id = bl.id
             LEFT OUTER JOIN incidents ON
                 builds.id = incidents.build_id
             WHERE
@@ -522,7 +526,8 @@ def get_hardware_details_summary(
                      as known_issues,
                  array[builds.compiler, builds.architecture] AS compiler_arch,
                  builds.config_name,
-                 tests.misc->>'runtime' AS lab,
+                 -- TODO remove misc->>'runtime' fallback after lab backfill
+                 COALESCE(tl.name, tests.misc->>'runtime') AS lab,
                  tests.environment_misc->>'platform' AS platform,
                  tests.environment_compatible,
                  checkouts.origin,
@@ -541,6 +546,7 @@ def get_hardware_details_summary(
                 tests.build_id = builds.id
             INNER JOIN checkouts ON
                 builds.checkout_id = checkouts.id
+            LEFT JOIN labs tl ON tests.lab_id = tl.id
             LEFT OUTER JOIN incidents ON
                 tests.id = incidents.test_id
             WHERE
@@ -626,13 +632,21 @@ def query_records(
                 issues.report_url AS incidents__issue__report_url,
                 incidents.test_id AS incidents__test_id,
                 T7.issue_id AS build__incidents__issue__id,
-                T8.version AS build__incidents__issue__version
+                T8.version AS build__incidents__issue__version,
+                -- TODO remove misc->>'runtime' fallback after lab backfill
+                COALESCE(tl.name, tests.misc->>'runtime') AS lab,
+                -- TODO remove misc->>'lab' fallback after lab backfill
+                COALESCE(bl.name, builds.misc->>'lab') AS build_lab
             FROM
                 tests
             INNER JOIN builds ON
                 tests.build_id = builds.id
             INNER JOIN checkouts ON
                 builds.checkout_id = checkouts.id
+            LEFT JOIN labs tl ON
+                tests.lab_id = tl.id
+            LEFT JOIN labs bl ON
+                builds.lab_id = bl.id
             LEFT OUTER JOIN incidents ON
                 tests.id = incidents.test_id
             LEFT OUTER JOIN issues ON
@@ -721,13 +735,16 @@ def get_hardware_summary_data(
                 checkouts.git_commit_name,
                 checkouts.git_commit_hash,
                 checkouts.tree_name,
-                checkouts.origin AS checkout_origin
+                checkouts.origin AS checkout_origin,
+                -- TODO remove misc->>'runtime' fallback after lab backfill
+                COALESCE(tl.name, tests.misc->>'runtime') AS lab
             FROM
                 tests
             INNER JOIN builds ON
                 tests.build_id = builds.id
             INNER JOIN checkouts ON
                 builds.checkout_id = checkouts.id
+            LEFT JOIN labs tl ON tests.lab_id = tl.id
             WHERE
                 tests.start_time >= %s
                 AND tests.start_time <= %s
@@ -740,7 +757,8 @@ def get_hardware_summary_data(
                     OR tests.environment_misc ->> 'platform' = key_list.hardware_id
                 )
                 AND tests.origin = key_list.origin
-                AND tests.misc ->> 'runtime' = key_list.lab_name
+                -- TODO remove misc->>'runtime' fallback after lab backfill
+                AND COALESCE(tl.name, tests.misc ->> 'runtime') = key_list.lab_name
                 )
             ORDER BY
                 tests.start_time DESC