fix: new tree listing count (#1706)

- Fixes problems where trees with null values weren't being correctly compared;
- Also changes behavior of treeListing to keep storing every commit for the trees,
because it is possible to receive multiple checkouts for the same commit.
In case we receive commit A, then commit B, then commit A again,
we should be able to keep the counts from the last commit A checkout.

Closes #1703

Author: MarceloRobert

backend/kernelCI_app/management/commands/helpers/aggregation_helpers.py

@@ -56,6 +56,8 @@ def update_tree_listing(checkouts_instances: Sequence[Checkouts]):
 
     We should also remove the old tree row from the listing
     """
+    if not checkouts_instances:
+        return
 
     t0 = time.time()
     checkout_values = [
@@ -74,7 +76,8 @@ def update_tree_listing(checkouts_instances: Sequence[Checkouts]):
     ]
 
     with connections["default"].cursor() as cursor:
-        # Set values as 0 when inserting a new tree and update as 0 when tree already exists
+        # Set values as 0 when inserting a new tree
+        # and only updates basic information when tree already exists
         cursor.executemany(
             """
             INSERT INTO tree_listing (
@@ -86,22 +89,13 @@ def update_tree_listing(checkouts_instances: Sequence[Checkouts]):
                 test_pass, test_failed, test_inc
             )
             VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 0, 0, 0, 0, 0, 0, 0, 0, 0)
-            ON CONFLICT (origin, tree_name, git_repository_url, git_repository_branch)
+            ON CONFLICT (origin, tree_name, git_repository_url, git_repository_branch, git_commit_hash)
             DO UPDATE SET
                 checkout_id = EXCLUDED.checkout_id,
                 git_commit_hash = EXCLUDED.git_commit_hash,
                 git_commit_name = EXCLUDED.git_commit_name,
                 git_commit_tags = EXCLUDED.git_commit_tags,
-                start_time = EXCLUDED.start_time,
-                build_pass = 0,
-                build_failed = 0,
-                build_inc = 0,
-                boot_pass = 0,
-                boot_failed = 0,
-                boot_inc = 0,
-                test_pass = 0,
-                test_failed = 0,
-                test_inc = 0
+                start_time = EXCLUDED.start_time
             WHERE tree_listing.start_time < EXCLUDED.start_time
             """,
             checkout_values,

backend/kernelCI_app/management/commands/process_pending_aggregations.py

@@ -592,7 +592,11 @@ def _process_tree_listing(
                 data["test_pass"],
                 data["test_failed"],
                 data["test_inc"],
-                data["checkout_id"],
+                data["origin"],
+                data["tree_name"],
+                data["git_repository_branch"],
+                data["git_repository_url"],
+                data["git_commit_hash"],
             )
             for data in tree_listing_data.values()
         ]
@@ -612,7 +616,13 @@ def _process_tree_listing(
                     test_pass = tree_listing.test_pass + %s,
                     test_failed = tree_listing.test_failed + %s,
                     test_inc = tree_listing.test_inc + %s
-                WHERE tree_listing.checkout_id = %s
+                WHERE
+                    -- IS NOT DISTINCT FROM used to treat NULLs as equal
+                    tree_listing.origin = %s
+                    AND tree_listing.tree_name IS NOT DISTINCT FROM %s
+                    AND tree_listing.git_repository_branch IS NOT DISTINCT FROM %s
+                    AND tree_listing.git_repository_url IS NOT DISTINCT FROM %s
+                    AND tree_listing.git_commit_hash IS NOT DISTINCT FROM %s
                 """,
                 values,
             )

backend/kernelCI_app/migrations/0016_alter_treelisting_unique_constraint.py

@@ -0,0 +1,31 @@
+# Generated by Django 5.2.9 on 2026-01-20 18:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("kernelCI_app", "0015_alter_pendingtest_platform"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="treelisting",
+            name="tree_listing_tree_unique",
+        ),
+        migrations.AddConstraint(
+            model_name="treelisting",
+            constraint=models.UniqueConstraint(
+                fields=(
+                    "origin",
+                    "tree_name",
+                    "git_repository_url",
+                    "git_repository_branch",
+                    "git_commit_hash",
+                ),
+                name="tree_listing_tree_unique",
+                nulls_distinct=False,
+            ),
+        ),
+    ]

backend/kernelCI_app/models.py

@@ -391,8 +391,10 @@ class Meta:
                     "tree_name",
                     "git_repository_url",
                     "git_repository_branch",
+                    "git_commit_hash",
                 ],
                 name="tree_listing_tree_unique",
+                nulls_distinct=False,
             ),
         ]
         indexes = [

backend/kernelCI_app/queries/tree.py

@@ -302,30 +302,37 @@ def get_tree_listing_data_denormalized(
     with connection.cursor() as cursor:
         cursor.execute(
             """
+            WITH latest AS (
+                SELECT
+                    checkout_id
+                FROM
+                    latest_checkout
+                WHERE
+                    start_time >= NOW() - INTERVAL %(interval_param)s
+                    AND origin = %(origin_param)s
+            )
             SELECT
-                checkout_id,
-                origin,
-                tree_name,
-                git_repository_url,
-                git_repository_branch,
-                git_commit_hash,
-                git_commit_name,
-                git_commit_tags,
-                start_time,
-                build_pass,
-                build_failed,
-                build_inc,
-                boot_pass,
-                boot_failed,
-                boot_inc,
-                test_pass,
-                test_failed,
-                test_inc
+                tl.checkout_id,
+                tl.origin,
+                tl.tree_name,
+                tl.git_repository_url,
+                tl.git_repository_branch,
+                tl.git_commit_hash,
+                tl.git_commit_name,
+                tl.git_commit_tags,
+                tl.start_time,
+                tl.build_pass,
+                tl.build_failed,
+                tl.build_inc,
+                tl.boot_pass,
+                tl.boot_failed,
+                tl.boot_inc,
+                tl.test_pass,
+                tl.test_failed,
+                tl.test_inc
             FROM
-                tree_listing
-            WHERE
-                start_time >= NOW() - INTERVAL %(interval_param)s
-                AND origin = %(origin_param)s
+                tree_listing tl
+                JOIN latest l ON tl.checkout_id = l.checkout_id
             """,
             params,
         )