refactor: handle branches and channels separately

This also makes an incremental move to displaying new evaluations based
on branches as such, while preserving display of channel-oriented evaluations.

Author: fricklerhandwerk

src/shared/auth/utils.py

@@ -19,7 +19,7 @@ def ismaintainer(user: Any) -> bool:
     return NixMaintainer.objects.filter(
         github_id=user.socialaccount_set.get(provider="github").uid,
         nixderivationmeta__derivation__parent_evaluation__commit_sha1=F(
-            "nixderivationmeta__derivation__parent_evaluation__channel__head_sha1_commit"
+            "nixderivationmeta__derivation__parent_evaluation__branch__head_sha1_commit"
         ),
     ).exists()
 

src/shared/cache_suggestions.py

@@ -19,7 +19,6 @@
     PackageOverlay,
     ReferenceUrlOverlay,
 )
-from shared.models.nix_evaluation import get_major_channel
 
 logger = logging.getLogger(__name__)
 
@@ -211,8 +210,9 @@ def cache_new_suggestions(suggestion: CVEDerivationClusterProposal) -> None:
             "metadata",
             "package_link__package",
             "parent_evaluation",
-            "parent_evaluation__channel",
+            "parent_evaluation__branch",
         )
+        .prefetch_related("parent_evaluation__branch__channels")
         .prefetch_related(
             Prefetch(
                 "metadata__maintainers",
@@ -316,69 +316,61 @@ def channel_structure(
         packages[attribute_path].derivation_ids.append(derivation.pk)
         if (
             derivation.metadata
-            and derivation.parent_evaluation.channel.is_tracking_branch
+            and derivation.parent_evaluation.branch.is_tracking_branch
         ):
             packages[attribute_path].description = derivation.metadata.get_description()
             packages[attribute_path].maintainers = [
                 CachedSuggestion.Maintainer.model_validate(to_dict(m))
                 for m in derivation.metadata.prefetched_maintainers
             ]
-        # Get the branch from which that derivation originates
-        branch_name = derivation.parent_evaluation.channel.channel_branch
-        # Get primary ("major") channel to which that branch belongs
-        major_channel = get_major_channel(branch_name)
-        # FIXME This quietly drops unfamiliar branch names
-        if major_channel:
-            # XXX(@fricklerhandwerk): Here we assign package information to channel names in iteration order, which in the query we have established to be oldest-first by evaluation time.
-            channels = packages[attribute_path].channels
-            if major_channel not in channels:
-                channels[major_channel] = CachedSuggestion.PackageOnPrimaryChannel(
-                    major_version=None,
-                    status=None,
-                    src_position=None,
-                    # XXX(@fricklerhandwerk): If this is not replaced in subsequent processing, it will display "-"
-                    uniform_versions=None,
-                    sub_branches=dict(),
-                    updated=None,
-                )
-            if branch_name == major_channel:
-                channels[major_channel] = CachedSuggestion.PackageOnPrimaryChannel(
+        all_channels = list(derivation.parent_evaluation.branch.channels.all())
+        status = is_version_affected(
+            [c.affects(package_version) for c in version_constraints]
+        )
+        src_position = get_src_position(derivation)
+        updated = derivation.parent_evaluation.updated_at
+        if len(all_channels) > 1:
+            # Old style: multiple channels were evaluated separately.
+            # Display under the primary channel with others nested.
+            # FIXME(@fricklerhandwerk): Ideally we'd garbage-collect all those evaluations,
+            # except the one with the latest commit.
+            # That would require knowing the order of evaluation commits:
+            # https://docs.github.com/en/rest/commits/commits?apiVersion=2026-03-10#compare-two-commits
+            primary = next((c for c in all_channels if c.primary), None)
+            if primary is None:
+                continue
+            group_name = primary.channel_branch
+            packages[attribute_path].channels[group_name] = (
+                CachedSuggestion.PackageOnPrimaryChannel(
                     major_version=package_version,
-                    status=is_version_affected(
-                        [c.affects(package_version) for c in version_constraints]
-                    ),
-                    src_position=get_src_position(derivation),
-                    uniform_versions=channels[major_channel].uniform_versions,
-                    sub_branches=channels[major_channel].sub_branches,
-                    updated=derivation.parent_evaluation.updated_at,
+                    status=status,
+                    src_position=src_position,
+                    uniform_versions=True,
+                    sub_branches={
+                        c.channel_branch: CachedSuggestion.PackageOnBranch(
+                            version=package_version,
+                            status=status,
+                            src_position=src_position,
+                            updated=updated,
+                        )
+                        for c in all_channels
+                        if not c.primary
+                    },
+                    updated=updated,
                 )
-            else:
-                channels[major_channel].sub_branches[branch_name] = (
-                    CachedSuggestion.PackageOnBranch(
-                        version=package_version,
-                        status=is_version_affected(
-                            [c.affects(package_version) for c in version_constraints]
-                        ),
-                        src_position=get_src_position(derivation),
-                        updated=derivation.parent_evaluation.updated_at,
-                    )
-                )
-
-    for package_name in packages:
-        channels = packages[package_name].channels
-        for mc in channels.keys():
-            uniform_versions = True
-            major_version = channels[mc].major_version
-            for _, branch in channels[mc].sub_branches.items():
-                uniform_versions = (
-                    uniform_versions and str(major_version) == branch.version
-                )
-            channels[mc].uniform_versions = uniform_versions
-            # We just sort branch names by length to get a good-enough order
-            channels[mc].sub_branches = dict(
-                sorted(
-                    channels[mc].sub_branches.items(),
-                    reverse=True,
+            )
+        else:
+            # New style: evaluations per release branch.
+            # Display the version for each branch separately
+            group_name = derivation.parent_evaluation.branch.name
+            packages[attribute_path].channels[group_name] = (
+                CachedSuggestion.PackageOnPrimaryChannel(
+                    major_version=package_version,
+                    status=status,
+                    src_position=src_position,
+                    uniform_versions=True,
+                    sub_branches={},
+                    updated=updated,
                 )
             )
     return packages

src/shared/channels.py

@@ -6,18 +6,18 @@
 from shared.models.cve import Container
 from shared.models.issue import NixpkgsIssue
 from shared.models.linkage import CVEDerivationClusterProposal
-from shared.models.nix_evaluation import NixChannel, NixEvaluation
+from shared.models.nix_evaluation import NixEvaluation, NixpkgsBranch
 
 
 @dataclass
-class NixChannelUpdateChannel(TriggerChannel):
-    model = NixChannel
+class NixpkgsBranchInsertChannel(TriggerChannel):
+    model = NixpkgsBranch
     lock_notifications = True
 
 
 @dataclass
-class NixChannelInsertChannel(TriggerChannel):
-    model = NixChannel
+class NixpkgsBranchUpdateChannel(TriggerChannel):
+    model = NixpkgsBranch
     lock_notifications = True
 
 

src/shared/listeners/automatic_linkage.py

@@ -25,7 +25,11 @@
 from shared.channels import ContainerChannel
 from shared.models.cve import Container, Cpe
 from shared.models.linkage import CVEDerivationClusterProposal, ProvenanceFlags
-from shared.models.nix_evaluation import NixChannel, NixDerivation, NixEvaluation
+from shared.models.nix_evaluation import (
+    NixChannel,
+    NixDerivation,
+    NixEvaluation,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -34,9 +38,9 @@ def produce_linkage_candidates(
     container: Container,
     filtered_affected: models.QuerySet,
 ) -> models.QuerySet:
-    latest_complete_channels = NixEvaluation.objects.filter(
-        channel__state__in=NixChannel.TRACKED_STATES,
-    ).latest_completed_per_channel()
+    latest_complete_evaluations = NixEvaluation.objects.filter(
+        branch__channels__state__in=NixChannel.TRACKED_STATES,
+    ).latest_completed_per_branch()
 
     package_names = (
         filtered_affected.exclude(package_name__isnull=True)
@@ -87,7 +91,7 @@ def produce_linkage_candidates(
         )
         .filter(
             package_q | product_q,
-            parent_evaluation__in=list(latest_complete_channels),
+            parent_evaluation__in=list(latest_complete_evaluations),
         )
         .select_related("metadata")
         .annotate(**annotations)

src/shared/listeners/nix_channels.py

@@ -2,53 +2,59 @@
 
 import pgpubsub
 
-from shared.channels import NixChannelInsertChannel, NixChannelUpdateChannel
-from shared.models import NixChannel, NixEvaluation
+from shared.channels import NixpkgsBranchInsertChannel, NixpkgsBranchUpdateChannel
+from shared.models import NixEvaluation
+from shared.models.nix_evaluation import NixpkgsBranch
 
 logger = logging.getLogger(__name__)
 
 
-def enqueue_evaluation_job(channel: NixChannel) -> tuple[NixEvaluation, bool]:
+def enqueue_evaluation_job(branch: NixpkgsBranch) -> tuple[NixEvaluation, bool]:
     eval_job, created = NixEvaluation.objects.get_or_create(
         defaults={
             # We will leave the scheduling to the evaluation channel
             # listener.
-            "state": NixEvaluation.EvaluationState.WAITING,
-            "channel": channel,
+            "state": NixEvaluation.EvaluationState.WAITING
         },
-        commit_sha1=channel.head_sha1_commit,
+        branch=branch,
+        commit_sha1=branch.head_sha1_commit,
     )
     # If the commit is shared by multiple channels, prefer the tracking branch.
     if (
         not created
-        and channel.is_tracking_branch
-        and not eval_job.channel.is_tracking_branch
+        and branch.is_tracking_branch
+        and not eval_job.branch.is_tracking_branch
     ):
-        eval_job.channel = channel
-        eval_job.save(update_fields=["channel", "updated_at"])
+        eval_job.branch = branch
+        eval_job.save(update_fields=["branch", "updated_at"])
 
     logger.info(
         f"Enqueued evaluation job {eval_job}{' (already existing!)' if not created else ''}"
     )
     return eval_job, created
 
 
-@pgpubsub.post_insert_listener(NixChannelInsertChannel)
-def start_evaluation_jobs_upon_insertion(old: NixChannel, new: NixChannel) -> None:
-    logger.info("Nix channel created: %s", new.head_sha1_commit)
-    if new.state in NixChannel.TRACKED_STATES:
-        enqueue_evaluation_job(new)
+@pgpubsub.post_insert_listener(NixpkgsBranchInsertChannel)
+def start_evaluation_jobs_upon_insertion(
+    old: NixpkgsBranch, new: NixpkgsBranch
+) -> None:
+    logger.info("Nixpkgs branch created: %s at %s", new.name, new.head_sha1_commit)
+    branch = NixpkgsBranch.objects.get(pk=new.name)
+    if branch.is_tracked:
+        enqueue_evaluation_job(branch)
 
 
 # XXX(@fricklerhandwerk): We can't reuse the same channel for different events
 # https://github.com/PaulGilmartin/django-pgpubsub/issues/86
-@pgpubsub.post_update_listener(NixChannelUpdateChannel)
-def start_evaluation_jobs_upon_updates(old: NixChannel, new: NixChannel) -> None:
+@pgpubsub.post_update_listener(NixpkgsBranchUpdateChannel)
+def start_evaluation_jobs_upon_updates(old: NixpkgsBranch, new: NixpkgsBranch) -> None:
     logger.info(
-        "Nix channel updated: %s -> %s", old.head_sha1_commit, new.head_sha1_commit
+        "Nixpkgs branch updated: %s %s -> %s",
+        new.name,
+        old.head_sha1_commit,
+        new.head_sha1_commit,
     )
-    if (
-        old.head_sha1_commit != new.head_sha1_commit
-        and new.state in NixChannel.TRACKED_STATES
-    ):
-        enqueue_evaluation_job(new)
+    if old.head_sha1_commit != new.head_sha1_commit:
+        branch = NixpkgsBranch.objects.get(pk=new.name)
+        if branch.is_tracked:
+            enqueue_evaluation_job(branch)

src/shared/listeners/nix_evaluation.py

@@ -121,7 +121,7 @@ async def realtime_batch_process_attributes(
         "%d attributes were ingested in %f seconds (%s, %s)",
         len(drvs),
         elapsed,
-        parent_evaluation.channel,
+        parent_evaluation.branch,
         parent_evaluation.commit_sha1[:8],
     )
 
@@ -305,9 +305,7 @@ def _try_acquire_slot() -> bool:
 
 @pgpubsub.post_insert_listener(NixEvaluationChannel)
 def run_evaluation_job(old: NixEvaluation, new: NixEvaluation) -> None:
-    evaluation = NixEvaluation.objects.select_related("channel__release_branch").get(
-        pk=new.pk
-    )
+    evaluation = NixEvaluation.objects.select_related("branch").get(pk=new.pk)
     average_evaluation_time = NixEvaluation.objects.aggregate(
         avg_eval_time=Avg("elapsed")
     )
@@ -316,14 +314,14 @@ def run_evaluation_job(old: NixEvaluation, new: NixEvaluation) -> None:
     if average_evaluation_time is not None:
         logger.info(
             "Nix evaluation requested: %s %s, expecting to finish in %f seconds",
-            evaluation.channel,
+            evaluation.branch,
             new.commit_sha1[:8],
             average_evaluation_time,
         )
     else:
         logger.info(
             "First nix evaluation requested: %s %s, no ETA",
-            evaluation.channel,
+            evaluation.branch,
             new.commit_sha1[:8],
         )
     # FIXME(@raitobezarius): Can we schedule this one instead of waiting on the lock?

src/shared/listeners/package_clustering.py

@@ -15,11 +15,11 @@ def cluster_after_evaluation(old: NixEvaluation, new: NixEvaluation) -> None:
         return
     if new.state != NixEvaluation.EvaluationState.COMPLETED:
         return
-    evaluation = NixEvaluation.objects.select_related("channel").get(pk=new.pk)
+    evaluation = NixEvaluation.objects.select_related("branch").get(pk=new.pk)
     logger.info("Clustering derivations from evaluation %s", evaluation)
     result = cluster_packages(
         NixDerivation.objects.filter(parent_evaluation_id=new.pk),
-        update_packages=evaluation.channel.is_tracking_branch,
+        update_packages=evaluation.branch.is_tracking_branch,
     )
     logger.info(
         f"Done. Clustered {result.derivations_processed} derivations: "

src/shared/management/commands/fetch_all_channels.py

@@ -10,7 +10,7 @@
 
 
 class Command(BaseCommand):
-    help = "Register Nix channels"
+    help = "Fetch current Nixpkgs channel tips and update source branches"
 
     def handle(self, *args: Any, **kwargs: Any) -> str | None:
         client = default_client()

src/shared/management/commands/garbage_collect.py

@@ -220,14 +220,16 @@ def _delete_inactive_channels(self, batch_size: int, dry_run: bool) -> None:
                     NixChannel.ChannelState.DEPRECATED,
                 ]
             )
-            .exclude(evaluations__derivations__cve_links_proposals__isnull=False)
+            .exclude(
+                release_branch__evaluations__derivations__cve_links_proposals__isnull=False
+            )
             .exclude(
                 # No user input must be attached.
                 # Currently only ignored/additional maintainers relate directly to derivations.
-                evaluations__derivations__metadata__maintainers__maintaineroverlay__isnull=False
+                release_branch__evaluations__derivations__metadata__maintainers__maintaineroverlay__isnull=False
             )
-            .exclude(evaluations__derivations__isnull=False)
-            .exclude(evaluations__isnull=False)
+            .exclude(release_branch__evaluations__derivations__isnull=False)
+            .exclude(release_branch__evaluations__isnull=False)
             .distinct()
         )