feat: fetch channel information from Hydra

Since the source of truth for latest evaluations and builds is in Hydra,
from now on we're only using the infrastructure team's `channel.nix`
for information about the release state of channels.

This change also reduces the number of evaluations to one per release branch.
We hard-code tracking the small channel variants, since those advance fastest.

Author: fricklerhandwerk

.gitignore

@@ -14,9 +14,11 @@ result
 *.dump
 *.jsonl
 *.log
+# Generated at build time for development use
 src/static
 src/webview/static/htmx.min.js
 src/webview/static/nixos-logo.svg
+src/shared/_release_channels.py
 # A shallow checkout of nixpkgs
 nixpkgs/
 nixpkgs-gc-roots/

default.nix

@@ -162,6 +162,7 @@ rec {
 
         ln -sf ${sources.htmx}/dist/htmx.js src/webview/static/htmx.min.js
         ln -sf ${sources.nixos-logo} src/webview/static/nixos-logo.svg
+        ln -sf ${package.passthru.release-channels} src/shared/_release_channels.py
 
         mkdir -p $CREDENTIALS_DIRECTORY
         # TODO(@fricklerhandwerk): move all configuration over to pydantic-settings

nix/overlay.nix

@@ -2,6 +2,7 @@ final: prev:
 let
   sources = import ../npins;
   meta = with builtins; fromTOML (readFile ../src/pyproject.toml);
+  release-channels = builtins.toFile "_release_channels.py" "channels = ${builtins.toJSON (import "${sources.infra}/channels.nix").channels}\n";
 in
 {
   /*
@@ -92,7 +93,10 @@ in
       django-vite
     ];
 
-    passthru.PLAYWRIGHT_BROWSERS_PATH = final.playwright-driver.browsers;
+    passthru = {
+      PLAYWRIGHT_BROWSERS_PATH = final.playwright-driver.browsers;
+      inherit release-channels;
+    };
 
     postInstall = ''
       mkdir -p $out/bin
@@ -101,6 +105,7 @@ in
       wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
       cp ${sources.htmx}/dist/htmx.min.js* $out/${final.python3.sitePackages}/webview/static/
       cp ${sources.nixos-logo} $out/${final.python3.sitePackages}/webview/static/nixos-logo.svg
+      cp ${release-channels} $out/${final.python3.sitePackages}/shared/_release_channels.py
     '';
   };
 }

nix/tests/channels.nix

@@ -15,8 +15,24 @@ let
       diskSize = 4096;
     };
   };
-  channels = with builtins; toFile "channels.json" (toJSON (import ./channels.nix));
-  channels-port = toString 8080;
+  hydra = {
+    port = toString 8080;
+    mock = pkgs.writeText "hydra-mock" ''
+      from http.server import BaseHTTPRequestHandler, HTTPServer
+      class H(BaseHTTPRequestHandler):
+          def do_GET(self):
+              self.send_response(200)
+              self.send_header("Content-Type", "application/json")
+              self.end_headers()
+              self.wfile.write(b'${
+                builtins.toJSON {
+                  inputs.nixpkgs.value = "https://github.com/NixOS/nixpkgs.git";
+                }
+              }')
+          log_message = lambda *_: None
+      HTTPServer(("", ${hydra.port}), H).serve_forever()
+    '';
+  };
 in
 pkgs.testers.runNixOSTest {
   name = "default";
@@ -65,7 +81,7 @@ pkgs.testers.runNixOSTest {
         domain = "example.org";
         settings = {
           DEBUG = true;
-          CHANNEL_MONITORING_URL = "http://localhost:${channels-port}/channels.json";
+          HYDRA_URL = "http://localhost:${hydra.port}";
           GIT_CLONE_URL = "file://${dummy-nixpkgs}";
           SYNC_GITHUB_STATE_AT_STARTUP = false;
           GH_ISSUES_PING_MAINTAINERS = true;
@@ -93,18 +109,11 @@ pkgs.testers.runNixOSTest {
             GH_APP_PRIVATE_KEY = dummy-str;
           };
       };
-      systemd.services.mock-channels = {
+      systemd.services.${hydra.mock.name} = {
         wantedBy = [ "multi-user.target" ];
-        before = [ "${application}-server.service" ];
-        path = with pkgs; [
-          python3
-          gnused
-        ];
-        script = ''
-          cd /tmp
-          sed "s/@commit@/$(cat ${dummy-nixpkgs}/REVISION)/g" ${channels} > channels.json
-          python -m http.server ${channels-port}
-        '';
+        before = [ "${application}-fetch-all-channels.service" ];
+        path = [ pkgs.python3 ];
+        script = "python ${hydra.mock}";
       };
       systemd.services.setup-git-repo = {
         wantedBy = [ "multi-user.target" ];
@@ -130,20 +139,22 @@ pkgs.testers.runNixOSTest {
     ''
       server.wait_for_unit("${application}-server.service")
       server.wait_for_unit("${application}-worker.service")
-      server.wait_for_unit("mock-channels.service")
+      server.wait_for_unit("${hydra.mock.name}.service")
 
       with subtest("Check that no migrations were missed"):
         server.succeed("wst-manage makemigrations --check --dry-run")
 
-      with subtest("Check that channel are fetched and evaluations enqueued"):
+      with subtest("Check that channels are fetched and evaluations enqueued"):
         server.succeed("wst-manage fetch_all_channels")
         ${in-shell "succeed" ''
           from shared.models import NixChannel
-          assert NixChannel.objects.count() == 4
+          from shared.models.nix_evaluation import NixpkgsBranch
+          assert NixpkgsBranch.objects.count() == 1, f"expected 1 branch, got {NixpkgsBranch.objects.count()}"
+          assert NixChannel.objects.count() >= 1, f"expected at least 1 channel, got {NixChannel.objects.count()}"
         ''}
-        ${in-shell "succeed " ''
+        ${in-shell "succeed" ''
           from shared.models import NixEvaluation
-          assert NixEvaluation.objects.count() == 1
+          assert NixEvaluation.objects.count() == 1, f"expected 1 evaluation, got {NixEvaluation.objects.count()}"
         ''}
 
       with subtest("Application tests"):
@@ -214,11 +225,11 @@ pkgs.testers.runNixOSTest {
               # Maintainers should only be attached to derivations from the tracking branch.
               from django.conf import settings
               tracking_meta = NixDerivationMeta.objects.get(
-                derivation__parent_evaluation__channel__channel_branch=settings.TRACKING_BRANCH,
+                derivation__parent_evaluation__channel__release_branch__name=settings.TRACKING_BRANCH,
               )
               assert tracking_meta.maintainers.exists(), f"{settings.TRACKING_BRANCH} meta has no maintainers"
               for m in NixDerivationMeta.objects.exclude(
-                derivation__parent_evaluation__channel__channel_branch=settings.TRACKING_BRANCH,
+                derivation__parent_evaluation__channel__release_branch__name=settings.TRACKING_BRANCH,
               ):
                 assert not m.maintainers.exists(), f"{m.derivation.parent_evaluation.channel.channel_branch}) has unexpected maintainers"
             ''

nix/tests/default.nix

@@ -100,13 +100,17 @@ class DjangoSettings(BaseModel):
             By default, in the root of this Git repository.
             """
         )
-        CHANNEL_MONITORING_URL: HttpUrl = Field(
+        HYDRA_URL: HttpUrl = Field(
             description="""
-            URL from which to fetch the current channel structure.
+            Base URL of the Hydra instance used to look up jobset inputs for branch resolution.
             """,
-            default=HttpUrl(
-                "https://monitoring.nixos.org/prometheus/api/v1/query?query=channel_revision"
-            ),
+            default=HttpUrl("https://hydra.nixos.org"),
+        )
+        NETWORK_REQUEST_TIMEOUT: int = Field(
+            description="""
+            Timeout in seconds for outbound network requests.
+            """,
+            default=60,
         )
         SYNC_GITHUB_STATE_AT_STARTUP: bool = Field(
             description="""
@@ -169,7 +173,7 @@ class DjangoSettings(BaseModel):
             The branch that tracks upstream development.
             Serves as the source of truth for package metadata such as maintainers and descriptions.
             """,
-            default="nixpkgs-unstable",
+            default="master",
         )
         MAX_MATCHES: int = Field(
             description="""

src/project/settings.py

@@ -4,17 +4,33 @@
 import os.path
 import pathlib
 import random
+import subprocess
 import time
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from typing import IO, Any
 
 from django.conf import settings
+from pydantic import AnyUrl
 
 logger = logging.getLogger(__name__)
 
 
+def get_head_sha1(url: AnyUrl, branch: str) -> str:
+    result = subprocess.run(
+        ["git", "ls-remote", str(url), f"refs/heads/{branch}"],
+        capture_output=True,
+        text=True,
+        check=True,
+        timeout=settings.NETWORK_REQUEST_TIMEOUT,
+    )
+    line = result.stdout.strip()
+    if not line:
+        raise ValueError(f"branch {branch!r} not found at {url!r}")
+    return line.split()[0]
+
+
 @dataclass
 class Worktree:
     path: pathlib.Path

src/shared/git.py

@@ -0,0 +1,59 @@
+import requests
+from django.conf import settings
+from pydantic import BaseModel, ConfigDict
+
+
+class JobsetInput(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    value: str
+
+
+class Jobset(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    inputs: dict[str, JobsetInput]
+
+    def input_branch(self, input_name: str, default: str) -> str:
+        """
+        Get the branch tracked by the named jobset input.
+        Use `default` when the input value specifies no explicit branch.
+
+        Raises `KeyError` when the input is absent.
+        """
+        parts = self.inputs[input_name].value.split()
+        return parts[1] if len(parts) > 1 else default
+
+
+def jobset_from_job(job: str) -> str:
+    """
+    Extract jobset path from a full job path.
+
+    Example: `"nixos/release-26.05/tested"` → `"nixos/release-26.05"`.
+    """
+    return "/".join(job.split("/")[:-1])
+
+
+class HydraClient:
+    """
+    Minimal client for the Hydra API.
+    """
+
+    def __init__(self, base_url: str) -> None:
+        self.base_url = base_url.rstrip("/")
+
+    def get_jobset(self, jobset_path: str) -> Jobset:
+        """
+        Fetches jobset metadata for the given `<project>/<jobset>` path.
+        """
+        resp = requests.get(
+            f"{self.base_url}/jobset/{jobset_path}",
+            headers={"Accept": "application/json"},
+            timeout=settings.NETWORK_REQUEST_TIMEOUT,
+        )
+        resp.raise_for_status()
+        return Jobset.model_validate(resp.json())
+
+
+def default_client() -> HydraClient:
+    return HydraClient(str(settings.HYDRA_URL))

src/shared/hydra.py

@@ -305,7 +305,9 @@ def _try_acquire_slot() -> bool:
 
 @pgpubsub.post_insert_listener(NixEvaluationChannel)
 def run_evaluation_job(old: NixEvaluation, new: NixEvaluation) -> None:
-    evaluation = NixEvaluation.objects.select_related("channel").get(pk=new.pk)
+    evaluation = NixEvaluation.objects.select_related("channel__release_branch").get(
+        pk=new.pk
+    )
     average_evaluation_time = NixEvaluation.objects.aggregate(
         avg_eval_time=Avg("elapsed")
     )