feat: fetch channel information from Hydra

.gitignore

@@ -14,9 +14,11 @@ result
 *.dump
 *.jsonl
 *.log
+# Generated at build time for development use
 src/static
 src/webview/static/htmx.min.js
 src/webview/static/nixos-logo.svg
+src/shared/_release_channels.py
 # A shallow checkout of nixpkgs
 nixpkgs/
 nixpkgs-gc-roots/

default.nix

@@ -162,6 +162,7 @@ rec {
 
         ln -sf ${sources.htmx}/dist/htmx.js src/webview/static/htmx.min.js
         ln -sf ${sources.nixos-logo} src/webview/static/nixos-logo.svg
+        ln -sf ${package.passthru.release-channels} src/shared/_release_channels.py
 
         mkdir -p $CREDENTIALS_DIRECTORY
         # TODO(@fricklerhandwerk): move all configuration over to pydantic-settings

docs/README.md

@@ -30,7 +30,7 @@ The tracker needs to communicate with third party services, namely:
 - GitHub repositories:
   - https://github.com/nixos/nixpkgs to pull the latest changes from Nixpkgs
   - https://github.com/CVEProject/cvelistV5 to pull CVE data
-- https://monitoring.nixos.org/prometheus/api/v1/query?query=channel_revision to get information about the latest channels
+- https://hydra.nixos.org to get information about the latest builds
 
 ## Storage space considerations
 

docs/architecture.mermaid

@@ -5,7 +5,7 @@ graph TB
         GitHub["GitHub API"]
         GitHubNixos["GitHub Repository<br/>nixos/nixpkgs"]
         GitHubCVEs["GitHub Repository<br/>CVEProject/cvelistV5"]
-        NixMonitoring["monitoring.nixos.org<br/>channel revision"]
+        Hydra["hydra.nixos.org<br/>channel revision"]
     end
 
     subgraph SecurityTracker ["Security Tracker Host"]
@@ -40,7 +40,7 @@ graph TB
 
     %% Timers
     SystemdTimerChannels -.->|Triggers Daily| FetchAllChannels
-    FetchAllChannels -->|1 Fetch Channels| NixMonitoring
+    FetchAllChannels -->|1 Fetch Channels| Hydra
     FetchAllChannels -->|2 Git pull| GitHubNixos
     FetchAllChannels -->|3 Update Repo| LocalGitCheckout
     FetchAllChannels -->|4 update channels| PostgreSQL
@@ -61,7 +61,7 @@ graph TB
     classDef subgraphClass fill:#fafafa,stroke:#424242,stroke-width:3px
 
     class Users userClass
-    class GitHub,GitHubNixos,GitHubCVEs,NixMonitoring externalClass
+    class GitHub,GitHubNixos,GitHubCVEs,Hydra externalClass
     class Nginx,DaphneAsgi webClass
     class FetchAllChannels,IngestCVEs commandClass
     class SystemdTimerChannels,SystemdTimerCVEs,NixEval,MatchingListener backgroundClass

nix/overlay.nix

@@ -2,6 +2,7 @@ final: prev:
 let
   sources = import ../npins;
   meta = with builtins; fromTOML (readFile ../src/pyproject.toml);
+  release-channels = builtins.toFile "_release_channels.py" "channels = ${builtins.toJSON (import "${sources.infra}/channels.nix").channels}\n";
 in
 {
   /*
@@ -88,7 +89,10 @@ in
       django-vite
     ];
 
-    passthru.PLAYWRIGHT_BROWSERS_PATH = final.playwright-driver.browsers;
+    passthru = {
+      PLAYWRIGHT_BROWSERS_PATH = final.playwright-driver.browsers;
+      inherit release-channels;
+    };
 
     postInstall = ''
       mkdir -p $out/bin
@@ -97,6 +101,7 @@ in
       wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH"
       cp ${sources.htmx}/dist/htmx.min.js* $out/${final.python3.sitePackages}/webview/static/
       cp ${sources.nixos-logo} $out/${final.python3.sitePackages}/webview/static/nixos-logo.svg
+      cp ${release-channels} $out/${final.python3.sitePackages}/shared/_release_channels.py
     '';
   };
 }

nix/tests/default.nix

@@ -15,8 +15,59 @@ let
       diskSize = 4096;
     };
   };
-  channels = with builtins; toFile "channels.json" (toJSON (import ./channels.nix));
-  channels-port = toString 8080;
+  dummy-nixpkgs =
+    pkgs.runCommand "dummy-nixpkgs"
+      {
+        nativeBuildInputs = [ pkgs.git ];
+      }
+      ''
+        mkdir -p $out/pkgs/top-level
+
+        cat > $out/pkgs/top-level/release.nix << EOF
+        { ... }:
+        {
+          hello.x86_64-linux = (import ${pkgs.path} {}).hello;
+        }
+        EOF
+
+        cd $out
+        git init --initial-branch=master
+        git add -A
+        git -c user.name=test -c user.email=test@test commit -m "test"
+        git rev-parse HEAD > REVISION
+      '';
+  hydra = {
+    port = toString 8080;
+    mock = pkgs.writeText "hydra-mock" ''
+      import json
+      from http.server import BaseHTTPRequestHandler, HTTPServer
+
+      nixpkgs_url = "file://${dummy-nixpkgs}"
+      with open("${dummy-nixpkgs}/REVISION") as f:
+          revision = f.read().strip()
+
+      responses = {
+          "jobset": {"inputs": {"nixpkgs": {"type": "git", "value": nixpkgs_url}}},
+          "job": {"id": 1, "jobsetevals": [1]},
+          "eval": {"id": 1, "jobsetevalinputs": {"nixpkgs": {"type": "git", "uri": nixpkgs_url, "revision": revision}}},
+      }
+
+      class Handler(BaseHTTPRequestHandler):
+          def do_GET(self):
+              body = responses.get(self.path.split("/")[1])
+              if body is None:
+                  self.send_response(404)
+                  self.end_headers()
+                  return
+              self.send_response(200)
+              self.send_header("Content-Type", "application/json")
+              self.end_headers()
+              self.wfile.write(json.dumps(body).encode())
+          log_message = lambda *_: None
+
+      HTTPServer(("", ${hydra.port}), Handler).serve_forever()
+    '';
+  };
 in
 pkgs.testers.runNixOSTest {
   name = "default";
@@ -25,27 +76,6 @@ pkgs.testers.runNixOSTest {
     { config, ... }:
     let
       cfg = config.services.${application};
-      dummy-nixpkgs =
-        pkgs.runCommand "dummy-nixpkgs"
-          {
-            nativeBuildInputs = [ pkgs.git ];
-          }
-          ''
-            mkdir -p $out/pkgs/top-level
-
-            cat > $out/pkgs/top-level/release.nix << EOF
-            { ... }:
-            {
-              hello.x86_64-linux = (import ${pkgs.path} {}).hello;
-            }
-            EOF
-
-            cd $out
-            git init --initial-branch=master
-            git add -A
-            git -c user.name=test -c user.email=test@test commit -m "test"
-            git rev-parse HEAD > REVISION
-          '';
     in
     {
       imports = [ module ];
@@ -65,7 +95,7 @@ pkgs.testers.runNixOSTest {
         domain = "example.org";
         settings = {
           DEBUG = true;
-          CHANNEL_MONITORING_URL = "http://localhost:${channels-port}/channels.json";
+          HYDRA_URL = "http://localhost:${hydra.port}";
           GIT_CLONE_URL = "file://${dummy-nixpkgs}";
           SYNC_GITHUB_STATE_AT_STARTUP = false;
           GH_ISSUES_PING_MAINTAINERS = true;
@@ -93,18 +123,11 @@ pkgs.testers.runNixOSTest {
             GH_APP_PRIVATE_KEY = dummy-str;
           };
       };
-      systemd.services.mock-channels = {
+      systemd.services.${hydra.mock.name} = {
         wantedBy = [ "multi-user.target" ];
-        before = [ "${application}-server.service" ];
-        path = with pkgs; [
-          python3
-          gnused
-        ];
-        script = ''
-          cd /tmp
-          sed "s/@commit@/$(cat ${dummy-nixpkgs}/REVISION)/g" ${channels} > channels.json
-          python -m http.server ${channels-port}
-        '';
+        before = [ "${application}-fetch-all-channels.service" ];
+        path = [ pkgs.python3 ];
+        script = "python ${hydra.mock}";
       };
       systemd.services.setup-git-repo = {
         wantedBy = [ "multi-user.target" ];
@@ -130,24 +153,33 @@ pkgs.testers.runNixOSTest {
     ''
       server.wait_for_unit("${application}-server.service")
       server.wait_for_unit("${application}-worker.service")
-      server.wait_for_unit("mock-channels.service")
+      server.wait_for_unit("${hydra.mock.name}.service")
 
       with subtest("Check that no migrations were missed"):
         server.succeed("wst-manage makemigrations --check --dry-run")
 
       with subtest("Check that channels are fetched and only small ones get enqueued for evaluation"):
         server.succeed("wst-manage fetch_all_channels")
         ${in-shell "succeed" ''
-          from shared.models import NixChannel
-          assert NixChannel.objects.count() == 6
+          import pathlib
+          from shared.models import NixChannel, NixpkgsBranch
+
+          assert NixpkgsBranch.objects.count() == 1, f"expected 1 branch, got {NixpkgsBranch.objects.count()}"
+          assert NixChannel.objects.count() >= 1, f"expected at least 1 channel, got {NixChannel.objects.count()}"
+
+          revision = pathlib.Path("${dummy-nixpkgs}/REVISION").read_text().strip()
+          branch = NixpkgsBranch.objects.get()
+          assert branch.head_sha1_commit == revision, f"expected {revision}, got {branch.head_sha1_commit}"
+          channel_tips = NixChannel.objects.values_list("head_sha1_commit", flat=True)
+          assert all(hash == revision for hash in channel_tips), f"unexpected channel tips: {[hash for hash in channel_tips if hash != revision]}"
         ''}
         ${
           # Give it some time to queue up the evaluations...
           ""
         }
         ${in-shell "succeed" ''
-          from shared.models import NixEvaluation
-          assert NixEvaluation.objects.count() == 1
+          from shared.models import NixChannel, NixEvaluation
+          assert NixEvaluation.objects.count() == 1, f"expected 1 evaluation, got {NixEvaluation.objects.count()}"
           assert NixEvaluation.objects.get().channel.variant == NixChannel.Variant.SMALL
         ''}
 
@@ -219,11 +251,11 @@ pkgs.testers.runNixOSTest {
               # Maintainers should only be attached to derivations from the tracking branch.
               from django.conf import settings
               tracking_meta = NixDerivationMeta.objects.get(
-                derivation__parent_evaluation__channel__channel_branch=settings.TRACKING_BRANCH,
+                derivation__parent_evaluation__channel__release_branch__name=settings.TRACKING_BRANCH,
               )
               assert tracking_meta.maintainers.exists(), f"{settings.TRACKING_BRANCH} meta has no maintainers"
               for m in NixDerivationMeta.objects.exclude(
-                derivation__parent_evaluation__channel__channel_branch=settings.TRACKING_BRANCH,
+                derivation__parent_evaluation__channel__release_branch__name=settings.TRACKING_BRANCH,
               ):
                 assert not m.maintainers.exists(), f"{m.derivation.parent_evaluation.channel.channel_branch}) has unexpected maintainers"
             ''

src/project/settings.py

@@ -92,21 +92,31 @@ class DjangoSettings(BaseModel):
             description="""
             URL from which to clone the Nix expressions encoding the software distribution.
             """,
-            default=HttpUrl("https://github.com/NixOS/nixpkgs"),
+            default=AnyUrl("https://github.com/NixOS/nixpkgs.git"),
         )
         LOCAL_NIXPKGS_CHECKOUT: DirectoryPath = Field(
             description="""
             This is the path where a local checkout of Nixpkgs will be instantiated for this application's needsr
             By default, in the root of this Git repository.
             """
         )
-        CHANNEL_MONITORING_URL: HttpUrl = Field(
+        HYDRA_URL: HttpUrl = Field(
             description="""
-            URL from which to fetch the current channel structure.
+            Base URL of the Hydra instance used to look up jobset inputs for branch resolution.
             """,
-            default=HttpUrl(
-                "https://monitoring.nixos.org/prometheus/api/v1/query?query=channel_revision"
-            ),
+            default=HttpUrl("https://hydra.nixos.org"),
+        )
+        HYDRA_INPUT_NAME: str = Field(
+            description="""
+            Name of the Hydra jobset input that refers to the source we're tracking.
+            """,
+            default="nixpkgs",
+        )
+        NETWORK_REQUEST_TIMEOUT: int = Field(
+            description="""
+            Timeout in seconds for outbound network requests.
+            """,
+            default=60,
         )
         SYNC_GITHUB_STATE_AT_STARTUP: bool = Field(
             description="""
@@ -169,7 +179,7 @@ class DjangoSettings(BaseModel):
             The branch that tracks upstream development.
             Serves as the source of truth for package metadata such as maintainers and descriptions.
             """,
-            default="nixos-unstable-small",
+            default="master",
         )
         MAX_MATCHES: int = Field(
             description="""