diff --git a/.gitignore b/.gitignore index db71b4c7..4cc47715 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,11 @@ result *.dump *.jsonl *.log +# Generated at build time for development use src/static src/webview/static/htmx.min.js src/webview/static/nixos-logo.svg +src/shared/_release_channels.py # A shallow checkout of nixpkgs nixpkgs/ nixpkgs-gc-roots/ diff --git a/default.nix b/default.nix index c199a7c3..ba06a597 100644 --- a/default.nix +++ b/default.nix @@ -162,6 +162,7 @@ rec { ln -sf ${sources.htmx}/dist/htmx.js src/webview/static/htmx.min.js ln -sf ${sources.nixos-logo} src/webview/static/nixos-logo.svg + ln -sf ${package.passthru.release-channels} src/shared/_release_channels.py mkdir -p $CREDENTIALS_DIRECTORY # TODO(@fricklerhandwerk): move all configuration over to pydantic-settings diff --git a/docs/README.md b/docs/README.md index f98e0d9f..1e4d33d1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,7 +30,7 @@ The tracker needs to communicate with third party services, namely: - GitHub repositories: - https://github.com/nixos/nixpkgs to pull the latest changes from Nixpkgs - https://github.com/CVEProject/cvelistV5 to pull CVE data -- https://monitoring.nixos.org/prometheus/api/v1/query?query=channel_revision to get information about the latest channels +- https://hydra.nixos.org to get information about the latest builds ## Storage space considerations diff --git a/docs/architecture.mermaid b/docs/architecture.mermaid index 53fb97ba..15456b8a 100644 --- a/docs/architecture.mermaid +++ b/docs/architecture.mermaid @@ -5,7 +5,7 @@ graph TB GitHub["GitHub API"] GitHubNixos["GitHub Repository
nixos/nixpkgs"] GitHubCVEs["GitHub Repository
CVEProject/cvelistV5"] - NixMonitoring["monitoring.nixos.org
channel revision"] + Hydra["hydra.nixos.org
channel revision"] end subgraph SecurityTracker ["Security Tracker Host"] @@ -40,7 +40,7 @@ graph TB %% Timers SystemdTimerChannels -.->|Triggers Daily| FetchAllChannels - FetchAllChannels -->|1 Fetch Channels| NixMonitoring + FetchAllChannels -->|1 Fetch Channels| Hydra FetchAllChannels -->|2 Git pull| GitHubNixos FetchAllChannels -->|3 Update Repo| LocalGitCheckout FetchAllChannels -->|4 update channels| PostgreSQL @@ -61,7 +61,7 @@ graph TB classDef subgraphClass fill:#fafafa,stroke:#424242,stroke-width:3px class Users userClass - class GitHub,GitHubNixos,GitHubCVEs,NixMonitoring externalClass + class GitHub,GitHubNixos,GitHubCVEs,Hydra externalClass class Nginx,DaphneAsgi webClass class FetchAllChannels,IngestCVEs commandClass class SystemdTimerChannels,SystemdTimerCVEs,NixEval,MatchingListener backgroundClass diff --git a/nix/overlay.nix b/nix/overlay.nix index ae6afa35..e887c828 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -2,6 +2,7 @@ final: prev: let sources = import ../npins; meta = with builtins; fromTOML (readFile ../src/pyproject.toml); + release-channels = builtins.toFile "_release_channels.py" "channels = ${builtins.toJSON (import "${sources.infra}/channels.nix").channels}\n"; in { /* @@ -88,7 +89,10 @@ in django-vite ]; - passthru.PLAYWRIGHT_BROWSERS_PATH = final.playwright-driver.browsers; + passthru = { + PLAYWRIGHT_BROWSERS_PATH = final.playwright-driver.browsers; + inherit release-channels; + }; postInstall = '' mkdir -p $out/bin @@ -97,6 +101,7 @@ in wrapProgram $out/bin/manage.py --prefix PYTHONPATH : "$PYTHONPATH" cp ${sources.htmx}/dist/htmx.min.js* $out/${final.python3.sitePackages}/webview/static/ cp ${sources.nixos-logo} $out/${final.python3.sitePackages}/webview/static/nixos-logo.svg + cp ${release-channels} $out/${final.python3.sitePackages}/shared/_release_channels.py ''; }; } diff --git a/nix/tests/channels.nix b/nix/tests/channels.nix deleted file mode 100644 index ee1a010a..00000000 --- a/nix/tests/channels.nix +++ /dev/null @@ -1,64 +0,0 @@ -/** - Example of a subset of the structure as it comes out of https://prometheus.nixos.org/api/v1/query?query=channel_revision -*/ -{ - status = "success"; - data = { - resultType = "vector"; - result = [ - { - metric = { - __name__ = "channel_revision"; - channel = "nixos-unstable"; - revision = "@commit@"; - status = "rolling"; - variant = "primary"; - }; - } - { - metric = { - __name__ = "channel_revision"; - channel = "nixos-unstable-small"; - revision = "@commit@"; - status = "rolling"; - variant = "small"; - }; - } - { - metric = { - __name__ = "channel_revision"; - channel = "nixpkgs-unstable"; - revision = "@commit@"; - status = "rolling"; - }; - } - { - metric = { - __name__ = "channel_revision"; - channel = "nixos-25.11"; - revision = "@commit@"; - status = "stable"; - variant = "primary"; - }; - } - { - metric = { - __name__ = "channel_revision"; - channel = "nixos-25.11-small"; - revision = "@commit@"; - status = "stable"; - variant = "small"; - }; - } - { - metric = { - __name__ = "channel_revision"; - channel = "nixos-25.05-small"; - revision = "@commit@"; - status = "unmaintained"; - variant = "small"; - }; - } - ]; - }; -} diff --git a/nix/tests/default.nix b/nix/tests/default.nix index 616f91b2..147f32e0 100644 --- a/nix/tests/default.nix +++ b/nix/tests/default.nix @@ -15,8 +15,59 @@ let diskSize = 4096; }; }; - channels = with builtins; toFile "channels.json" (toJSON (import ./channels.nix)); - channels-port = toString 8080; + dummy-nixpkgs = + pkgs.runCommand "dummy-nixpkgs" + { + nativeBuildInputs = [ pkgs.git ]; + } + '' + mkdir -p $out/pkgs/top-level + + cat > $out/pkgs/top-level/release.nix << EOF + { ... }: + { + hello.x86_64-linux = (import ${pkgs.path} {}).hello; + } + EOF + + cd $out + git init --initial-branch=master + git add -A + git -c user.name=test -c user.email=test@test commit -m "test" + git rev-parse HEAD > REVISION + ''; + hydra = { + port = toString 8080; + mock = pkgs.writeText "hydra-mock" '' + import json + from http.server import BaseHTTPRequestHandler, HTTPServer + + nixpkgs_url = "file://${dummy-nixpkgs}" + with open("${dummy-nixpkgs}/REVISION") as f: + revision = f.read().strip() + + responses = { + "jobset": {"inputs": {"nixpkgs": {"type": "git", "value": nixpkgs_url}}}, + "job": {"id": 1, "jobsetevals": [1]}, + "eval": {"id": 1, "jobsetevalinputs": {"nixpkgs": {"type": "git", "uri": nixpkgs_url, "revision": revision}}}, + } + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = responses.get(self.path.split("/")[1]) + if body is None: + self.send_response(404) + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(body).encode()) + log_message = lambda *_: None + + HTTPServer(("", ${hydra.port}), Handler).serve_forever() + ''; + }; in pkgs.testers.runNixOSTest { name = "default"; @@ -25,27 +76,6 @@ pkgs.testers.runNixOSTest { { config, ... }: let cfg = config.services.${application}; - dummy-nixpkgs = - pkgs.runCommand "dummy-nixpkgs" - { - nativeBuildInputs = [ pkgs.git ]; - } - '' - mkdir -p $out/pkgs/top-level - - cat > $out/pkgs/top-level/release.nix << EOF - { ... }: - { - hello.x86_64-linux = (import ${pkgs.path} {}).hello; - } - EOF - - cd $out - git init --initial-branch=master - git add -A - git -c user.name=test -c user.email=test@test commit -m "test" - git rev-parse HEAD > REVISION - ''; in { imports = [ module ]; @@ -65,7 +95,7 @@ pkgs.testers.runNixOSTest { domain = "example.org"; settings = { DEBUG = true; - CHANNEL_MONITORING_URL = "http://localhost:${channels-port}/channels.json"; + HYDRA_URL = "http://localhost:${hydra.port}"; GIT_CLONE_URL = "file://${dummy-nixpkgs}"; SYNC_GITHUB_STATE_AT_STARTUP = false; GH_ISSUES_PING_MAINTAINERS = true; @@ -93,18 +123,11 @@ pkgs.testers.runNixOSTest { GH_APP_PRIVATE_KEY = dummy-str; }; }; - systemd.services.mock-channels = { + systemd.services.${hydra.mock.name} = { wantedBy = [ "multi-user.target" ]; - before = [ "${application}-server.service" ]; - path = with pkgs; [ - python3 - gnused - ]; - script = '' - cd /tmp - sed "s/@commit@/$(cat ${dummy-nixpkgs}/REVISION)/g" ${channels} > channels.json - python -m http.server ${channels-port} - ''; + before = [ "${application}-fetch-all-channels.service" ]; + path = [ pkgs.python3 ]; + script = "python ${hydra.mock}"; }; systemd.services.setup-git-repo = { wantedBy = [ "multi-user.target" ]; @@ -130,7 +153,7 @@ pkgs.testers.runNixOSTest { '' server.wait_for_unit("${application}-server.service") server.wait_for_unit("${application}-worker.service") - server.wait_for_unit("mock-channels.service") + server.wait_for_unit("${hydra.mock.name}.service") with subtest("Check that no migrations were missed"): server.succeed("wst-manage makemigrations --check --dry-run") @@ -138,16 +161,25 @@ pkgs.testers.runNixOSTest { with subtest("Check that channels are fetched and only small ones get enqueued for evaluation"): server.succeed("wst-manage fetch_all_channels") ${in-shell "succeed" '' - from shared.models import NixChannel - assert NixChannel.objects.count() == 6 + import pathlib + from shared.models import NixChannel, NixpkgsBranch + + assert NixpkgsBranch.objects.count() == 1, f"expected 1 branch, got {NixpkgsBranch.objects.count()}" + assert NixChannel.objects.count() >= 1, f"expected at least 1 channel, got {NixChannel.objects.count()}" + + revision = pathlib.Path("${dummy-nixpkgs}/REVISION").read_text().strip() + branch = NixpkgsBranch.objects.get() + assert branch.head_sha1_commit == revision, f"expected {revision}, got {branch.head_sha1_commit}" + channel_tips = NixChannel.objects.values_list("head_sha1_commit", flat=True) + assert all(hash == revision for hash in channel_tips), f"unexpected channel tips: {[hash for hash in channel_tips if hash != revision]}" ''} ${ # Give it some time to queue up the evaluations... "" } ${in-shell "succeed" '' - from shared.models import NixEvaluation - assert NixEvaluation.objects.count() == 1 + from shared.models import NixChannel, NixEvaluation + assert NixEvaluation.objects.count() == 1, f"expected 1 evaluation, got {NixEvaluation.objects.count()}" assert NixEvaluation.objects.get().channel.variant == NixChannel.Variant.SMALL ''} @@ -219,11 +251,11 @@ pkgs.testers.runNixOSTest { # Maintainers should only be attached to derivations from the tracking branch. from django.conf import settings tracking_meta = NixDerivationMeta.objects.get( - derivation__parent_evaluation__channel__channel_branch=settings.TRACKING_BRANCH, + derivation__parent_evaluation__channel__release_branch__name=settings.TRACKING_BRANCH, ) assert tracking_meta.maintainers.exists(), f"{settings.TRACKING_BRANCH} meta has no maintainers" for m in NixDerivationMeta.objects.exclude( - derivation__parent_evaluation__channel__channel_branch=settings.TRACKING_BRANCH, + derivation__parent_evaluation__channel__release_branch__name=settings.TRACKING_BRANCH, ): assert not m.maintainers.exists(), f"{m.derivation.parent_evaluation.channel.channel_branch}) has unexpected maintainers" '' diff --git a/src/project/settings.py b/src/project/settings.py index 2cec883a..55fd73c5 100644 --- a/src/project/settings.py +++ b/src/project/settings.py @@ -92,7 +92,7 @@ class DjangoSettings(BaseModel): description=""" URL from which to clone the Nix expressions encoding the software distribution. """, - default=HttpUrl("https://github.com/NixOS/nixpkgs"), + default=AnyUrl("https://github.com/NixOS/nixpkgs.git"), ) LOCAL_NIXPKGS_CHECKOUT: DirectoryPath = Field( description=""" @@ -100,13 +100,23 @@ class DjangoSettings(BaseModel): By default, in the root of this Git repository. """ ) - CHANNEL_MONITORING_URL: HttpUrl = Field( + HYDRA_URL: HttpUrl = Field( description=""" - URL from which to fetch the current channel structure. + Base URL of the Hydra instance used to look up jobset inputs for branch resolution. """, - default=HttpUrl( - "https://monitoring.nixos.org/prometheus/api/v1/query?query=channel_revision" - ), + default=HttpUrl("https://hydra.nixos.org"), + ) + HYDRA_INPUT_NAME: str = Field( + description=""" + Name of the Hydra jobset input that refers to the source we're tracking. + """, + default="nixpkgs", + ) + NETWORK_REQUEST_TIMEOUT: int = Field( + description=""" + Timeout in seconds for outbound network requests. + """, + default=60, ) SYNC_GITHUB_STATE_AT_STARTUP: bool = Field( description=""" @@ -169,7 +179,7 @@ class DjangoSettings(BaseModel): The branch that tracks upstream development. Serves as the source of truth for package metadata such as maintainers and descriptions. """, - default="nixos-unstable-small", + default="master", ) MAX_MATCHES: int = Field( description=""" diff --git a/src/shared/git.py b/src/shared/git.py index 5121bd70..b9e6866f 100644 --- a/src/shared/git.py +++ b/src/shared/git.py @@ -4,6 +4,7 @@ import os.path import pathlib import random +import subprocess import time from collections.abc import AsyncGenerator from contextlib import asynccontextmanager @@ -11,10 +12,25 @@ from typing import IO, Any from django.conf import settings +from pydantic import AnyUrl logger = logging.getLogger(__name__) +def get_head_sha1(url: AnyUrl, branch: str) -> str: + result = subprocess.run( + ["git", "ls-remote", str(url), f"refs/heads/{branch}"], + capture_output=True, + text=True, + check=True, + timeout=settings.NETWORK_REQUEST_TIMEOUT, + ) + line = result.stdout.strip() + if not line: + raise ValueError(f"branch {branch!r} not found at {url!r}") + return line.split()[0] + + @dataclass class Worktree: path: pathlib.Path diff --git a/src/shared/hydra.py b/src/shared/hydra.py new file mode 100644 index 00000000..d8c690f0 --- /dev/null +++ b/src/shared/hydra.py @@ -0,0 +1,139 @@ +from typing import Annotated + +import requests +from django.conf import settings +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + model_validator, +) + +r""" +The following regular expressions were obtained like this: + +nix develop github:NixOS/hydra --command perl -I subprojects/hydra/lib -MJSON -MHydra::Helper::CatalystUtils -e ' + my @names = qw(projectNameRE jobsetNameRE jobNameRE inputNameRE); + no strict "refs"; + my %re = map { $_ => ${"Hydra::Helper::CatalystUtils::$_"} } @names; + print encode_json(\%re); +' +""" +# FIXME(@fricklerhandwerk): Evaluate the Perl source at build time to get these without hard-coding. +project_name = r"(?:[A-Za-z_][A-Za-z0-9-_]*)" +jobset_name = r"(?:[A-Za-z_][A-Za-z0-9-_.]*)" +job_name = r"(?:(?:[A-Za-z_][A-Za-z0-9-_]*)(?:\.(?:[A-Za-z_][A-Za-z0-9-_]*))*)" +input_name = r"(?:[A-Za-z_][A-Za-z0-9-_]*)" + +ProjectName = Annotated[str, Field(pattern=project_name)] +JobsetName = Annotated[str, Field(pattern=jobset_name)] +JobName = Annotated[str, Field(pattern=job_name)] +InputName = Annotated[str, Field(pattern=input_name)] + +GitHash = Annotated[str, Field(pattern="[0-9a-f]{40}")] + + +class JobsetInput(BaseModel): + model_config = ConfigDict(extra="ignore") + + url: AnyUrl + branch: JobsetName | None + + @model_validator(mode="before") + @classmethod + def parse_string(cls, data: dict) -> dict: + parts = data["value"].split(None, 1) + return {"url": parts[0], "branch": parts[1] if len(parts) > 1 else None} + + def get_branch(self, default: str) -> str: + return self.branch if self.branch else default + + +class Jobset(BaseModel): + model_config = ConfigDict(extra="ignore") + + inputs: dict[str, JobsetInput] + + @model_validator(mode="before") + @classmethod + def filter_git(cls, data: dict) -> dict: + return { + **data, + "inputs": { + k: v for k, v in data["inputs"].items() if v.get("type") == "git" + }, + } + + +class EvalInput(BaseModel): + uri: AnyUrl + revision: GitHash + + +class Evaluation(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: int + jobsetevalinputs: dict[InputName, EvalInput] + + @model_validator(mode="before") + @classmethod + def filter_git(cls, data: dict) -> dict: + return { + **data, + "jobsetevalinputs": { + k: v + for k, v in data["jobsetevalinputs"].items() + if v.get("type") == "git" + }, + } + + +class Build(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: int + jobsetevals: Annotated[list[int], Field(min_length=1)] + + +class HydraClient: + """ + Minimal client for the Hydra API. + """ + + def __init__(self, base_url: str) -> None: + self.base_url = base_url.rstrip("/") + + def get_jobset(self, project: ProjectName, jobset: JobsetName) -> Jobset: + resp = requests.get( + f"{self.base_url}/jobset/{project}/{jobset}", + headers={"Accept": "application/json"}, + timeout=settings.NETWORK_REQUEST_TIMEOUT, + ) + resp.raise_for_status() + return Jobset.model_validate(resp.json()) + + def get_latest_build( + self, project: ProjectName, jobset: JobsetName, job: JobName + ) -> Build: + resp = requests.get( + f"{self.base_url}/job/{project}/{jobset}/{job}/latest", + headers={"Accept": "application/json"}, + timeout=settings.NETWORK_REQUEST_TIMEOUT, + ) + resp.raise_for_status() + return Build.model_validate(resp.json()) + + def get_evaluation(self, evaluation: int) -> Evaluation: + resp = requests.get( + f"{self.base_url}/eval/{evaluation}", + headers={"Accept": "application/json"}, + timeout=settings.NETWORK_REQUEST_TIMEOUT, + ) + resp.raise_for_status() + return Evaluation.model_validate(resp.json()) + + +def default_client() -> HydraClient: + return HydraClient(str(settings.HYDRA_URL)) diff --git a/src/shared/listeners/nix_evaluation.py b/src/shared/listeners/nix_evaluation.py index b2abbbe8..7cdbf054 100644 --- a/src/shared/listeners/nix_evaluation.py +++ b/src/shared/listeners/nix_evaluation.py @@ -305,7 +305,9 @@ def _try_acquire_slot() -> bool: @pgpubsub.post_insert_listener(NixEvaluationChannel) def run_evaluation_job(old: NixEvaluation, new: NixEvaluation) -> None: - evaluation = NixEvaluation.objects.select_related("channel").get(pk=new.pk) + evaluation = NixEvaluation.objects.select_related("channel__release_branch").get( + pk=new.pk + ) average_evaluation_time = NixEvaluation.objects.aggregate( avg_eval_time=Avg("elapsed") ) diff --git a/src/shared/management/commands/fetch_all_channels.py b/src/shared/management/commands/fetch_all_channels.py index 60920f82..ab7ebd0d 100644 --- a/src/shared/management/commands/fetch_all_channels.py +++ b/src/shared/management/commands/fetch_all_channels.py @@ -1,86 +1,94 @@ -import asyncio -import sys -from collections.abc import Coroutine -from dataclasses import dataclass -from pprint import pprint from typing import Any -import requests from django.conf import settings from django.core.management.base import BaseCommand +from pydantic import BaseModel, field_validator -from shared.git import GitRepo -from shared.models.nix_evaluation import NixChannel, release_branch +# Populated at build time from `github:NixOS/infra//channels.nix` +from shared._release_channels import channels # type: ignore[reportMissingImports] +from shared import hydra +from shared.git import get_head_sha1 +from shared.models.nix_evaluation import NixChannel, NixpkgsBranch -@dataclass -class MonitoredChannel: - name: str - revision: str - status: str - variant: str | None +class Job(BaseModel): + project: hydra.ProjectName + jobset: hydra.JobsetName + name: hydra.JobName -def aggregate_by_channels(data: list[dict[str, Any]]) -> dict[str, MonitoredChannel]: - channels = {} - for metric in data: - m = metric["metric"] - channels[m["channel"]] = MonitoredChannel( - name=m["channel"], - revision=m["revision"], - status=m["status"], - variant=m.get("variant"), - ) - return channels +class Channel(BaseModel): + """ + A release channel as defined in `NixOS/infra`. + """ -def fetch_from_monitoring() -> dict[str, MonitoredChannel]: - resp = requests.get( - # XXX(@fricklerhandwerk): The sources for this are declared in the `NixOS/infra` repo. [tag:channel-structure] - # exporter logic: - # https://github.com/NixOS/infra/blob/795508213eb35eee099b1b8d12dd46a9f7b03697/build/pluto/prometheus/exporters/channel-exporter.py#L13-L17 - # systemd service: - # https://github.com/NixOS/infra/blob/795508213eb35eee099b1b8d12dd46a9f7b03697/build/pluto/prometheus/exporters/channel.nix#L4-L6 - # channel structure: - # https://github.com/NixOS/infra/blob/795508213eb35eee099b1b8d12dd46a9f7b03697/channels.nix - settings.CHANNEL_MONITORING_URL - ) - resp.raise_for_status() - return aggregate_by_channels(resp.json()["data"]["result"]) + job: Job + status: NixChannel.ChannelState + variant: NixChannel.Variant | None = None - -async def wait_for_parallel_fetches( - parallel_fetches: list[Coroutine[Any, Any, bool]], -) -> list[Any]: - return await asyncio.gather(*parallel_fetches, return_exceptions=True) + @field_validator("job", mode="before") + @classmethod + def parse_job(cls, v: str) -> dict: + project, jobset, name = v.split("/") + return {"project": project, "jobset": jobset, "name": name} class Command(BaseCommand): - help = "Register Nix channels" + help = "Fetch current channel tips and update source branches" def handle(self, *args: Any, **kwargs: Any) -> str | None: - fresh_channels = fetch_from_monitoring() - for channel in fresh_channels.values(): - channel_branch = channel.name - branch_info = { - "release_branch": release_branch(channel.name), - "state": NixChannel.ChannelState(channel.status), - "head_sha1_commit": channel.revision, - "variant": channel.variant, - } - pprint(branch_info | {"channel_branch": channel.name}) - NixChannel.objects.update_or_create( - branch_info, channel_branch=channel_branch + client = hydra.default_client() + + # FIXME(@fricklerhandwerk): Run requests async. + for channel_name, _raw in channels.items(): + channel = Channel.model_validate(_raw) # type: ignore[reportArgumentType] + job = channel.job + + jobset = client.get_jobset(project=job.project, jobset=job.jobset) + source = jobset.inputs[settings.HYDRA_INPUT_NAME] + branch_name = source.get_branch(default=settings.TRACKING_BRANCH) + + # By pre-configuring the source location on our end, we're decoupling the routing (where to get the data) from the parameters (which piece of the data to get). + # This slightly reduces our reliance on Hydra to be trustworthy: + # Limit the blast radius of compromise to be denial of service for updates instead of silent poisoning of new data. + assert str(settings.GIT_CLONE_URL).lower() == str(source.url).lower(), ( + f"Unexpected source URL: {source.url!r}, expected {settings.GIT_CLONE_URL!r}" + ) + + # FIXME(@fricklerhandwerk): Deduplicate the release branches beforehand. + # Due to which channels are currently marked "primary" in practice, this bumps each release branch exactly once. + # If something invalidates the heuristic, this will send unnecessary requests. + if channel.variant == NixChannel.Variant.PRIMARY: + commit_sha1 = get_head_sha1(source.url, branch_name) + branch, _ = NixpkgsBranch.objects.update_or_create( + name=branch_name, + defaults={"head_sha1_commit": commit_sha1}, + ) + else: + branch = NixpkgsBranch.objects.get(name=branch_name) + + latest_build = client.get_latest_build( + project=job.project, jobset=job.jobset, job=job.name + ) + evaluation = client.get_evaluation(latest_build.jobsetevals[0]) + eval_input = evaluation.jobsetevalinputs[settings.HYDRA_INPUT_NAME] + assert ( + str(eval_input.uri).lower() == str(source.url).lower() + ), ( # Sanity check + f"Unexpected eval input URI: {eval_input.uri!r}, expected {source.url!r}" ) + commit = eval_input.revision - repo = GitRepo( - settings.LOCAL_NIXPKGS_CHECKOUT, - stderr=sys.stderr.fileno(), - ) - parallel_fetches = [] - for channel in NixChannel.objects.iterator(): - parallel_fetches.append(repo.update_from_ref(channel.head_sha1_commit)) - - results = asyncio.run(wait_for_parallel_fetches(parallel_fetches)) - # FIXME(@fricklerhandwerk): Fold that into `branch_info`, so there's only one output. - print("Parallel fetches results", results) + ch, created = NixChannel.objects.update_or_create( + channel_branch=channel_name, + defaults={ + "release_branch": branch, + "head_sha1_commit": commit, + "state": channel.status, + "variant": channel.variant, + }, + ) + self.stdout.write( + f"Channel {ch.channel_branch} ({ch.state}) {'created at' if created else 'updated to'} {branch.head_sha1_commit[:8]}" + ) diff --git a/src/shared/migrations/0098_nixpkgsbranch_nixchannel_release_branch.py b/src/shared/migrations/0098_nixpkgsbranch_nixchannel_release_branch.py new file mode 100644 index 00000000..21cffa5a --- /dev/null +++ b/src/shared/migrations/0098_nixpkgsbranch_nixchannel_release_branch.py @@ -0,0 +1,74 @@ +import django.db.models.deletion +from django.db import migrations, models + + +def populate_release_branch(apps, schema_editor): + NixChannel = apps.get_model("shared", "NixChannel") + NixpkgsBranch = apps.get_model("shared", "NixpkgsBranch") + for channel in NixChannel.objects.all(): + branch, _ = NixpkgsBranch.objects.get_or_create( + name=channel.release_branch_name, + defaults={"head_sha1_commit": channel.head_sha1_commit}, + ) + channel.release_branch = branch + channel.save(update_fields=["release_branch"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("shared", "0097_nixchannel_variant"), + ] + + operations = [ + migrations.CreateModel( + name="NixpkgsBranch", + fields=[ + ( + "name", + models.CharField(max_length=126, primary_key=True, serialize=False), + ), + ("head_sha1_commit", models.CharField(max_length=126)), + ], + ), + migrations.RenameField( + model_name="nixchannel", + old_name="release_branch", + new_name="release_branch_name", + ), + migrations.AddField( + model_name="nixchannel", + name="release_branch", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="channels", + to="shared.nixpkgsbranch", + ), + ), + # The table has triggers registered. + # Altering it within the migration will defer them to after the transaction, thus failing the whole thing. + migrations.RunSQL("ALTER TABLE shared_nixchannel DISABLE TRIGGER USER"), + migrations.RunPython( + code=populate_release_branch, + reverse_code=migrations.RunPython.noop, + ), + migrations.AlterField( + model_name="nixchannel", + name="release_branch", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="channels", + to="shared.nixpkgsbranch", + ), + ), + migrations.RemoveField( + model_name="nixchannel", + name="release_branch_name", + ), + migrations.RemoveField( + model_name="nixchannel", + name="repository", + ), + migrations.RunSQL("ALTER TABLE shared_nixchannel ENABLE TRIGGER USER"), + ] diff --git a/src/shared/models/nix_evaluation.py b/src/shared/models/nix_evaluation.py index 978400c9..1cf3fdc8 100644 --- a/src/shared/models/nix_evaluation.py +++ b/src/shared/models/nix_evaluation.py @@ -1,5 +1,3 @@ -import re - from django.conf import settings from django.contrib.postgres import fields from django.contrib.postgres.indexes import BTreeIndex, GinIndex @@ -129,6 +127,18 @@ def __str__(self) -> str: return self.description or "" +class NixpkgsBranch(models.Model): + """ + A Nixpkgs branch that gets evaluated, e.g. `master` or `release-26.05`. + """ + + name = models.CharField(max_length=126, primary_key=True) + head_sha1_commit = models.CharField(max_length=126) + + def __str__(self) -> str: + return self.name + + class NixChannel(TimeStampMixin): """ This represents a "Nixpkgs" (*) channel, e.g. @@ -160,9 +170,12 @@ class Variant(models.TextChoices): ChannelState.UNSTABLE, ) - # A staging branch is the `release-$number` branch or `master` for unstable. - # Not to confuse with the `staging` branch itself. - release_branch = models.CharField(max_length=255) + # The underlying Nixpkgs branch (e.g. `master` or `release-25.05`). + release_branch = models.ForeignKey( + NixpkgsBranch, + on_delete=models.PROTECT, + related_name="channels", + ) # A channel branch is the `nixos-$number` branch of # `nixos-unstable(-small)` for unstable(-small). Not to confuse with the # channel tarballs and scripts from releases.nixos.org. @@ -171,14 +184,9 @@ class Variant(models.TextChoices): head_sha1_commit = models.CharField(max_length=255) state = models.CharField(max_length=126, choices=ChannelState.choices) variant = models.CharField(max_length=126, choices=Variant.choices, null=True) - # Repository can be stored as URLs for now... - # We can always reparse them as proper GitHub URIs if necessary - # It's a bit annoying though - # TODO(raitobezarius): make a proper ForeignKey? - repository = models.CharField(max_length=255) def __str__(self) -> str: - return f"{self.release_branch} -> {self.channel_branch}" + return self.channel_branch @property def is_tracking_branch(self) -> bool: @@ -187,7 +195,7 @@ def is_tracking_branch(self) -> bool: It's the source of truth for metadata such as package descriptions and maintainer information. """ - return self.channel_branch == settings.TRACKING_BRANCH + return self.release_branch.name == settings.TRACKING_BRANCH class NixEvaluationQuerySet(models.QuerySet): @@ -330,18 +338,3 @@ def get_major_channel(branch_name: str) -> str | None: if mc in branch_name: return f"nixos-{mc}" return None - - -def get_release(channel_branch: str) -> str: - match = re.match( - r"^(?P[a-z]+)-(?P\d\d\.\d\d|unstable)(?:-(?P[a-z]+))?$", - channel_branch, - ) - if match is None: - raise ValueError(f"unexpected channel branch name: {channel_branch!r}") - return match.group("release") - - -def release_branch(channel_branch: str) -> str: - release = get_release(channel_branch) - return "master" if release == "unstable" else f"release-{release}" diff --git a/src/shared/tests/conftest.py b/src/shared/tests/conftest.py index 4bff6c02..9cc47f93 100644 --- a/src/shared/tests/conftest.py +++ b/src/shared/tests/conftest.py @@ -37,7 +37,7 @@ NixDerivationMeta, NixEvaluation, NixMaintainer, - release_branch, + NixpkgsBranch, ) from shared.models.package import Package, PackageAttrpath from shared.notify_users import create_package_subscription_notifications @@ -134,11 +134,33 @@ def cve(make_container: Callable[..., Container]) -> Container: @pytest.fixture -def make_channel(db: None) -> Callable[..., NixChannel]: +def make_branch(db: None) -> Callable[..., NixpkgsBranch]: + def wrapped(name: str = settings.TRACKING_BRANCH) -> NixpkgsBranch: + branch, _ = NixpkgsBranch.objects.get_or_create( + name=name, + defaults={"head_sha1_commit": secrets.token_hex(20)}, + ) + return branch + + return wrapped + + +@pytest.fixture +def branch(make_branch: Callable[..., NixpkgsBranch]) -> NixpkgsBranch: + return make_branch() + + +@pytest.fixture +def make_channel( + db: None, + make_branch: Callable[..., NixpkgsBranch], + branch: NixpkgsBranch, +) -> Callable[..., NixChannel]: # FIXME(@fricklerhandwerk): This will fall apart when we obtain the channel structure dynamically [ref:channel-structure] def wrapped( - channel_branch: str = settings.TRACKING_BRANCH, + channel_branch: str = "nixpkgs-unstable", state: NixChannel.ChannelState = NixChannel.ChannelState.UNSTABLE, + branch: NixpkgsBranch = branch, variant: NixChannel.Variant | None = None, ) -> NixChannel: channel, _ = NixChannel.objects.get_or_create( @@ -146,9 +168,8 @@ def wrapped( defaults=dict( head_sha1_commit=secrets.token_hex(16), state=state, + release_branch=branch, variant=variant, - release_branch=release_branch(channel_branch), - repository="https://github.com/NixOS/nixpkgs", ), ) return channel diff --git a/src/shared/tests/hydra_fixtures/build.json b/src/shared/tests/hydra_fixtures/build.json new file mode 100644 index 00000000..ddc23b98 --- /dev/null +++ b/src/shared/tests/hydra_fixtures/build.json @@ -0,0 +1,24 @@ +{ + "buildmetrics": {}, + "buildoutputs": { + "out": { + "path": "/nix/store/nqr39d6fv455fqb81jwwjkpzxcq32lfs-nixos-26.11pre1025869.8465ac306246" + } + }, + "buildproducts": {}, + "buildstatus": 0, + "drvpath": "/nix/store/far9rcw9ilkjrbqnz9z9y6pwk7ck6ns6-nixos-26.11pre1025869.8465ac306246.drv", + "finished": 1, + "id": 334072527, + "job": "tested", + "jobset": "unstable-small", + "jobsetevals": [1826766], + "nixname": "nixos-26.11pre1025869.8465ac306246", + "priority": 100, + "project": "nixos", + "releasename": null, + "starttime": 1782918492, + "stoptime": 1782918495, + "system": "x86_64-linux", + "timestamp": 1782915595 +} diff --git a/src/shared/tests/hydra_fixtures/evaluation.json b/src/shared/tests/hydra_fixtures/evaluation.json new file mode 100644 index 00000000..51172333 --- /dev/null +++ b/src/shared/tests/hydra_fixtures/evaluation.json @@ -0,0 +1,44 @@ +{ + "builds": [ + 332754213, 332754215, 332754219, 332754222, 332754223, 332754224, 332754226, + 332754227, 332754231, 332754234, 332754235, 332754236, 332754237, 332754238, + 332754239, 332754240, 332754241, 332754243, 332754244, 332754245, 332754246, + 332754247, 332754248, 332754249, 332754250, 332754251, 332754252, 332754253, + 332754254, 332754255, 332754256, 332754258, 332754259, 332754260, 332754261, + 332754262, 332754263, 332754264, 333204565, 333204566, 333462561, 333462562, + 333462563, 333462564, 333462565, 333462567, 333462568, 333462570, 333462571, + 333462572, 333462573, 333462574, 333462575, 333462576, 333462583, 333462584, + 333462586, 333462587, 333462589, 333462590, 333462591, 333462592, 333462593, + 333462594, 333462595, 333462596, 333462598, 333462599, 333462600, 333462601, + 333462602, 333462604, 333462605, 333462606, 333462607, 333462608, 333462609, + 333462610, 333462611, 333462612, 333462613, 333462614, 333462616, 333462617, + 333462618, 333462619, 333462620, 333462621, 333462622, 333462623, 333462624, + 333462625, 333462627, 333462629, 333462630, 333462631, 333463241, 333463243, + 333463244, 333463245, 333653810, 333653811, 334072510, 334072511, 334072512, + 334072513, 334072514, 334072515, 334072516, 334072517, 334072518, 334072519, + 334072520, 334072521, 334072522, 334072523, 334072524, 334072525, 334072526, + 334072527 + ], + "checkouttime": 31, + "evaltime": 35, + "flake": null, + "hasnewbuilds": 1, + "id": 1826766, + "jobsetevalinputs": { + "nixpkgs": { + "dependency": null, + "revision": "8465ac306246eef81317cd771e7c27c8ac415aef", + "type": "git", + "uri": "https://github.com/NixOS/nixpkgs.git", + "value": null + }, + "stableBranch": { + "dependency": null, + "revision": null, + "type": "boolean", + "uri": null, + "value": "false" + } + }, + "timestamp": 1782915595 +} diff --git a/src/shared/tests/hydra_fixtures/jobset.json b/src/shared/tests/hydra_fixtures/jobset.json new file mode 100644 index 00000000..5a8ae2da --- /dev/null +++ b/src/shared/tests/hydra_fixtures/jobset.json @@ -0,0 +1,37 @@ +{ + "checkinterval": 43200, + "description": "NixOS Unstable (small)", + "emailoverride": "", + "enable_dynamic_run_command": false, + "enabled": 1, + "enableemail": false, + "errormsg": "", + "errortime": 1782915561, + "fetcherrormsg": "", + "flake": "", + "inputs": { + "nixpkgs": { + "emailresponsible": false, + "name": "nixpkgs", + "type": "git", + "value": "https://github.com/NixOS/nixpkgs.git" + }, + "stableBranch": { + "emailresponsible": false, + "name": "stableBranch", + "type": "boolean", + "value": "false" + } + }, + "keepnr": 3, + "lastcheckedtime": 1782915596, + "name": "unstable-small", + "nixexprinput": "nixpkgs", + "nixexprpath": "nixos/release-small.nix", + "project": "nixos", + "schedulingshares": 2000000, + "starttime": null, + "triggertime": null, + "type": 0, + "visible": true +} diff --git a/src/shared/tests/test_fetch_all_channels.py b/src/shared/tests/test_fetch_all_channels.py new file mode 100644 index 00000000..f147123b --- /dev/null +++ b/src/shared/tests/test_fetch_all_channels.py @@ -0,0 +1,137 @@ +from collections.abc import Callable, Generator +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pytest +from django.conf import settings +from django.core.management import call_command + +from shared.hydra import Build, EvalInput, Evaluation, Jobset, JobsetInput +from shared.management.commands.fetch_all_channels import Channel, Job +from shared.models.nix_evaluation import NixChannel, NixpkgsBranch + + +@pytest.fixture +def mock_channels() -> Callable: + default_channels = { + "nixos-unstable": Channel.model_construct( + job=Job(project="nixos", jobset="unstable", name="tested"), + status=NixChannel.ChannelState.UNSTABLE, + variant=NixChannel.Variant.PRIMARY, + ), + "nixos-unstable-small": Channel.model_construct( + job=Job(project="nixos", jobset="unstable-small", name="tested"), + status=NixChannel.ChannelState.UNSTABLE, + variant=NixChannel.Variant.SMALL, + ), + } + + @contextmanager + def factory(channels: dict[str, Channel] = default_channels) -> Generator[None]: + with patch( + "shared.management.commands.fetch_all_channels.channels", + channels, + ): + yield + + return factory + + +@pytest.fixture +def mock_hydra() -> Callable: + default_evaluation = Evaluation.model_construct( + id=42, + jobsetevalinputs={ + settings.HYDRA_INPUT_NAME: EvalInput( + uri=settings.GIT_CLONE_URL, + revision="c" * 40, + ) + }, + ) + default_jobsets = { + "nixos/unstable": Jobset.model_construct( + inputs={ + settings.HYDRA_INPUT_NAME: JobsetInput.model_construct( + url=settings.GIT_CLONE_URL, branch=None + ) + } + ), + "nixos/unstable-small": Jobset.model_construct( + inputs={ + settings.HYDRA_INPUT_NAME: JobsetInput.model_construct( + url=settings.GIT_CLONE_URL, branch=None + ) + } + ), + } + + @contextmanager + def factory( + jobsets: dict[str, Jobset] = default_jobsets, + evaluation: Evaluation = default_evaluation, + ) -> Generator[None]: + client = MagicMock() + client.get_jobset.side_effect = lambda project, jobset: jobsets[ + f"{project}/{jobset}" + ] + client.get_latest_build.return_value = Build(id=1, jobsetevals=[42]) + client.get_evaluation.return_value = evaluation + with patch( + "shared.hydra.default_client", + return_value=client, + ): + yield + + return factory + + +@pytest.fixture +def mock_head_sha1() -> Callable: + @contextmanager + def factory(branch_shas: dict[str, str]) -> Generator[None]: + with patch( + "shared.management.commands.fetch_all_channels.get_head_sha1", + side_effect=lambda _, branch: branch_shas[branch], + ): + yield + + return factory + + +@pytest.mark.django_db +def test_fetch_all_channels_creates_nix_branches( + mock_channels: Callable, mock_hydra: Callable, mock_head_sha1: Callable +) -> None: + with ( + mock_channels(), + mock_hydra(), + mock_head_sha1(branch_shas={"master": "a" * 40}), + ): + call_command("fetch_all_channels") + + assert NixpkgsBranch.objects.count() == 1 + assert NixpkgsBranch.objects.filter( + name=settings.TRACKING_BRANCH, head_sha1_commit="a" * 40 + ).exists() + assert NixChannel.objects.filter(channel_branch="nixos-unstable").exists() + assert NixChannel.objects.filter(channel_branch="nixos-unstable-small").exists() + + +@pytest.mark.django_db +def test_fetch_all_channels_updates_branch_head( + mock_channels: Callable, mock_hydra: Callable, mock_head_sha1: Callable +) -> None: + """ + Only primary channels fetch the branch HEAD. + Small channels sharing the same branch reuse the result. + """ + NixpkgsBranch.objects.create(name="master", head_sha1_commit="b" * 40) + + with ( + mock_channels(), + mock_hydra(), + mock_head_sha1(branch_shas={"master": "a" * 40}), + ): + call_command("fetch_all_channels") + + assert NixpkgsBranch.objects.get(name="master").head_sha1_commit == "a" * 40 diff --git a/src/shared/tests/test_hydra.py b/src/shared/tests/test_hydra.py new file mode 100644 index 00000000..4624d962 --- /dev/null +++ b/src/shared/tests/test_hydra.py @@ -0,0 +1,93 @@ +import json +from pathlib import Path + +import pytest +from pydantic import BaseModel + +from shared.hydra import Build, Evaluation, Jobset + +""" +To refresh fixtures, run from repository root: + +hydra() { curl -sLH 'Accept: application/json' "https://hydra.nixos.org/$1" | jq --sort-keys; } +hydra jobset/nixos/unstable-small \ + > src/shared/tests/hydra_fixtures/jobset.json + +hydra job/nixos/unstable-small/tested/latest \ + > src/shared/tests/hydra_fixtures/build.json + +eval=$(jq .jobsetevals[0] src/shared/tests/hydra_fixtures/build.json) +hydra eval/$eval \ + > src/shared/tests/hydra_fixtures/evaluation.json +""" + +# FIXME(@fricklerhandwerk): Update these on each dependency bump automatically, so we don't silently go out of sync if changes happen. +fixtures = Path(__file__).parent / "hydra_fixtures" + +jobset = json.loads((fixtures / "jobset.json").read_text()) + +build = json.loads((fixtures / "build.json").read_text()) + +evaluation = json.loads((fixtures / "evaluation.json").read_text()) + + +@pytest.mark.parametrize( + ("model", "data"), + [ + (Jobset, jobset), + (Build, build), + (Evaluation, evaluation), + ], +) +def test_hydra_output_parsing(model: type[BaseModel], data: dict) -> None: + model.model_validate(data) + + +def test_jobset_filters_non_git_inputs() -> None: + assert any(v["type"] != "git" for v in jobset["inputs"].values()) + parsed = Jobset.model_validate(jobset) + assert set(parsed.inputs.keys()) == { + k for k, v in jobset["inputs"].items() if v["type"] == "git" + } + + +def test_jobset_input_url_without_branch() -> None: + parsed = Jobset.model_validate( + { + "inputs": { + "nixpkgs": { + "type": "git", + "value": "https://github.com/NixOS/nixpkgs.git", + "emailresponsible": False, + "name": "nixpkgs", + } + } + } + ) + assert parsed.inputs["nixpkgs"].branch is None + assert parsed.inputs["nixpkgs"].get_branch(default="foo") == "foo" + + +def test_jobset_input_url_with_branch() -> None: + parsed = Jobset.model_validate( + { + "inputs": { + "nixpkgs": { + "type": "git", + "value": "https://github.com/NixOS/nixpkgs.git release-25.11", + "emailresponsible": False, + "name": "nixpkgs", + } + } + } + ) + assert parsed.inputs["nixpkgs"].branch == "release-25.11" + assert parsed.inputs["nixpkgs"].get_branch(default="master") == "release-25.11" + + +def test_evaluation_filters_non_git_inputs() -> None: + assert any(v["type"] != "git" for v in evaluation["jobsetevalinputs"].values()) + parsed = Evaluation.model_validate(evaluation) + assert set(parsed.jobsetevalinputs.keys()) == { + k for k, v in evaluation["jobsetevalinputs"].items() if v["type"] == "git" + } diff --git a/src/shared/tests/test_suggestion_caching.py b/src/shared/tests/test_suggestion_caching.py index 4ab6a2aa..2a67b495 100644 --- a/src/shared/tests/test_suggestion_caching.py +++ b/src/shared/tests/test_suggestion_caching.py @@ -17,6 +17,7 @@ NixDerivation, NixEvaluation, NixMaintainer, + NixpkgsBranch, ) from shared.models.package import Package from shared.package_clustering import cluster_packages @@ -86,6 +87,7 @@ def test_caching_newest_package( @pytest.mark.parametrize("rolling_has_maintainers", [True, False]) def test_maintainers_come_from_rolling_release_channel( cve: Container, + make_branch: Callable[..., NixpkgsBranch], make_channel: Callable[..., NixChannel], make_evaluation: Callable[..., NixEvaluation], make_drv: Callable[..., NixDerivation], @@ -99,8 +101,13 @@ def test_maintainers_come_from_rolling_release_channel( Its maintainers must appear regardless of evaluation order, and stable-only maintainers must never appear. """ - stable_channel = make_channel(channel_branch="nixos-26.05-small") - rolling_channel = make_channel(channel_branch=settings.TRACKING_BRANCH) + stable_channel = make_channel( + channel_branch="nixos-26.05-small", branch=make_branch(name="release-26.05") + ) + rolling_channel = make_channel( + channel_branch="nixos-unstable", + branch=make_branch(settings.TRACKING_BRANCH), + ) stable_age = timedelta(hours=1) if stable_is_older else timedelta(0) rolling_age = timedelta(0) if stable_is_older else timedelta(hours=1) eval_stable = make_evaluation(channel=stable_channel, age=stable_age)