diff --git a/README.md b/README.md index d35f4257..bab3b8d8 100644 --- a/README.md +++ b/README.md @@ -66,3 +66,14 @@ All provided images and tags can be searched at https://bob.hex.pm/docker. If you need a specific patch version that is not built automatically, request it at https://bob.hex.pm/request. Sign in with your hex.pm account, then pick the image kind (Erlang or Elixir), the OS and base image version, and the exact Erlang (and Elixir) version. The image is built for both architectures, including the multi-arch manifest; when an Elixir image needs an Erlang base image that does not exist yet, the base is built first and the Elixir image follows automatically. Requested builds are limited to ten image builds per user per hour. Build progress can be followed on the jobs dashboard at https://bob.hex.pm. + +Requesting an image also reserves it from cleanup, permanently. If the image already exists it is reserved without rebuilding, so requesting a build doubles as a way to keep an old image you depend on from being removed (see Retention and cleanup below). + +### Retention and cleanup + +Old tags are removed automatically so the repositories don't grow without bound: + +* The `hexpm/erlang` and `hexpm/elixir` repositories hold the multi-arch images you pull. A tag that has been neither pulled nor rebuilt for 180 days is removed. +* The per-architecture repositories (`hexpm/erlang-amd64`, `hexpm/erlang-arm64`, `hexpm/elixir-amd64`, `hexpm/elixir-arm64`) only hold the single-arch images the multi-arch manifests are assembled from. They keep the last 30 days of builds; older single-arch tags are removed even though the multi-arch image they back stays available. + +Tags reserved by a build request are never removed and are flagged as `reserved` on the [Docker tags page](https://bob.hex.pm/docker). To keep an image indefinitely, request it at https://bob.hex.pm/request. diff --git a/assets/css/app.css b/assets/css/app.css index dbba12a3..493a4cb1 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -977,7 +977,7 @@ pre, } .jt--dk { - min-width: 1040px; + min-width: 1180px; table-layout: fixed; } @@ -986,7 +986,7 @@ pre, } .col-dk-tag { - width: 510px; + width: 430px; } .col-dk-archs { @@ -1004,6 +1004,41 @@ pre, white-space: nowrap; } +.dk-tag-cell { + display: flex; + align-items: center; + gap: 8px; +} + +.dk-tag-cell .dk-tag-code { + flex: 1; + min-width: 0; +} + +.dk-reserved, +.dk-removing { + flex: none; + border-radius: 5px; + padding: 1px 7px; + font-size: 11px; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; + white-space: nowrap; +} + +.dk-reserved { + border: 1px solid var(--color-green-200); + background: var(--color-green-50); + color: var(--color-green-700); +} + +.dk-removing { + border: 1px solid var(--color-yellow-200); + background: var(--color-yellow-50); + color: var(--color-yellow-700); +} + .pager { display: flex; align-items: center; diff --git a/config/config.exs b/config/config.exs index c8d70650..a74c8b70 100644 --- a/config/config.exs +++ b/config/config.exs @@ -31,6 +31,12 @@ config :bob, time: {1, 0, 0}, queue: true ], + [ + module: Bob.Job.DockerCleanup, + period: :day, + time: {2, 0, 0}, + queue: true + ], [ module: Bob.Job.ReconcileBaseImages, period: {1, :hour}, @@ -51,7 +57,10 @@ config :bob, parallel_jobs: 1, local_jobs: [], remote_jobs: [], - hexpm_impl: Bob.Hexpm.Impl + hexpm_impl: Bob.Hexpm.Impl, + # :dry_run logs the counts a live run would delete without touching Docker + # Hub; :live deletes. + docker_cleanup_mode: :dry_run config :mime, :types, %{ "application/vnd.bob+erlang" => ["erlang"] diff --git a/lib/bob/artifacts.ex b/lib/bob/artifacts.ex index 68712b1f..e9c16e5c 100644 --- a/lib/bob/artifacts.ex +++ b/lib/bob/artifacts.ex @@ -6,7 +6,7 @@ defmodule Bob.Artifacts do @builds_txt_lock 4_771_002 - # Each staging row binds six parameters; Postgres caps a statement at 65535, + # Each staging row binds eight parameters; Postgres caps a statement at 65535, # so a page is inserted in chunks well under that ceiling. @staging_chunk 5000 @@ -17,6 +17,33 @@ defmodule Bob.Artifacts do # wraps. @long_query_timeout 5 * 60 * 1000 + @docker_cleanup_per_arch_repos ~w( + hexpm/erlang-amd64 hexpm/erlang-arm64 hexpm/elixir-amd64 hexpm/elixir-arm64 + ) + @docker_cleanup_manifest_repos ~w(hexpm/erlang hexpm/elixir) + + # A tag is reserved (never auto-deleted) while a non-expired build request pins + # it. A request maps deterministically to a single tag name, so the match is a + # plain equality on the indexed `tag` column — a hash anti-join against the + # small build_requests table — rather than extracting JSONB per scanned row. + # The erlang and elixir tag-name formats are disjoint (elixir carries + # `-erlang-`), so the name alone identifies the kind. `d` is the docker_tags + # row the predicate is applied to. The CASE mirrors + # `Bob.BuildRequests.request_target/1`; a test asserts the two encodings + # agree — update both together. + @reserved_predicate """ + EXISTS ( + SELECT 1 + FROM build_requests br + WHERE br.state IN ('pending', 'completed') + AND d.tag = + CASE br.kind + WHEN 'erlang' THEN br.erlang || '-' || br.os || '-' || br.os_version + ELSE br.elixir || '-erlang-' || br.erlang || '-' || br.os || '-' || br.os_version + END + ) + """ + def add(attrs) do upsert(attrs) generate_builds_txt(attrs.arch, attrs.os) @@ -24,15 +51,16 @@ defmodule Bob.Artifacts do :ok end - def add_docker_tag(repo, tag, archs, built_at \\ DateTime.utc_now()) do + def add_docker_tag(repo, tag, archs, built_at \\ DateTime.utc_now(), last_pulled \\ nil) do now = NaiveDateTime.utc_now() built_at = dump_utc_datetime(built_at) + last_pulled = last_pulled && dump_utc_datetime(last_pulled) search = DockerTagSearch.metadata(repo, tag) Repo.query!( """ - INSERT INTO docker_tags (repo, tag, archs, search, built_at, inserted_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $6) + INSERT INTO docker_tags (repo, tag, archs, search, built_at, last_pulled, inserted_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $7) ON CONFLICT (repo, tag) DO UPDATE SET archs = ( @@ -41,16 +69,17 @@ defmodule Bob.Artifacts do ), search = EXCLUDED.search, built_at = EXCLUDED.built_at, + last_pulled = COALESCE(EXCLUDED.last_pulled, docker_tags.last_pulled), updated_at = EXCLUDED.updated_at """, - [repo, tag, archs, search, built_at, now] + [repo, tag, archs, search, built_at, last_pulled, now] ) :ok end @doc """ - Inserts a page of `{tag, archs, built_at}` tuples into the staging table under `token`. + Inserts a page of `{tag, archs, built_at, last_pulled}` tuples into the staging table under `token`. Reconcile streams Docker Hub a page at a time into staging so the full tag list (up to ~1M for hexpm/elixir) is never held in memory and no connection is @@ -64,7 +93,7 @@ defmodule Bob.Artifacts do now = NaiveDateTime.utc_now() tag_archs_built_at - |> Enum.map(fn {tag, archs, built_at} -> + |> Enum.map(fn {tag, archs, built_at, last_pulled} -> %{ token: token, repo: repo, @@ -72,6 +101,7 @@ defmodule Bob.Artifacts do archs: archs, search: DockerTagSearch.metadata(repo, tag), built_at: built_at, + last_pulled: last_pulled, inserted_at: now } end) @@ -85,7 +115,9 @@ defmodule Bob.Artifacts do Applies the tags staged under `token` for `repo` to `docker_tags`, in one transaction: upsert every staged tag, prune any `docker_tags` row whose tag is no longer staged, then drop the staging rows. `DISTINCT ON` collapses any - duplicate tags Docker Hub returned across pages. + duplicate tags Docker Hub returned across pages. A staged tag with no pull + time keeps the row's recorded `last_pulled` — Docker Hub omits it + transiently, and losing it would expose a pulled tag to cleanup. Only call this once the full fetch succeeded — a partial fetch would prune tags that were merely not yet fetched. @@ -97,8 +129,8 @@ defmodule Bob.Artifacts do fn -> Repo.query!( """ - INSERT INTO docker_tags (repo, tag, archs, search, built_at, inserted_at, updated_at) - SELECT DISTINCT ON (repo, tag) repo, tag, archs, search, built_at, $3, $3 + INSERT INTO docker_tags (repo, tag, archs, search, built_at, last_pulled, inserted_at, updated_at) + SELECT DISTINCT ON (repo, tag) repo, tag, archs, search, built_at, last_pulled, $3, $3 FROM docker_tags_staging WHERE token = $1 AND repo = $2 ORDER BY repo, tag, built_at DESC @@ -107,10 +139,13 @@ defmodule Bob.Artifacts do archs = EXCLUDED.archs, search = EXCLUDED.search, built_at = EXCLUDED.built_at, + last_pulled = COALESCE(EXCLUDED.last_pulled, docker_tags.last_pulled), updated_at = EXCLUDED.updated_at WHERE docker_tags.archs IS DISTINCT FROM EXCLUDED.archs OR docker_tags.search IS DISTINCT FROM EXCLUDED.search OR docker_tags.built_at IS DISTINCT FROM EXCLUDED.built_at + OR COALESCE(EXCLUDED.last_pulled, docker_tags.last_pulled) + IS DISTINCT FROM docker_tags.last_pulled """, [token, repo, now], timeout: @long_query_timeout @@ -473,6 +508,128 @@ defmodule Bob.Artifacts do Enum.map(rows, fn [tag, archs] -> {tag, archs} end) end + def docker_cleanup_per_arch_repos(), do: @docker_cleanup_per_arch_repos + def docker_cleanup_manifest_repos(), do: @docker_cleanup_manifest_repos + + @doc """ + Counts, per repo, the tags the cleanup would delete: per-arch tags built + before `cutoff` and manifest tags neither pulled nor built since `cutoff`. + `GREATEST` keeps a tag whose last pull is ancient but that was re-pushed + recently — deleting it would 404 pullers only for the checker to rebuild the + manifest from its live per-arch tags. Reserved tags are excluded. Used by the + cleanup's dry run to report what live deletion would remove. + """ + def count_stale_per_arch_tags(cutoff) do + count_stale(@docker_cleanup_per_arch_repos, "d.built_at < $2", cutoff) + end + + def count_stale_manifest_tags(cutoff) do + count_stale( + @docker_cleanup_manifest_repos, + "GREATEST(d.last_pulled, d.built_at) < $2", + cutoff + ) + end + + defp count_stale(repos, age_predicate, cutoff) do + %{rows: rows} = + Repo.query!( + """ + SELECT d.repo, count(*) + FROM docker_tags d + WHERE d.repo = ANY($1) AND #{age_predicate} AND NOT #{@reserved_predicate} + GROUP BY d.repo + """, + [repos, dump_utc_datetime(cutoff)], + timeout: @long_query_timeout + ) + + Map.new(rows, fn [repo, count] -> {repo, count} end) + end + + @doc """ + A batch of `{repo, tag}` the cleanup may delete, same predicates as + `count_stale_*`. `limit` bounds the batch so a single run deletes a manageable + slice and converges over successive runs. + """ + def stale_per_arch_tags(cutoff, limit) do + stale(@docker_cleanup_per_arch_repos, "d.built_at < $2", cutoff, limit) + end + + def stale_manifest_tags(cutoff, limit) do + stale( + @docker_cleanup_manifest_repos, + "GREATEST(d.last_pulled, d.built_at) < $2", + cutoff, + limit + ) + end + + defp stale(repos, age_predicate, cutoff, limit) do + %{rows: rows} = + Repo.query!( + """ + SELECT d.repo, d.tag + FROM docker_tags d + WHERE d.repo = ANY($1) AND #{age_predicate} AND NOT #{@reserved_predicate} + LIMIT $3 + """, + [repos, dump_utc_datetime(cutoff), limit], + timeout: @long_query_timeout + ) + + Enum.map(rows, fn [repo, tag] -> {repo, tag} end) + end + + @doc "Whether a build request currently reserves `tag` in `repo`." + def docker_tag_reserved?(repo, tag) do + %{rows: [[reserved?]]} = + Repo.query!( + """ + SELECT EXISTS( + SELECT 1 FROM docker_tags d + WHERE d.repo = $1 AND d.tag = $2 AND #{@reserved_predicate} + ) + """, + [repo, tag] + ) + + reserved? + end + + @doc """ + The ids of the given `DockerTag` rows that a build request reserves, as a + MapSet. Lets the tags page flag reserved tags without re-deriving the match. + """ + def reserved_docker_tag_ids([]), do: MapSet.new() + + def reserved_docker_tag_ids(tags) do + ids = Enum.map(tags, & &1.id) + + %{rows: rows} = + Repo.query!( + "SELECT d.id FROM docker_tags d WHERE d.id = ANY($1::bigint[]) AND #{@reserved_predicate}", + [ids] + ) + + MapSet.new(rows, fn [id] -> id end) + end + + @doc "Drops `docker_tags` rows for the given `{repo, tag}` pairs, after Docker Hub deletion." + def delete_docker_tags([]), do: :ok + + def delete_docker_tags(repo_tags) do + repo_tags + |> Enum.group_by(fn {repo, _tag} -> repo end, fn {_repo, tag} -> tag end) + |> Enum.each(fn {repo, tags} -> + Repo.query!("DELETE FROM docker_tags WHERE repo = $1 AND tag = ANY($2)", [repo, tags], + timeout: @long_query_timeout + ) + end) + + :ok + end + def base_image_tags(repo) do Repo.all( from(b in BaseImageTag, diff --git a/lib/bob/artifacts/docker_tag.ex b/lib/bob/artifacts/docker_tag.ex index ab5a8806..cdfc18c7 100644 --- a/lib/bob/artifacts/docker_tag.ex +++ b/lib/bob/artifacts/docker_tag.ex @@ -7,6 +7,7 @@ defmodule Bob.Artifacts.DockerTag do field(:archs, {:array, :string}) field(:search, :map, default: %{}) field(:built_at, :utc_datetime_usec) + field(:last_pulled, :utc_datetime_usec) timestamps(type: :utc_datetime_usec) end end diff --git a/lib/bob/artifacts/docker_tag_staging.ex b/lib/bob/artifacts/docker_tag_staging.ex index a096f283..53ec325a 100644 --- a/lib/bob/artifacts/docker_tag_staging.ex +++ b/lib/bob/artifacts/docker_tag_staging.ex @@ -9,6 +9,7 @@ defmodule Bob.Artifacts.DockerTagStaging do field(:archs, {:array, :string}) field(:search, :map, default: %{}) field(:built_at, :utc_datetime_usec) + field(:last_pulled, :utc_datetime_usec) field(:inserted_at, :naive_datetime_usec) end end diff --git a/lib/bob/build_requests.ex b/lib/bob/build_requests.ex index 7b04a2ce..f9055433 100644 --- a/lib/bob/build_requests.ex +++ b/lib/bob/build_requests.ex @@ -16,6 +16,7 @@ defmodule Bob.BuildRequests do with :ok <- validate_combo(request) do case Bob.Job.DockerChecker.request_jobs(request) do [] -> + reserve(request, attrs) {:ok, :already_built} jobs -> @@ -24,6 +25,57 @@ defmodule Bob.BuildRequests do end end + # An already-built image still records a request, so a manual request doubles as + # a permanent reservation that exempts the image from cleanup. No build runs, so + # it costs nothing against the hourly limit; an existing reservation for the same + # target is left in place rather than duplicated. The check-then-insert is + # serialized on the target — reservations dedupe across users, so a per-user + # lock would not keep two users' concurrent submits from both inserting. + defp reserve(request, attrs) do + {:ok, :ok} = + Repo.transaction(fn -> + lock_target(request) + + unless reserved?(request) do + {:ok, created} = create(Map.put(attrs, :builds_count, 0)) + created = complete(created) + + Logger.info( + "BUILD RESERVATION by #{created.username}: #{created.kind} #{request_target(created)}" + ) + end + + :ok + end) + + :ok + end + + defp lock_target(request) do + Repo.query!("SELECT pg_advisory_xact_lock(hashtext($1))", [ + "#{request.kind}:#{request_target(request)}" + ]) + end + + defp reserved?(request) do + query = + from(request_row in BuildRequest, + where: + request_row.state in ["pending", "completed"] and request_row.kind == ^request.kind and + request_row.erlang == ^request.erlang and request_row.os == ^request.os and + request_row.os_version == ^request.os_version + ) + + query = + if request.kind == "elixir" do + from(request_row in query, where: request_row.elixir == ^request.elixir) + else + query + end + + Repo.exists?(query) + end + defp submit_jobs(request, attrs, jobs) do builds_count = builds_count(request.kind, jobs) @@ -108,11 +160,16 @@ defmodule Bob.BuildRequests do builds_count_for_user_since(username, hour_ago) + builds_count > @hourly_build_limit end - defp request_target(%{kind: "erlang"} = request) do + @doc """ + The Docker tag name a request maps to. The reservation predicate in + `Bob.Artifacts` rebuilds this same mapping in SQL, and a test asserts the two + encodings agree — update both together. + """ + def request_target(%{kind: "erlang"} = request) do "#{request.erlang}-#{request.os}-#{request.os_version}" end - defp request_target(%{kind: "elixir"} = request) do + def request_target(%{kind: "elixir"} = request) do "#{request.elixir}-erlang-#{request.erlang}-#{request.os}-#{request.os_version}" end @@ -162,8 +219,10 @@ defmodule Bob.BuildRequests do end @doc """ - Deletes finished requests older than `older_than_seconds`. Pending requests - are left alone; the checker reconciliation expires or completes them first. + Deletes expired requests older than `older_than_seconds`. Completed requests + are permanent reservations and pending requests are still in flight, so only + expired dead-ends (an os_version that rotated out, or a request past its TTL) + are reclaimed. """ def prune(older_than_seconds) do cutoff = DateTime.add(DateTime.utc_now(), -older_than_seconds, :second) @@ -171,7 +230,7 @@ defmodule Bob.BuildRequests do {count, _} = Repo.delete_all( from(request in BuildRequest, - where: request.state in ["completed", "expired"] and request.inserted_at < ^cutoff + where: request.state == "expired" and request.inserted_at < ^cutoff ) ) diff --git a/lib/bob/docker_cleanup.ex b/lib/bob/docker_cleanup.ex new file mode 100644 index 00000000..cfe0ecd3 --- /dev/null +++ b/lib/bob/docker_cleanup.ex @@ -0,0 +1,125 @@ +defmodule Bob.DockerCleanup do + @moduledoc """ + Prunes Docker Hub tags Bob no longer needs to keep. + + Per-arch repos (build staging) drop tags built more than 30 days ago; manifest + repos drop tags neither pulled nor built in 180 days. Tags pinned by a build + request reservation are always kept — see `Bob.BuildRequests`. + + `run/1` honours `:docker_cleanup_mode` (`:dry_run` by default, or `:live`). A + dry run logs and returns the counts a live run would delete without touching + Docker Hub; a live run deletes a bounded batch, paced by the shared rate + limiter, and converges over successive daily runs. + """ + + require Logger + + alias Bob.Artifacts + + @per_arch_max_age_days 30 + @manifest_unpulled_days 180 + @default_batch 100_000 + + def run(opts \\ []) do + case Keyword.get(opts, :mode, configured_mode()) do + :dry_run -> dry_run() + :live -> live(opts) + end + end + + def per_arch_max_age_days(), do: @per_arch_max_age_days + def manifest_unpulled_days(), do: @manifest_unpulled_days + + @doc """ + When a tag becomes eligible for removal, or `nil` for a repo not under cleanup. + Per-arch tags expire a fixed number of days after they were built; manifest + tags that many days after their last pull or build, whichever is later. + Mirrors the cleanup's own rule so the UI and the deleter never disagree. + """ + def removal_at(repo, built_at, last_pulled) do + cond do + repo in Artifacts.docker_cleanup_per_arch_repos() -> + DateTime.add(built_at, @per_arch_max_age_days, :day) + + repo in Artifacts.docker_cleanup_manifest_repos() -> + last_activity = Enum.max([built_at | List.wrap(last_pulled)], DateTime) + DateTime.add(last_activity, @manifest_unpulled_days, :day) + + true -> + nil + end + end + + defp configured_mode(), do: Application.get_env(:bob, :docker_cleanup_mode, :dry_run) + + defp dry_run() do + per_arch = Artifacts.count_stale_per_arch_tags(per_arch_cutoff()) + manifest = Artifacts.count_stale_manifest_tags(manifest_cutoff()) + + Logger.info( + "DOCKER CLEANUP dry-run: per-arch built_at < #{@per_arch_max_age_days}d -> " <> + "#{format_counts(per_arch)}; manifest pulled/built < #{@manifest_unpulled_days}d -> " <> + "#{format_counts(manifest)}" + ) + + {:dry_run, %{per_arch: per_arch, manifest: manifest}} + end + + defp live(opts) do + deleter = Keyword.get(opts, :deleter, &Bob.DockerHub.delete_tag/2) + limit = Keyword.get(opts, :limit, configured_batch()) + + # `limit` budgets the whole run — per-arch candidates first, the remainder + # for manifests — so backlog in both groups can't double a day's + # rate-limited deletes. + per_arch = Artifacts.stale_per_arch_tags(per_arch_cutoff(), limit) + candidates = per_arch ++ stale_manifest_tags(limit - length(per_arch)) + + deleted = delete(candidates, deleter) + Logger.info("DOCKER CLEANUP deleted #{deleted}/#{length(candidates)} tag(s)") + {:live, deleted} + end + + defp stale_manifest_tags(limit) when limit <= 0, do: [] + defp stale_manifest_tags(limit), do: Artifacts.stale_manifest_tags(manifest_cutoff(), limit) + + # Each tag is removed from docker_tags only once Docker Hub confirms deletion, + # so a tag that errors is left in place and reappears as a candidate next run. + # Reservations are re-checked per tag: the rate-limited batch can run for + # hours after the candidates were selected, and a request made mid-run must + # still pin its tag. + defp delete(candidates, deleter) do + deleted = + Enum.filter(candidates, fn {repo, tag} -> + not Artifacts.docker_tag_reserved?(repo, tag) and confirmed_delete?(deleter, repo, tag) + end) + + Artifacts.delete_docker_tags(deleted) + length(deleted) + end + + defp confirmed_delete?(deleter, repo, tag) do + case deleter.(repo, tag) do + :ok -> + true + + {:error, reason} -> + Logger.error("DOCKER CLEANUP failed to delete #{repo}:#{tag}: #{inspect(reason)}") + false + end + end + + defp per_arch_cutoff(), do: DateTime.add(DateTime.utc_now(), -@per_arch_max_age_days, :day) + defp manifest_cutoff(), do: DateTime.add(DateTime.utc_now(), -@manifest_unpulled_days, :day) + + defp configured_batch(), do: Application.get_env(:bob, :docker_cleanup_batch, @default_batch) + + defp format_counts(counts) do + total = counts |> Map.values() |> Enum.sum() + + detail = + counts |> Enum.sort() |> Enum.map_join(", ", fn {repo, count} -> "#{repo}=#{count}" end) + + "#{total} (#{detail})" + end +end diff --git a/lib/bob/docker_hub.ex b/lib/bob/docker_hub.ex index 553cdb41..e0a7a8ca 100644 --- a/lib/bob/docker_hub.ex +++ b/lib/bob/docker_hub.ex @@ -1,6 +1,12 @@ defmodule Bob.DockerHub do + alias Bob.DockerHub.RateLimiter + @dockerhub_url "https://hub.docker.com/" + # Caps re-attempts after a 429 so a permanently throttled IP can't loop forever; + # each attempt first feeds the limiter, which blocks until the window resets. + @max_rate_limit_retries 20 + def auth(username, password) do url = @dockerhub_url <> "v2/users/login/" headers = [{"content-type", "application/json"}] @@ -18,10 +24,10 @@ defmodule Bob.DockerHub do @doc """ Pages every tag of `repo` from Docker Hub, invoking `on_page` with each page's - `{tag, archs, built_at}` list as it arrives. Returns `:ok` once the full set - is fetched or `:error` if any page failed, so the caller can avoid applying a - partial set. Pages stream through `on_page` rather than accumulating, so the - response set is never held in memory in full. + `{tag, archs, built_at, last_pulled}` list as it arrives. Returns `:ok` once + the full set is fetched or `:error` if any page failed, so the caller can avoid + applying a partial set. Pages stream through `on_page` rather than accumulating, + so the response set is never held in memory in full. """ def stream_repo_tags(repo, on_page) do url = @dockerhub_url <> "v2/repositories/#{repo}/tags?page=${page}&page_size=100" @@ -52,19 +58,54 @@ defmodule Bob.DockerHub do end end + @doc """ + Deletes a tag on Docker Hub, paced by the shared rate limiter. A missing tag + (404) counts as success so the cleanup is idempotent across retries. Returns + `{:error, _}` for any other outcome so the caller can skip the tag and retry it + on a later run rather than crash the batch. + """ def delete_tag(repo, tag) do url = @dockerhub_url <> "v2/repositories/#{repo}/tags/#{tag}" - headers = headers() - opts = [recv_timeout: 20_000] + + case paced_request(:delete, url) do + {:ok, status, _headers, _body} when status in [204, 404] -> :ok + other -> {:error, other} + end + end + + @doc """ + Sends `method` to `url` paced by the shared rate limiter: acquires a slot + (blocking until the window has budget), feeds the response back so the gate + tracks the limit, and waits out 429s up to the retry cap. Returns the final + non-429 result for the caller to match on. Every outcome reports to the + limiter — a response without usable rate headers, or a transport error, + releases the calibration probe's slot — so a failed request can't leave the + gate shut for every later caller. + """ + def paced_request(method, url), do: paced_request(method, url, 0) + + defp paced_request(method, url, attempts) do + RateLimiter.acquire() result = - Bob.HTTP.retry("DockerHub #{url}", fn -> - Bob.HTTP.request(:delete, url, headers, "", opts) - end) + Bob.HTTP.retry( + "DockerHub #{url}", + fn -> Bob.HTTP.request(method, url, headers(), "", recv_timeout: 20_000) end, + retry_rate_limit?: false + ) case result do - {:ok, 204, _headers, _body} -> :ok - {:ok, 404, _headers, _body} -> :ok + {:ok, 429, response_headers, _body} when attempts < @max_rate_limit_retries -> + RateLimiter.throttle(response_headers) + paced_request(method, url, attempts + 1) + + {:ok, _status, response_headers, _body} -> + RateLimiter.observe(response_headers) + result + + {:error, _reason} -> + RateLimiter.observe([]) + result end end @@ -90,7 +131,17 @@ defmodule Bob.DockerHub do else # DockerHub returns dupes sometimes? archs = images |> Enum.map(&:binary.copy(&1["architecture"])) |> Enum.uniq() - {:binary.copy(result["name"]), archs, built_at} + {:binary.copy(result["name"]), archs, built_at, last_pulled(result)} + end + end + + # `tag_last_pulled` is the last time Docker Hub served this tag's manifest. It + # is null for a tag that has never been pulled, and historically a zero + # sentinel; both map to nil so callers can treat "never pulled" uniformly. + defp last_pulled(result) do + case result["tag_last_pulled"] do + "0001-01-01" <> _ -> nil + value -> parse_timestamp(value) end end diff --git a/lib/bob/docker_hub/pager.ex b/lib/bob/docker_hub/pager.ex index 80eda855..8c309709 100644 --- a/lib/bob/docker_hub/pager.ex +++ b/lib/bob/docker_hub/pager.ex @@ -1,15 +1,9 @@ defmodule Bob.DockerHub.Pager do use GenServer - alias Bob.DockerHub.RateLimiter - @concurrency 50 @timeout 60 * 60 * 1000 - # Cap re-acquires after a 429 so a permanently throttled IP can't loop forever; - # each re-acquire already blocks on the rate limiter until the window resets. - @max_rate_limit_retries 20 - def start_link(url, on_page) do GenServer.start_link(__MODULE__, {url, on_page}) end @@ -83,7 +77,7 @@ defmodule Bob.DockerHub.Pager do defp next_request(state) do if MapSet.size(state.tasks) < @concurrency do url = String.replace(state.url, "${page}", Integer.to_string(state.page)) - task = Task.async(fn -> fetch_page(url, 0) end) + task = Task.async(fn -> fetch_page(url) end) state = %{state | page: state.page + 1, tasks: MapSet.put(state.tasks, task.ref)} next_request(state) else @@ -91,36 +85,15 @@ defmodule Bob.DockerHub.Pager do end end - # Each attempt acquires from the rate limiter (which blocks until the window - # has budget), so a 429 simply feeds the limiter and re-acquires — the next - # attempt waits for the window to reset rather than hammering. - defp fetch_page(url, attempts) do - headers = Bob.DockerHub.headers() - opts = [recv_timeout: 20_000] - - RateLimiter.acquire() - - result = - Bob.HTTP.retry( - "DockerHub #{url}", - fn -> Bob.HTTP.request(:get, url, headers, "", opts) end, - retry_rate_limit?: false - ) - - case result do - {:ok, 200, headers, body} -> - RateLimiter.observe(headers) + defp fetch_page(url) do + case Bob.DockerHub.paced_request(:get, url) do + {:ok, 200, _headers, body} -> decoded = JSON.decode!(body) {:ok, Enum.flat_map(decoded["results"], &List.wrap(Bob.DockerHub.parse(&1)))} - {:ok, 404, headers, _body} -> - RateLimiter.observe(headers) + {:ok, 404, _headers, _body} -> :done - {:ok, 429, headers, _body} when attempts < @max_rate_limit_retries -> - RateLimiter.throttle(headers) - fetch_page(url, attempts + 1) - other -> {:error, other} end diff --git a/lib/bob/docker_hub/rate_limiter.ex b/lib/bob/docker_hub/rate_limiter.ex index 79a2c5a5..815fbe10 100644 --- a/lib/bob/docker_hub/rate_limiter.ex +++ b/lib/bob/docker_hub/rate_limiter.ex @@ -15,9 +15,10 @@ defmodule Bob.DockerHub.RateLimiter do it, when our clock runs slightly ahead of Docker Hub's. A request sent before any response has anchored the window is the lone - calibration probe. The gate monitors it and reclaims its slot if it dies - without reporting a response — a non-2xx/429 status or a crash — so a probe - that never feeds back can't wedge every later caller behind it forever. + calibration probe. The gate reclaims its slot if it reports a response + without usable rate headers, or dies without reporting at all, so a probe + that never anchors the window can't wedge every later caller behind it + forever. """ use GenServer @@ -44,9 +45,15 @@ defmodule Bob.DockerHub.RateLimiter do GenServer.call(server, :acquire, :infinity) end - @doc "Feeds a response's headers back so the gate tracks the limit and window." + @doc """ + Feeds a response's headers back so the gate tracks the limit and window. + Callers report every completed request, headers or not (`[]` for a transport + error): a report without usable rate headers releases the calibration + probe's slot when it comes from the probe itself, since that request + finished without anchoring the window. + """ def observe(headers, server \\ __MODULE__) do - GenServer.cast(server, {:observe, parse_rate(headers)}) + GenServer.cast(server, {:observe, parse_rate(headers), self()}) end @doc """ @@ -82,6 +89,7 @@ defmodule Bob.DockerHub.RateLimiter do offset_ms: Keyword.get(opts, :offset_ms, @resume_offset_ms), timer: nil, probe_mon: nil, + probe_pid: nil, waiters: :queue.new() }} end @@ -97,10 +105,17 @@ defmodule Bob.DockerHub.RateLimiter do end end + # The probe finished without usable rate headers, so nothing anchored the + # window: reclaim its slot the same way its death would, or the gate stays + # shut when the probe runs in a long-lived process. @impl true - def handle_cast({:observe, nil}, state), do: {:noreply, state} + def handle_cast({:observe, nil, pid}, %{probe_pid: pid} = state) do + {:noreply, schedule_resume(release(reclaim_probe(state)))} + end + + def handle_cast({:observe, nil, _pid}, state), do: {:noreply, state} - def handle_cast({:observe, rate}, state) do + def handle_cast({:observe, rate, _pid}, state) do state = apply_rate(state, rate) state = if state.window != nil, do: clear_probe_mon(state), else: state {:noreply, schedule_resume(release(state))} @@ -127,13 +142,11 @@ defmodule Bob.DockerHub.RateLimiter do {:noreply, schedule_resume(release(roll(%{state | timer: nil})))} end - # The calibration probe finished. If it never anchored the window, reclaim its + # The calibration probe died. If it never anchored the window, reclaim its # slot and wake the next caller, so a probe that failed to report a response # can't leave the gate shut with nothing left to roll or release it. def handle_info({:DOWN, ref, :process, _pid, _reason}, %{probe_mon: ref} = state) do - state = %{state | probe_mon: nil} - state = if state.window == nil, do: %{state | sent: max(state.sent - 1, 0)}, else: state - {:noreply, schedule_resume(release(state))} + {:noreply, schedule_resume(release(reclaim_probe(state)))} end def handle_info({:DOWN, _ref, :process, _pid, _reason}, state) do @@ -152,19 +165,27 @@ defmodule Bob.DockerHub.RateLimiter do state = %{state | sent: state.sent + 1} if state.window == nil and state.probe_mon == nil do - %{state | probe_mon: Process.monitor(elem(from, 0))} + pid = elem(from, 0) + %{state | probe_mon: Process.monitor(pid), probe_pid: pid} else state end end + # The probe finished without anchoring the window: give its slot back so the + # next caller becomes the new probe. + defp reclaim_probe(state) do + state = clear_probe_mon(state) + if state.window == nil, do: %{state | sent: max(state.sent - 1, 0)}, else: state + end + # Stop tracking the calibration probe once the window is anchored; its eventual # exit no longer tells us anything. defp clear_probe_mon(%{probe_mon: nil} = state), do: state defp clear_probe_mon(%{probe_mon: ref} = state) do Process.demonitor(ref, [:flush]) - %{state | probe_mon: nil} + %{state | probe_mon: nil, probe_pid: nil} end # Folds a response's rate headers into the window: anchor it on the window's diff --git a/lib/bob/job/docker_checker.ex b/lib/bob/job/docker_checker.ex index 686bc5fc..5eca571b 100644 --- a/lib/bob/job/docker_checker.ex +++ b/lib/bob/job/docker_checker.ex @@ -527,42 +527,54 @@ defmodule Bob.Job.DockerChecker do @request_ttl_days 14 # User-requested builds are one-time: pinned to the os_version current at - # request time and reconciled here until every target tag exists. Requests - # whose os_version rotated out of builds() expire instead. + # request time and reconciled here until every target tag exists. A fully + # built request completes — and stays a permanent reservation — even if its + # os_version has since rotated out of builds() or its TTL has passed; only + # requests that still need jobs expire on those grounds. def requests() do builds = builds() Enum.each(Bob.BuildRequests.pending(), fn request -> - cond do - request.os_version not in Map.get(builds, request.os, []) -> - Bob.BuildRequests.expire(request) + case request_jobs(request) do + [] -> + Bob.BuildRequests.complete(request) - DateTime.diff(DateTime.utc_now(), request.inserted_at, :day) >= @request_ttl_days -> - Bob.BuildRequests.expire(request) + jobs -> + cond do + request.os_version not in Map.get(builds, request.os, []) -> + Bob.BuildRequests.expire(request) - true -> - case request_jobs(request) do - [] -> Bob.BuildRequests.complete(request) - jobs -> Bob.Queue.add_many(jobs) + DateTime.diff(DateTime.utc_now(), request.inserted_at, :day) >= @request_ttl_days -> + Bob.BuildRequests.expire(request) + + true -> + Bob.Queue.add_many(jobs) end end end) end + # A complete manifest means the image users pull already exists, so there is + # nothing to build even when the per-arch staging tags have been cleaned + # away — a request for it is satisfied (and reserved) as-is. def request_jobs(%{kind: "erlang"} = request) do %{erlang: erlang, os: os, os_version: os_version} = request tag = "#{erlang}-#{os}-#{os_version}" - arch_jobs = - Enum.flat_map(@archs, fn arch -> - if tag_present?("hexpm/erlang-#{arch}", tag) do - [] - else - [{{Bob.Job.BuildDockerErlang, arch}, [erlang, os, os_version]}] - end - end) + if manifest_complete?("erlang", tag) do + [] + else + arch_jobs = + Enum.flat_map(@archs, fn arch -> + if tag_present?("hexpm/erlang-#{arch}", tag) do + [] + else + [{{Bob.Job.BuildDockerErlang, arch}, [erlang, os, os_version]}] + end + end) - arch_jobs ++ manifest_jobs(arch_jobs, "erlang", tag, {erlang, os, os_version}) + arch_jobs ++ manifest_jobs(arch_jobs, "erlang", {erlang, os, os_version}) + end end # The elixir image is built FROM the erlang image, and failed jobs are not @@ -574,35 +586,32 @@ defmodule Bob.Job.DockerChecker do erlang_tag = "#{erlang}-#{os}-#{os_version}" elixir_tag = "#{elixir}-erlang-#{erlang}-#{os}-#{os_version}" - arch_jobs = - Enum.flat_map(@archs, fn arch -> - cond do - tag_present?("hexpm/elixir-#{arch}", elixir_tag) -> - [] + if manifest_complete?("elixir", elixir_tag) do + [] + else + arch_jobs = + Enum.flat_map(@archs, fn arch -> + cond do + tag_present?("hexpm/elixir-#{arch}", elixir_tag) -> + [] - tag_present?("hexpm/erlang-#{arch}", erlang_tag) -> - [{{Bob.Job.BuildDockerElixir, arch}, [elixir, erlang, os, os_version]}] + tag_present?("hexpm/erlang-#{arch}", erlang_tag) -> + [{{Bob.Job.BuildDockerElixir, arch}, [elixir, erlang, os, os_version]}] - true -> - [{{Bob.Job.BuildDockerErlang, arch}, [erlang, os, os_version]}] - end - end) + true -> + [{{Bob.Job.BuildDockerErlang, arch}, [erlang, os, os_version]}] + end + end) - arch_jobs ++ manifest_jobs(arch_jobs, "elixir", elixir_tag, {elixir, erlang, os, os_version}) + arch_jobs ++ manifest_jobs(arch_jobs, "elixir", {elixir, erlang, os, os_version}) + end end # A request is only done once its manifest spans every architecture, so while # the per-arch tags are still building we wait, and once they exist we enqueue # the manifest ourselves rather than relying solely on manifest/0. - defp manifest_jobs(pending_arch_jobs, _kind, _tag, _key) when pending_arch_jobs != [], do: [] - - defp manifest_jobs([], kind, tag, key) do - if manifest_complete?(kind, tag) do - [] - else - [{Bob.Job.DockerManifest, [kind, key]}] - end - end + defp manifest_jobs(pending_arch_jobs, _kind, _key) when pending_arch_jobs != [], do: [] + defp manifest_jobs([], kind, key), do: [{Bob.Job.DockerManifest, [kind, key]}] defp manifest_complete?(kind, tag) do case Bob.Artifacts.docker_tag_archs("hexpm/#{kind}", tag) do diff --git a/lib/bob/job/docker_cleanup.ex b/lib/bob/job/docker_cleanup.ex new file mode 100644 index 00000000..dfd128c1 --- /dev/null +++ b/lib/bob/job/docker_cleanup.ex @@ -0,0 +1,9 @@ +defmodule Bob.Job.DockerCleanup do + def run() do + Bob.DockerCleanup.run() + end + + def priority(), do: 1 + def weight(), do: 1 + def concurrency(), do: :shared +end diff --git a/lib/bob/reconcile.ex b/lib/bob/reconcile.ex index 0259ba54..fef83605 100644 --- a/lib/bob/reconcile.ex +++ b/lib/bob/reconcile.ex @@ -71,7 +71,9 @@ defmodule Bob.Reconcile do defp sync_per_arch_repos(stream) do Enum.each(@per_arch_repos, fn {repo, arch} -> swap_docker_tags(stream, repo, fn page -> - Enum.map(page, fn {tag, _archs, built_at} -> {tag, [arch], built_at} end) + Enum.map(page, fn {tag, _archs, built_at, last_pulled} -> + {tag, [arch], built_at, last_pulled} + end) end) end) end @@ -79,7 +81,9 @@ defmodule Bob.Reconcile do defp sync_manifest_repos(stream) do Enum.each(@manifest_repos, fn repo -> swap_docker_tags(stream, repo, fn page -> - Enum.map(page, fn {tag, archs, built_at} -> {tag, known_archs(archs), built_at} end) + Enum.map(page, fn {tag, archs, built_at, last_pulled} -> + {tag, known_archs(archs), built_at, last_pulled} + end) end) end) end diff --git a/lib/bob_web/live/docker_tags_live.ex b/lib/bob_web/live/docker_tags_live.ex index 8e37811f..cba258be 100644 --- a/lib/bob_web/live/docker_tags_live.ex +++ b/lib/bob_web/live/docker_tags_live.ex @@ -4,6 +4,10 @@ defmodule BobWeb.DockerTagsLive do @page 100 @filter_keys ~w(repo tag arch elixir_version erlang_version os os_version)a + # Flag a tag as removing once it is within this many days of (or past) its + # cleanup removal date. + @warn_days 14 + @impl true def mount(_params, _session, socket) do options = Bob.Artifacts.docker_tag_filter_options() @@ -16,7 +20,8 @@ defmodule BobWeb.DockerTagsLive do oses: options.oses, total: nil, loading: true, - results: [] + results: [], + reserved: MapSet.new() ) {:ok, socket} @@ -110,12 +115,32 @@ defmodule BobWeb.DockerTagsLive do |> max(offset + length(results)) end - assign(socket, results: results, total: total, loading: false) + reserved = Bob.Artifacts.reserved_docker_tag_ids(results) + + assign(socket, results: results, total: total, loading: false, reserved: reserved) end defp fmt(nil), do: "—" defp fmt(datetime), do: Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S") + # The retention badge a tag earns: reserved (kept), {:removing, days_left} when + # within @warn_days of (or past) its removal date, or :none. + defp retention_status(true, _d), do: :reserved + + defp retention_status(false, d) do + case Bob.DockerCleanup.removal_at(d.repo, d.built_at, d.last_pulled) do + nil -> + :none + + at -> + days = DateTime.diff(at, DateTime.utc_now(), :day) + if days <= @warn_days, do: {:removing, days}, else: :none + end + end + + defp removing_label(days) when days <= 0, do: "removing" + defp removing_label(days), do: "removing ~#{days}d" + defp count_label(nil), do: "Loading tags" defp count_label(0), do: "0 tags" defp count_label(count), do: "#{format_count(count)} #{format_unit("tags", count)}" @@ -176,7 +201,24 @@ defmodule BobWeb.DockerTagsLive do <:col :let={d} label="tag" class="col-dk-tag"> - <%= d.tag %> +
+ <%= d.tag %> + <% status = retention_status(MapSet.member?(@reserved, d.id), d) %> + + reserved + + + <%= removing_label(elem(status, 1)) %> + +
<:col :let={d} label="archs" class="col-dk-archs">
@@ -186,6 +228,9 @@ defmodule BobWeb.DockerTagsLive do <:col :let={d} label="built" class="col-time"> <%= fmt(d.built_at) %> + <:col :let={d} label="last pulled" class="col-time"> + <%= fmt(d.last_pulled) %> +
No matching tags.
diff --git a/lib/bob_web/live/request_live.ex b/lib/bob_web/live/request_live.ex index 641960dd..032e5e3e 100644 --- a/lib/bob_web/live/request_live.ex +++ b/lib/bob_web/live/request_live.ex @@ -134,7 +134,11 @@ defmodule BobWeb.RequestLive do |> replace_flash(:info, "Build queued, follow the progress on the jobs dashboard.") {:ok, :already_built} -> - replace_flash(socket, :info, "This image is already built, find it under Docker tags.") + replace_flash( + socket, + :info, + "This image is already built and is now reserved from cleanup, find it under Docker tags." + ) {:error, :rate_limited} -> replace_flash( @@ -271,6 +275,12 @@ defmodule BobWeb.RequestLive do Erlang and Elixir version. Need an older patch version? Request it here and it is built for both architectures, including the multi-arch manifest.

+

+ Requesting an image also reserves it from cleanup, permanently — and if the image + already exists it is reserved without rebuilding. Tags that are not reserved are + removed over time: multi-arch images that have not been pulled in 180 days, and the + per-architecture build images after 30 days. +

<.filter_select diff --git a/priv/repo/migrations/20260616120000_add_last_pulled_to_docker_tags.exs b/priv/repo/migrations/20260616120000_add_last_pulled_to_docker_tags.exs new file mode 100644 index 00000000..20ed588e --- /dev/null +++ b/priv/repo/migrations/20260616120000_add_last_pulled_to_docker_tags.exs @@ -0,0 +1,13 @@ +defmodule Bob.Repo.Migrations.AddLastPulledToDockerTags do + use Ecto.Migration + + def change do + alter table(:docker_tags) do + add(:last_pulled, :utc_datetime_usec) + end + + alter table(:docker_tags_staging) do + add(:last_pulled, :utc_datetime_usec) + end + end +end diff --git a/priv/repo/migrations/20260616130000_index_for_docker_cleanup.exs b/priv/repo/migrations/20260616130000_index_for_docker_cleanup.exs new file mode 100644 index 00000000..8c09164e --- /dev/null +++ b/priv/repo/migrations/20260616130000_index_for_docker_cleanup.exs @@ -0,0 +1,27 @@ +defmodule Bob.Repo.Migrations.IndexForDockerCleanup do + use Ecto.Migration + + @disable_ddl_transaction true + @disable_migration_lock true + + def up do + # Per-arch cleanup scans a repo's tags by built_at; manifest cleanup by + # last_pulled. Both are scoped to a handful of repos, so lead with repo. + # Reservations are matched by a hash anti-join against the small + # build_requests table, so no index is needed there. + execute(""" + CREATE INDEX CONCURRENTLY IF NOT EXISTS docker_tags_repo_built_at_index + ON docker_tags (repo, built_at) + """) + + execute(""" + CREATE INDEX CONCURRENTLY IF NOT EXISTS docker_tags_repo_last_pulled_index + ON docker_tags (repo, last_pulled) + """) + end + + def down do + execute("DROP INDEX CONCURRENTLY IF EXISTS docker_tags_repo_built_at_index") + execute("DROP INDEX CONCURRENTLY IF EXISTS docker_tags_repo_last_pulled_index") + end +end diff --git a/test/bob/artifacts_test.exs b/test/bob/artifacts_test.exs index 62e0f43c..a14fe63d 100644 --- a/test/bob/artifacts_test.exs +++ b/test/bob/artifacts_test.exs @@ -3,9 +3,11 @@ defmodule Bob.ArtifactsTest do alias Bob.Artifacts alias Bob.Artifacts.{Artifact, BaseImageTag, DockerTag, DockerTagStaging} + alias Bob.BuildRequests @docker_built_at ~U[2025-01-02 03:04:05.000000Z] @newer_docker_built_at ~U[2025-02-03 04:05:06.000000Z] + @docker_last_pulled ~U[2026-03-04 05:06:07.000000Z] describe "Artifact.changeset/2" do test "casts a posted artifact, parsing the ISO8601 date" do @@ -216,6 +218,22 @@ defmodule Bob.ArtifactsTest do assert [%DockerTag{built_at: @docker_built_at}] = Repo.all(DockerTag) end + test "stores last_pulled and preserves it across a rebuild that reports none" do + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "27.0-ubuntu-noble-20250101", + ["amd64"], + @docker_built_at, + @docker_last_pulled + ) + + assert [%DockerTag{last_pulled: @docker_last_pulled}] = Repo.all(DockerTag) + + Artifacts.add_docker_tag("hexpm/erlang-amd64", "27.0-ubuntu-noble-20250101", ["amd64"]) + + assert [%DockerTag{last_pulled: @docker_last_pulled}] = Repo.all(DockerTag) + end + test "unions archs on conflicting (repo, tag)" do Artifacts.add_docker_tag("hexpm/erlang", "27.0-ubuntu-noble-20250101", ["amd64"]) Artifacts.add_docker_tag("hexpm/erlang", "27.0-ubuntu-noble-20250101", ["arm64"]) @@ -284,6 +302,41 @@ defmodule Bob.ArtifactsTest do assert [%DockerTag{built_at: @docker_built_at}] = Repo.all(DockerTag) end + test "swaps the staged last_pulled into the row" do + replace("hexpm/erlang-amd64", [ + docker_tag("27.0-ubuntu-noble-20250101", ["amd64"], @docker_built_at, @docker_last_pulled) + ]) + + assert [%DockerTag{last_pulled: @docker_last_pulled}] = Repo.all(DockerTag) + end + + test "updates last_pulled for otherwise unchanged rows" do + replace("hexpm/erlang-amd64", [ + docker_tag("27.0-ubuntu-noble-20250101", ["amd64"], @docker_built_at) + ]) + + replace("hexpm/erlang-amd64", [ + docker_tag("27.0-ubuntu-noble-20250101", ["amd64"], @docker_built_at, @docker_last_pulled) + ]) + + assert [%DockerTag{last_pulled: @docker_last_pulled}] = Repo.all(DockerTag) + end + + test "keeps a known last_pulled when the staged value is missing" do + replace("hexpm/erlang", [ + docker_tag("27.0-ubuntu-noble-20250101", ["amd64"], @docker_built_at, @docker_last_pulled) + ]) + + # Docker Hub transiently reports no pull time (null or the zero sentinel): + # the recorded pull time must survive the swap, or an old-but-pulled + # manifest tag would fall back to built_at and become a cleanup candidate. + replace("hexpm/erlang", [ + docker_tag("27.0-ubuntu-noble-20250101", ["amd64"], @docker_built_at) + ]) + + assert [%DockerTag{last_pulled: @docker_last_pulled}] = Repo.all(DockerTag) + end + test "updates built_at for otherwise unchanged rows" do Artifacts.add_docker_tag( "hexpm/erlang-amd64", @@ -904,17 +957,235 @@ defmodule Bob.ArtifactsTest do end end + describe "docker cleanup candidates" do + test "per-arch candidates are tags built before the cutoff, excluding manifest repos" do + old = days_ago(40) + recent = days_ago(10) + cutoff = days_ago(30) + + Artifacts.add_docker_tag("hexpm/erlang-amd64", "27.0-ubuntu-noble-20250101", ["amd64"], old) + + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "28.0-ubuntu-noble-20250101", + ["amd64"], + recent + ) + + Artifacts.add_docker_tag( + "hexpm/erlang", + "26.0-ubuntu-noble-20250101", + ["amd64", "arm64"], + old + ) + + assert Artifacts.count_stale_per_arch_tags(cutoff) == %{"hexpm/erlang-amd64" => 1} + + assert Artifacts.stale_per_arch_tags(cutoff, 100) == + [{"hexpm/erlang-amd64", "27.0-ubuntu-noble-20250101"}] + end + + test "manifest candidates are tags neither pulled nor built since the cutoff" do + old = days_ago(250) + recent = days_ago(10) + cutoff = days_ago(180) + + # old build, old last_pulled -> stale + Artifacts.add_docker_tag( + "hexpm/erlang", + "27.0-ubuntu-noble-20250101", + ["amd64"], + old, + old + ) + + # recent last_pulled despite old build -> kept + Artifacts.add_docker_tag( + "hexpm/erlang", + "26.0-ubuntu-noble-20250101", + ["amd64"], + old, + recent + ) + + # rebuilt recently despite an old last_pulled -> kept; deleting it would + # 404 pullers only for the checker to rebuild it from the per-arch tags + Artifacts.add_docker_tag( + "hexpm/erlang", + "25.0-ubuntu-noble-20250101", + ["amd64"], + recent, + old + ) + + # never pulled, old build -> stale via fallback + Artifacts.add_docker_tag( + "hexpm/elixir", + "1.18.0-erlang-27.0-ubuntu-noble-20250101", + ["amd64"], + old + ) + + # never pulled, recent build -> kept via fallback + Artifacts.add_docker_tag( + "hexpm/elixir", + "1.18.1-erlang-27.0-ubuntu-noble-20250101", + ["amd64"], + recent + ) + + assert Artifacts.count_stale_manifest_tags(cutoff) == + %{"hexpm/erlang" => 1, "hexpm/elixir" => 1} + end + + test "a non-expired build request reserves its per-arch and manifest tags" do + old = days_ago(250) + + Artifacts.add_docker_tag( + "hexpm/elixir-amd64", + "1.18.0-erlang-27.0-ubuntu-noble-20250101", + ["amd64"], + old + ) + + Artifacts.add_docker_tag( + "hexpm/elixir", + "1.18.0-erlang-27.0-ubuntu-noble-20250101", + ["amd64"], + old, + old + ) + + BuildRequests.create(%{ + username: "eric", + kind: "elixir", + elixir: "1.18.0", + erlang: "27.0", + os: "ubuntu", + os_version: "noble-20250101", + builds_count: 0 + }) + + assert Artifacts.count_stale_per_arch_tags(days_ago(30)) == %{} + assert Artifacts.count_stale_manifest_tags(days_ago(180)) == %{} + end + + test "an expired build request does not reserve its tags" do + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "27.0-ubuntu-noble-20250101", + ["amd64"], + days_ago(40) + ) + + {:ok, request} = + BuildRequests.create(%{ + username: "eric", + kind: "erlang", + erlang: "27.0", + os: "ubuntu", + os_version: "noble-20250101", + builds_count: 0 + }) + + BuildRequests.expire(request) + + assert Artifacts.count_stale_per_arch_tags(days_ago(30)) == %{"hexpm/erlang-amd64" => 1} + end + + test "the SQL reservation predicate matches the tag name request_target/1 derives" do + erlang_attrs = %{ + username: "eric", + kind: "erlang", + erlang: "27.0", + os: "ubuntu", + os_version: "noble-20250101", + builds_count: 0 + } + + elixir_attrs = %{erlang_attrs | kind: "elixir"} |> Map.put(:elixir, "1.18.0") + + # The reservation predicate rebuilds the request->tag mapping in SQL; if + # either encoding drifts, reservations silently stop matching and the + # cleanup deletes tags users pinned. Derive the tag through the Elixir + # mapping and assert the SQL side agrees. + for {repo, attrs} <- [{"hexpm/erlang", erlang_attrs}, {"hexpm/elixir", elixir_attrs}] do + {:ok, request} = BuildRequests.create(attrs) + tag = BuildRequests.request_target(request) + + Artifacts.add_docker_tag(repo, tag, ["amd64"]) + assert Artifacts.docker_tag_reserved?(repo, tag) + end + end + + test "delete_docker_tags removes the given rows" do + Artifacts.add_docker_tag("hexpm/erlang-amd64", "a", ["amd64"]) + Artifacts.add_docker_tag("hexpm/erlang-amd64", "b", ["amd64"]) + + assert Artifacts.delete_docker_tags([{"hexpm/erlang-amd64", "a"}]) == :ok + assert Artifacts.docker_tags("hexpm/erlang-amd64") == [{"b", ["amd64"]}] + end + + test "reserved_docker_tag_ids returns the ids matched by a build request, across repos" do + Artifacts.add_docker_tag("hexpm/elixir", "1.18.0-erlang-27.0-ubuntu-noble-20250101", [ + "amd64", + "arm64" + ]) + + Artifacts.add_docker_tag("hexpm/elixir-amd64", "1.18.0-erlang-27.0-ubuntu-noble-20250101", [ + "amd64" + ]) + + Artifacts.add_docker_tag("hexpm/elixir", "1.17.0-erlang-27.0-ubuntu-noble-20250101", [ + "amd64", + "arm64" + ]) + + BuildRequests.create(%{ + username: "eric", + kind: "elixir", + elixir: "1.18.0", + erlang: "27.0", + os: "ubuntu", + os_version: "noble-20250101", + builds_count: 0 + }) + + tags = Repo.all(DockerTag) + reserved = Artifacts.reserved_docker_tag_ids(tags) + + reserved_tags = + tags + |> Enum.filter(&MapSet.member?(reserved, &1.id)) + |> Enum.map(&{&1.repo, &1.tag}) + |> Enum.sort() + + assert reserved_tags == [ + {"hexpm/elixir", "1.18.0-erlang-27.0-ubuntu-noble-20250101"}, + {"hexpm/elixir-amd64", "1.18.0-erlang-27.0-ubuntu-noble-20250101"} + ] + + assert Artifacts.reserved_docker_tag_ids([]) == MapSet.new() + end + end + + defp days_ago(days), do: DateTime.add(DateTime.utc_now(), -days, :day) + defp replace(repo, tag_archs) do token = Ecto.UUID.generate() Artifacts.stage_docker_tags(token, repo, Enum.map(tag_archs, &docker_tag_tuple/1)) Artifacts.swap_docker_tags(token, repo) end - defp docker_tag(tag, archs, built_at \\ @docker_built_at), do: {tag, archs, built_at} + defp docker_tag(tag, archs, built_at \\ @docker_built_at, last_pulled \\ nil), + do: {tag, archs, built_at, last_pulled} defp docker_tag_tuple({tag, archs}), do: docker_tag(tag, archs) defp docker_tag_tuple({tag, archs, built_at}), do: docker_tag(tag, archs, built_at) + defp docker_tag_tuple({tag, archs, built_at, last_pulled}), + do: docker_tag(tag, archs, built_at, last_pulled) + defp attrs() do %{ kind: "otp", diff --git a/test/bob/build_requests_test.exs b/test/bob/build_requests_test.exs index 503fab5e..1d411ad2 100644 --- a/test/bob/build_requests_test.exs +++ b/test/bob/build_requests_test.exs @@ -47,14 +47,40 @@ defmodule Bob.BuildRequestsTest do assert [%Job{module_key: {Bob.Job.BuildDockerErlang, "arm64"}}] = Repo.all(Job) end - test "reports already built tags without recording a request" do + test "records a completed reservation for an already-built image, no build" do Artifacts.add_docker_tag("hexpm/erlang-amd64", "27.0-ubuntu-noble-20250101", ["amd64"]) Artifacts.add_docker_tag("hexpm/erlang-arm64", "27.0-ubuntu-noble-20250101", ["arm64"]) Artifacts.add_docker_tag("hexpm/erlang", "27.0-ubuntu-noble-20250101", ["amd64", "arm64"]) assert BuildRequests.submit(@erlang_attrs) == {:ok, :already_built} assert Repo.all(Job) == [] - assert Repo.all(BuildRequest) == [] + + assert [%BuildRequest{state: "completed", builds_count: 0, erlang: "27.0"}] = + Repo.all(BuildRequest) + end + + test "reserves without rebuilding when only the staging tags were cleaned" do + # Per-arch staging tags are removed 30 days after build while the + # manifest lives on. Requesting the surviving image reserves it for + # free — the whole point of the request — rather than forcing a full + # rebuild of staging tags nobody pulls. + Artifacts.add_docker_tag("hexpm/erlang", "27.0-ubuntu-noble-20250101", ["amd64", "arm64"]) + + assert BuildRequests.submit(@erlang_attrs) == {:ok, :already_built} + assert Repo.all(Job) == [] + + assert [%BuildRequest{state: "completed", builds_count: 0}] = Repo.all(BuildRequest) + end + + test "does not duplicate a reservation for the same already-built image" do + Artifacts.add_docker_tag("hexpm/erlang-amd64", "27.0-ubuntu-noble-20250101", ["amd64"]) + Artifacts.add_docker_tag("hexpm/erlang-arm64", "27.0-ubuntu-noble-20250101", ["arm64"]) + Artifacts.add_docker_tag("hexpm/erlang", "27.0-ubuntu-noble-20250101", ["amd64", "arm64"]) + + assert BuildRequests.submit(@erlang_attrs) == {:ok, :already_built} + assert BuildRequests.submit(@erlang_attrs) == {:ok, :already_built} + + assert [%BuildRequest{}] = Repo.all(BuildRequest) end test "enqueues only the manifest when the arch images already exist" do @@ -235,16 +261,16 @@ defmodule Bob.BuildRequestsTest do end describe "prune/1" do - test "deletes finished requests older than the cutoff and keeps the rest" do - _old_completed = insert_aged(%{state: "completed"}, -100) + test "deletes only expired requests older than the cutoff, keeping reservations" do + old_completed = insert_aged(%{state: "completed"}, -100) _old_expired = insert_aged(%{state: "expired"}, -100) old_pending = insert_aged(%{state: "pending"}, -100) - recent_completed = insert_aged(%{state: "completed"}, -1) + recent_expired = insert_aged(%{state: "expired"}, -1) - assert BuildRequests.prune(90 * 24 * 60 * 60) == 2 + assert BuildRequests.prune(90 * 24 * 60 * 60) == 1 ids = Repo.all(BuildRequest) |> Enum.map(& &1.id) |> Enum.sort() - assert ids == Enum.sort([old_pending.id, recent_completed.id]) + assert ids == Enum.sort([old_completed.id, old_pending.id, recent_expired.id]) end end diff --git a/test/bob/docker_cleanup_test.exs b/test/bob/docker_cleanup_test.exs new file mode 100644 index 00000000..0c6f32eb --- /dev/null +++ b/test/bob/docker_cleanup_test.exs @@ -0,0 +1,235 @@ +defmodule Bob.DockerCleanupTest do + use Bob.DataCase + + alias Bob.{Artifacts, BuildRequests, DockerCleanup} + + defp days_ago(days), do: DateTime.add(DateTime.utc_now(), -days, :day) + defp old(), do: days_ago(40) + defp ancient(), do: days_ago(250) + defp recent(), do: days_ago(10) + + describe "removal_at/3" do + test "per-arch tags expire 30 days after build, ignoring last_pulled" do + built = ~U[2026-01-01 00:00:00Z] + + assert DockerCleanup.removal_at("hexpm/erlang-amd64", built, nil) == + DateTime.add(built, 30, :day) + + assert DockerCleanup.removal_at("hexpm/elixir-arm64", built, ~U[2026-06-01 00:00:00Z]) == + DateTime.add(built, 30, :day) + end + + test "manifest tags expire 180 days after last pull, falling back to build" do + built = ~U[2026-01-01 00:00:00Z] + pulled = ~U[2026-03-01 00:00:00Z] + + assert DockerCleanup.removal_at("hexpm/erlang", built, pulled) == + DateTime.add(pulled, 180, :day) + + assert DockerCleanup.removal_at("hexpm/elixir", built, nil) == + DateTime.add(built, 180, :day) + end + + test "manifest tags rebuilt after their last pull expire from the rebuild" do + built = ~U[2026-05-01 00:00:00Z] + pulled = ~U[2026-01-01 00:00:00Z] + + assert DockerCleanup.removal_at("hexpm/erlang", built, pulled) == + DateTime.add(built, 180, :day) + end + + test "returns nil for a repo not under cleanup" do + assert DockerCleanup.removal_at("library/alpine", ~U[2026-01-01 00:00:00Z], nil) == nil + end + end + + describe "run/1 in dry-run mode" do + test "returns the counts a live run would delete, deleting nothing" do + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "26.0-ubuntu-noble-20250101", + ["amd64"], + old() + ) + + Artifacts.add_docker_tag( + "hexpm/erlang", + "25.0-ubuntu-noble-20250101", + ["amd64", "arm64"], + ancient(), + ancient() + ) + + assert {:dry_run, %{per_arch: per_arch, manifest: manifest}} = + DockerCleanup.run(mode: :dry_run) + + assert per_arch == %{"hexpm/erlang-amd64" => 1} + assert manifest == %{"hexpm/erlang" => 1} + + assert Artifacts.docker_tags("hexpm/erlang-amd64") == + [{"26.0-ubuntu-noble-20250101", ["amd64"]}] + end + end + + describe "run/1 in live mode" do + test "deletes stale tags via the deleter and drops their rows, keeping recent and reserved" do + test = self() + + deleter = fn repo, tag -> + send(test, {:deleted, repo, tag}) + :ok + end + + # stale, not reserved -> deleted + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "26.0-ubuntu-noble-20250101", + ["amd64"], + old() + ) + + # recent -> kept + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "28.0-ubuntu-noble-20250101", + ["amd64"], + recent() + ) + + # stale but reserved -> kept + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "27.0-ubuntu-noble-20250101", + ["amd64"], + old() + ) + + BuildRequests.create(%{ + username: "eric", + kind: "erlang", + erlang: "27.0", + os: "ubuntu", + os_version: "noble-20250101", + builds_count: 0 + }) + + # stale manifest -> deleted + Artifacts.add_docker_tag( + "hexpm/erlang", + "25.0-ubuntu-noble-20250101", + ["amd64", "arm64"], + ancient(), + ancient() + ) + + assert {:live, 2} = DockerCleanup.run(mode: :live, deleter: deleter) + + assert_received {:deleted, "hexpm/erlang-amd64", "26.0-ubuntu-noble-20250101"} + assert_received {:deleted, "hexpm/erlang", "25.0-ubuntu-noble-20250101"} + refute_received {:deleted, "hexpm/erlang-amd64", "27.0-ubuntu-noble-20250101"} + refute_received {:deleted, "hexpm/erlang-amd64", "28.0-ubuntu-noble-20250101"} + + assert Enum.sort(Artifacts.docker_tags("hexpm/erlang-amd64")) == + Enum.sort([ + {"27.0-ubuntu-noble-20250101", ["amd64"]}, + {"28.0-ubuntu-noble-20250101", ["amd64"]} + ]) + + assert Artifacts.docker_tags("hexpm/erlang") == [] + end + + test "skips a tag that was reserved after the candidates were selected" do + test = self() + + # The rate-limited batch can run for hours after the candidate list is + # built. Reserving every candidate while the first is deleted simulates a + # user requesting an image mid-run: the remaining tag must survive. + deleter = fn repo, tag -> + send(test, {:deleted, repo, tag}) + + for erlang <- ["26.0", "27.0"] do + BuildRequests.create(%{ + username: "eric", + kind: "erlang", + erlang: erlang, + os: "ubuntu", + os_version: "noble-20250101", + builds_count: 0 + }) + end + + :ok + end + + for erlang <- ["26.0", "27.0"] do + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "#{erlang}-ubuntu-noble-20250101", + ["amd64"], + old() + ) + end + + assert {:live, 1} = DockerCleanup.run(mode: :live, deleter: deleter) + + assert_received {:deleted, "hexpm/erlang-amd64", _tag} + refute_received {:deleted, _repo, _tag} + assert length(Artifacts.docker_tags("hexpm/erlang-amd64")) == 1 + end + + test "keeps the row when deletion errors" do + deleter = fn _repo, _tag -> {:error, :boom} end + + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "26.0-ubuntu-noble-20250101", + ["amd64"], + old() + ) + + assert {:live, 0} = DockerCleanup.run(mode: :live, deleter: deleter) + + assert Artifacts.docker_tags("hexpm/erlang-amd64") == + [{"26.0-ubuntu-noble-20250101", ["amd64"]}] + end + + test "respects the batch limit so a run deletes a bounded slice" do + deleter = fn _repo, _tag -> :ok end + + for n <- 1..3 do + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "2#{n}.0-ubuntu-noble-20250101", + ["amd64"], + old() + ) + end + + assert {:live, 2} = DockerCleanup.run(mode: :live, deleter: deleter, limit: 2) + assert length(Artifacts.docker_tags("hexpm/erlang-amd64")) == 1 + end + + test "the batch limit bounds the whole run across both repo groups" do + deleter = fn _repo, _tag -> :ok end + + for n <- 1..2 do + Artifacts.add_docker_tag( + "hexpm/erlang-amd64", + "2#{n}.0-ubuntu-noble-20250101", + ["amd64"], + old() + ) + + Artifacts.add_docker_tag( + "hexpm/erlang", + "2#{n}.1-ubuntu-noble-20250101", + ["amd64", "arm64"], + ancient(), + ancient() + ) + end + + assert {:live, 3} = DockerCleanup.run(mode: :live, deleter: deleter, limit: 3) + end + end +end diff --git a/test/bob/docker_hub/rate_limiter_test.exs b/test/bob/docker_hub/rate_limiter_test.exs index 29141138..afc5f169 100644 --- a/test/bob/docker_hub/rate_limiter_test.exs +++ b/test/bob/docker_hub/rate_limiter_test.exs @@ -133,6 +133,43 @@ defmodule Bob.DockerHub.RateLimiterTest do assert RateLimiter.acquire(limiter) == :ok end + test "reclaims the probe slot when the probe reports a response without rate headers" do + limiter = start_limiter(offset_ms: 0) + + assert RateLimiter.acquire(limiter) == :ok + + # The probe's request completed but carried no usable rate headers (a 401, + # a 5xx, or a transport error). Its slot is reclaimed even though the + # probe process lives on, so a long-lived caller can't wedge the gate. + RateLimiter.observe([], limiter) + + refute blocks?(limiter) + end + + test "ignores a headerless response from a caller that is not the probe" do + limiter = start_limiter(offset_ms: 0) + test_pid = self() + + probe = + spawn_link(fn -> + RateLimiter.acquire(limiter) + send(test_pid, :granted) + + receive do + :stop -> :ok + end + end) + + assert_receive :granted + + # A straggler that never held the probe slot reports a header-less + # response: the in-flight probe's slot must not be reclaimed on its behalf. + RateLimiter.observe([], limiter) + assert blocks?(limiter) + + send(probe, :stop) + end + test "keeps the probe's slot when it anchored the window before exiting" do limiter = start_limiter(offset_ms: 0) reset = System.os_time(:second) + 3600 diff --git a/test/bob/docker_hub_test.exs b/test/bob/docker_hub_test.exs index 56af27ec..203880a7 100644 --- a/test/bob/docker_hub_test.exs +++ b/test/bob/docker_hub_test.exs @@ -5,18 +5,20 @@ defmodule Bob.DockerHubTest do @built_at ~U[2025-01-02 03:04:05.123456Z] @image_pushed_at ~U[2025-02-03 04:05:06.000000Z] + @last_pulled ~U[2026-06-16 13:17:58.638032Z] describe "parse/1" do - test "returns tag, archs, and Docker Hub last_updated timestamp" do + test "returns tag, archs, built_at, and tag_last_pulled" do assert DockerHub.parse( tag_payload(%{ "last_updated" => "2025-01-02T03:04:05.123456Z", + "tag_last_pulled" => "2026-06-16T13:17:58.638032692Z", "images" => [ image("amd64", "sha256:amd64", "2025-02-03T04:05:06Z"), image("arm64", "sha256:arm64", "2025-02-03T04:05:06Z") ] }) - ) == {"27.0", ["amd64", "arm64"], @built_at} + ) == {"27.0", ["amd64", "arm64"], @built_at, @last_pulled} end test "falls back to image last_pushed when the tag timestamp is absent" do @@ -27,7 +29,23 @@ defmodule Bob.DockerHubTest do image("arm64", "sha256:arm64", "2025-02-03T04:05:06Z") ] }) - ) == {"27.0", ["amd64", "arm64"], @image_pushed_at} + ) == {"27.0", ["amd64", "arm64"], @image_pushed_at, nil} + end + + test "leaves last_pulled nil for a tag that has never been pulled" do + for never_pulled <- [nil, "", "0001-01-01T00:00:00Z"] do + assert {"27.0", ["amd64", "arm64"], @built_at, nil} = + DockerHub.parse( + tag_payload(%{ + "last_updated" => "2025-01-02T03:04:05.123456Z", + "tag_last_pulled" => never_pulled, + "images" => [ + image("amd64", "sha256:amd64", "2025-02-03T04:05:06Z"), + image("arm64", "sha256:arm64", "2025-02-03T04:05:06Z") + ] + }) + ) + end end test "rejects images without a digest" do diff --git a/test/bob/job/docker_checker_test.exs b/test/bob/job/docker_checker_test.exs index 77c17f29..a80e18e8 100644 --- a/test/bob/job/docker_checker_test.exs +++ b/test/bob/job/docker_checker_test.exs @@ -447,6 +447,43 @@ defmodule Bob.Job.DockerCheckerTest do assert Repo.reload!(request).state == "expired" end + test "completes a fully built request even when its os_version rotated out" do + request = insert_request(kind: "erlang", erlang: "27.0", os_version: "noble-20240101") + Artifacts.add_docker_tag("hexpm/erlang-amd64", "27.0-ubuntu-noble-20240101", ["amd64"]) + Artifacts.add_docker_tag("hexpm/erlang-arm64", "27.0-ubuntu-noble-20240101", ["arm64"]) + + Artifacts.add_docker_tag("hexpm/erlang", "27.0-ubuntu-noble-20240101", [ + "amd64", + "arm64" + ]) + + DockerChecker.requests() + + # The request backs a permanent reservation; expiring it would lift the + # reservation and expose the built image to cleanup. + assert Repo.all(Job) == [] + assert Repo.reload!(request).state == "completed" + end + + test "completes a fully built request even past the ttl" do + request = insert_request(kind: "erlang", erlang: "27.0") + Artifacts.add_docker_tag("hexpm/erlang-amd64", "27.0-ubuntu-noble-20250101", ["amd64"]) + Artifacts.add_docker_tag("hexpm/erlang-arm64", "27.0-ubuntu-noble-20250101", ["arm64"]) + + Artifacts.add_docker_tag("hexpm/erlang", "27.0-ubuntu-noble-20250101", [ + "amd64", + "arm64" + ]) + + request + |> Ecto.Changeset.change(inserted_at: DateTime.add(DateTime.utc_now(), -15, :day)) + |> Repo.update!() + + DockerChecker.requests() + + assert Repo.reload!(request).state == "completed" + end + test "expires requests that never completed within the ttl" do request = insert_request(kind: "erlang", erlang: "27.0") diff --git a/test/bob/queue/maintenance_test.exs b/test/bob/queue/maintenance_test.exs index 1941f6d9..58155466 100644 --- a/test/bob/queue/maintenance_test.exs +++ b/test/bob/queue/maintenance_test.exs @@ -132,16 +132,16 @@ defmodule Bob.Queue.MaintenanceTest do |> Repo.update!() end - test "prunes finished build requests past the retention window" do - insert_build_request("completed", ago(91 * @day)) + test "prunes expired build requests past the retention window" do + insert_build_request("expired", ago(91 * @day)) Maintenance.run() assert Repo.all(BuildRequest) == [] end - test "keeps recent and pending build requests" do - insert_build_request("completed", DateTime.utc_now()) + test "keeps completed reservations and pending build requests regardless of age" do + insert_build_request("completed", ago(91 * @day)) insert_build_request("pending", ago(91 * @day)) Maintenance.run() diff --git a/test/bob/reconcile_test.exs b/test/bob/reconcile_test.exs index 1cc4a076..bb6e79a2 100644 --- a/test/bob/reconcile_test.exs +++ b/test/bob/reconcile_test.exs @@ -5,9 +5,10 @@ defmodule Bob.ReconcileTest do @docker_built_at ~U[2025-01-02 03:04:05.000000Z] @newer_docker_built_at ~U[2025-02-03 04:05:06.000000Z] + @docker_last_pulled ~U[2026-03-04 05:06:07.000000Z] - # Streamer stub: invokes on_page once with the canned {tag, archs, built_at} list for the - # repo (mimicking a single Docker Hub page), or nothing for an empty repo. + # Streamer stub: invokes on_page once with the canned {tag, archs, built_at, last_pulled} list + # for the repo (mimicking a single Docker Hub page), or nothing for an empty repo. defp streamer(map) do fn repo, on_page -> case Map.get(map, repo, []) do @@ -24,7 +25,12 @@ defmodule Bob.ReconcileTest do stream = streamer(%{ "hexpm/erlang-amd64" => [ - docker_tag("27.0-ubuntu-noble-20250101", ["amd64", "arm64"], @docker_built_at) + docker_tag( + "27.0-ubuntu-noble-20250101", + ["amd64", "arm64"], + @docker_built_at, + @docker_last_pulled + ) ], "hexpm/elixir-arm64" => [ docker_tag( @@ -43,7 +49,10 @@ defmodule Bob.ReconcileTest do assert Artifacts.docker_tags("hexpm/elixir-arm64") == [{"1.18.0-erlang-27.0-ubuntu-noble-20250101", ["arm64"]}] - assert %Bob.Artifacts.DockerTag{built_at: @docker_built_at} = + assert %Bob.Artifacts.DockerTag{ + built_at: @docker_built_at, + last_pulled: @docker_last_pulled + } = Repo.get_by!(Bob.Artifacts.DockerTag, repo: "hexpm/erlang-amd64", tag: "27.0-ubuntu-noble-20250101" @@ -249,5 +258,6 @@ defmodule Bob.ReconcileTest do end end - defp docker_tag(tag, archs, built_at \\ @docker_built_at), do: {tag, archs, built_at} + defp docker_tag(tag, archs, built_at \\ @docker_built_at, last_pulled \\ nil), + do: {tag, archs, built_at, last_pulled} end diff --git a/test/bob_web/live/docker_tags_live_test.exs b/test/bob_web/live/docker_tags_live_test.exs index a7b9aab3..9379f34b 100644 --- a/test/bob_web/live/docker_tags_live_test.exs +++ b/test/bob_web/live/docker_tags_live_test.exs @@ -52,6 +52,75 @@ defmodule BobWeb.DockerTagsLiveTest do refute docker_tag?(html, "27.0-ubuntu-noble-20250101") end + test "shows last pulled and flags tags nearing removal", %{conn: conn} do + now = DateTime.utc_now() + + # per-arch built long ago -> removing (age-based) + Bob.Artifacts.add_docker_tag( + "hexpm/elixir-amd64", + "1.10.0-erlang-23.0-ubuntu-noble-20200101", + ["amd64"], + DateTime.add(now, -90, :day) + ) + + # manifest pulled recently despite an old build -> kept + Bob.Artifacts.add_docker_tag( + "hexpm/erlang", + "26.0-ubuntu-noble-20250101", + ["amd64", "arm64"], + DateTime.add(now, -300, :day), + DateTime.add(now, -1, :day) + ) + + {:ok, _view, html} = live(conn, ~p"/docker") + + assert html =~ "last pulled" + assert removing?(html, "1.10.0-erlang-23.0-ubuntu-noble-20200101") + refute removing?(html, "26.0-ubuntu-noble-20250101") + end + + test "a reserved tag is not also flagged as removing", %{conn: conn} do + Bob.Artifacts.add_docker_tag( + "hexpm/elixir-amd64", + "1.18.0-erlang-27.0-ubuntu-noble-20250101", + ["amd64"], + DateTime.add(DateTime.utc_now(), -90, :day) + ) + + Bob.BuildRequests.create(%{ + username: "eric", + kind: "elixir", + elixir: "1.18.0", + erlang: "27.0", + os: "ubuntu", + os_version: "noble-20250101", + builds_count: 0 + }) + + {:ok, _view, html} = live(conn, ~p"/docker") + + assert reserved?(html, "1.18.0-erlang-27.0-ubuntu-noble-20250101") + refute removing?(html, "1.18.0-erlang-27.0-ubuntu-noble-20250101") + end + + test "flags tags reserved by a build request", %{conn: conn} do + Bob.BuildRequests.create(%{ + username: "eric", + kind: "elixir", + elixir: "1.18.0", + erlang: "27.0", + os: "ubuntu", + os_version: "noble-20250101", + builds_count: 0 + }) + + {:ok, _view, html} = live(conn, ~p"/docker") + + assert reserved?(html, "1.18.0-erlang-27.0-ubuntu-noble-20250101") + refute reserved?(html, "1.18.1-erlang-27.0-ubuntu-noble-20250101") + refute reserved?(html, "27.0-ubuntu-noble-20250101") + end + test "applies filters from URL params on load", %{conn: conn} do {:ok, _view, html} = live(conn, ~p"/docker?tag=1.18") @@ -161,7 +230,16 @@ defmodule BobWeb.DockerTagsLiveTest do end defp docker_tag?(html, tag) do + html =~ ~r/]*>#{Regex.escape(tag)}<\/code>/ + end + + defp reserved?(html, tag) do + html =~ + ~r/]*>#{Regex.escape(tag)}<\/code>\s*]*dk-reserved/ + end + + defp removing?(html, tag) do html =~ - ~r/]*>#{Regex.escape(tag)}<\/code><\/td>/ + ~r/]*>#{Regex.escape(tag)}<\/code>\s*]*dk-removing/ end end diff --git a/test/bob_web/live/request_live_test.exs b/test/bob_web/live/request_live_test.exs index ba9d2f00..8f49ae96 100644 --- a/test/bob_web/live/request_live_test.exs +++ b/test/bob_web/live/request_live_test.exs @@ -193,7 +193,8 @@ defmodule BobWeb.RequestLiveTest do }) assert html =~ "already built" - assert Repo.all(BuildRequest) == [] + assert html =~ "reserved" + assert [%BuildRequest{state: "completed", builds_count: 0}] = Repo.all(BuildRequest) assert Repo.all(Job) == [] end