Docker tag cleanup

README.md

@@ -66,3 +66,14 @@ All provided images and tags can be searched at https://bob.hex.pm/docker.
 If you need a specific patch version that is not built automatically, request it at https://bob.hex.pm/request. Sign in with your hex.pm account, then pick the image kind (Erlang or Elixir), the OS and base image version, and the exact Erlang (and Elixir) version. The image is built for both architectures, including the multi-arch manifest; when an Elixir image needs an Erlang base image that does not exist yet, the base is built first and the Elixir image follows automatically.
 
 Requested builds are limited to ten image builds per user per hour. Build progress can be followed on the jobs dashboard at https://bob.hex.pm.
+
+Requesting an image also reserves it from cleanup, permanently. If the image already exists it is reserved without rebuilding, so requesting a build doubles as a way to keep an old image you depend on from being removed (see Retention and cleanup below).
+
+### Retention and cleanup
+
+Old tags are removed automatically so the repositories don't grow without bound:
+
+* The `hexpm/erlang` and `hexpm/elixir` repositories hold the multi-arch images you pull. A tag that has been neither pulled nor rebuilt for 180 days is removed.
+* The per-architecture repositories (`hexpm/erlang-amd64`, `hexpm/erlang-arm64`, `hexpm/elixir-amd64`, `hexpm/elixir-arm64`) only hold the single-arch images the multi-arch manifests are assembled from. They keep the last 30 days of builds; older single-arch tags are removed even though the multi-arch image they back stays available.
+
+Tags reserved by a build request are never removed and are flagged as `reserved` on the [Docker tags page](https://bob.hex.pm/docker). To keep an image indefinitely, request it at https://bob.hex.pm/request.

assets/css/app.css

@@ -977,7 +977,7 @@ pre,
 }
 
 .jt--dk {
-  min-width: 1040px;
+  min-width: 1180px;
   table-layout: fixed;
 }
 
@@ -986,7 +986,7 @@ pre,
 }
 
 .col-dk-tag {
-  width: 510px;
+  width: 430px;
 }
 
 .col-dk-archs {
@@ -1004,6 +1004,41 @@ pre,
   white-space: nowrap;
 }
 
+.dk-tag-cell {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+
+.dk-tag-cell .dk-tag-code {
+  flex: 1;
+  min-width: 0;
+}
+
+.dk-reserved,
+.dk-removing {
+  flex: none;
+  border-radius: 5px;
+  padding: 1px 7px;
+  font-size: 11px;
+  font-weight: 700;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+  white-space: nowrap;
+}
+
+.dk-reserved {
+  border: 1px solid var(--color-green-200);
+  background: var(--color-green-50);
+  color: var(--color-green-700);
+}
+
+.dk-removing {
+  border: 1px solid var(--color-yellow-200);
+  background: var(--color-yellow-50);
+  color: var(--color-yellow-700);
+}
+
 .pager {
   display: flex;
   align-items: center;

config/config.exs

@@ -31,6 +31,12 @@ config :bob,
       time: {1, 0, 0},
       queue: true
     ],
+    [
+      module: Bob.Job.DockerCleanup,
+      period: :day,
+      time: {2, 0, 0},
+      queue: true
+    ],
     [
       module: Bob.Job.ReconcileBaseImages,
       period: {1, :hour},
@@ -51,7 +57,10 @@ config :bob,
   parallel_jobs: 1,
   local_jobs: [],
   remote_jobs: [],
-  hexpm_impl: Bob.Hexpm.Impl
+  hexpm_impl: Bob.Hexpm.Impl,
+  # :dry_run logs the counts a live run would delete without touching Docker
+  # Hub; :live deletes.
+  docker_cleanup_mode: :dry_run
 
 config :mime, :types, %{
   "application/vnd.bob+erlang" => ["erlang"]

lib/bob/artifacts.ex

@@ -6,7 +6,7 @@ defmodule Bob.Artifacts do
 
   @builds_txt_lock 4_771_002
 
-  # Each staging row binds six parameters; Postgres caps a statement at 65535,
+  # Each staging row binds eight parameters; Postgres caps a statement at 65535,
   # so a page is inserted in chunks well under that ceiling.
   @staging_chunk 5000
 
@@ -17,22 +17,50 @@ defmodule Bob.Artifacts do
   # wraps.
   @long_query_timeout 5 * 60 * 1000
 
+  @docker_cleanup_per_arch_repos ~w(
+    hexpm/erlang-amd64 hexpm/erlang-arm64 hexpm/elixir-amd64 hexpm/elixir-arm64
+  )
+  @docker_cleanup_manifest_repos ~w(hexpm/erlang hexpm/elixir)
+
+  # A tag is reserved (never auto-deleted) while a non-expired build request pins
+  # it. A request maps deterministically to a single tag name, so the match is a
+  # plain equality on the indexed `tag` column — a hash anti-join against the
+  # small build_requests table — rather than extracting JSONB per scanned row.
+  # The erlang and elixir tag-name formats are disjoint (elixir carries
+  # `-erlang-`), so the name alone identifies the kind. `d` is the docker_tags
+  # row the predicate is applied to. The CASE mirrors
+  # `Bob.BuildRequests.request_target/1`; a test asserts the two encodings
+  # agree — update both together.
+  @reserved_predicate """
+  EXISTS (
+    SELECT 1
+    FROM build_requests br
+    WHERE br.state IN ('pending', 'completed')
+      AND d.tag =
+        CASE br.kind
+          WHEN 'erlang' THEN br.erlang || '-' || br.os || '-' || br.os_version
+          ELSE br.elixir || '-erlang-' || br.erlang || '-' || br.os || '-' || br.os_version
+        END
+  )
+  """
+
   def add(attrs) do
     upsert(attrs)
     generate_builds_txt(attrs.arch, attrs.os)
     Bob.Fastly.purge_builds(purge_keys(attrs.arch, attrs.os, attrs.name))
     :ok
   end
 
-  def add_docker_tag(repo, tag, archs, built_at \\ DateTime.utc_now()) do
+  def add_docker_tag(repo, tag, archs, built_at \\ DateTime.utc_now(), last_pulled \\ nil) do
     now = NaiveDateTime.utc_now()
     built_at = dump_utc_datetime(built_at)
+    last_pulled = last_pulled && dump_utc_datetime(last_pulled)
     search = DockerTagSearch.metadata(repo, tag)
 
     Repo.query!(
       """
-      INSERT INTO docker_tags (repo, tag, archs, search, built_at, inserted_at, updated_at)
-      VALUES ($1, $2, $3, $4, $5, $6, $6)
+      INSERT INTO docker_tags (repo, tag, archs, search, built_at, last_pulled, inserted_at, updated_at)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $7)
       ON CONFLICT (repo, tag)
       DO UPDATE SET
         archs = (
@@ -41,16 +69,17 @@ defmodule Bob.Artifacts do
         ),
         search = EXCLUDED.search,
         built_at = EXCLUDED.built_at,
+        last_pulled = COALESCE(EXCLUDED.last_pulled, docker_tags.last_pulled),
         updated_at = EXCLUDED.updated_at
       """,
-      [repo, tag, archs, search, built_at, now]
+      [repo, tag, archs, search, built_at, last_pulled, now]
     )
 
     :ok
   end
 
   @doc """
-  Inserts a page of `{tag, archs, built_at}` tuples into the staging table under `token`.
+  Inserts a page of `{tag, archs, built_at, last_pulled}` tuples into the staging table under `token`.
 
   Reconcile streams Docker Hub a page at a time into staging so the full tag
   list (up to ~1M for hexpm/elixir) is never held in memory and no connection is
@@ -64,14 +93,15 @@ defmodule Bob.Artifacts do
     now = NaiveDateTime.utc_now()
 
     tag_archs_built_at
-    |> Enum.map(fn {tag, archs, built_at} ->
+    |> Enum.map(fn {tag, archs, built_at, last_pulled} ->
       %{
         token: token,
         repo: repo,
         tag: tag,
         archs: archs,
         search: DockerTagSearch.metadata(repo, tag),
         built_at: built_at,
+        last_pulled: last_pulled,
         inserted_at: now
       }
     end)
@@ -85,7 +115,9 @@ defmodule Bob.Artifacts do
   Applies the tags staged under `token` for `repo` to `docker_tags`, in one
   transaction: upsert every staged tag, prune any `docker_tags` row whose tag is
   no longer staged, then drop the staging rows. `DISTINCT ON` collapses any
-  duplicate tags Docker Hub returned across pages.
+  duplicate tags Docker Hub returned across pages. A staged tag with no pull
+  time keeps the row's recorded `last_pulled` — Docker Hub omits it
+  transiently, and losing it would expose a pulled tag to cleanup.
 
   Only call this once the full fetch succeeded — a partial fetch would prune
   tags that were merely not yet fetched.
@@ -97,8 +129,8 @@ defmodule Bob.Artifacts do
       fn ->
         Repo.query!(
           """
-          INSERT INTO docker_tags (repo, tag, archs, search, built_at, inserted_at, updated_at)
-          SELECT DISTINCT ON (repo, tag) repo, tag, archs, search, built_at, $3, $3
+          INSERT INTO docker_tags (repo, tag, archs, search, built_at, last_pulled, inserted_at, updated_at)
+          SELECT DISTINCT ON (repo, tag) repo, tag, archs, search, built_at, last_pulled, $3, $3
           FROM docker_tags_staging
           WHERE token = $1 AND repo = $2
           ORDER BY repo, tag, built_at DESC
@@ -107,10 +139,13 @@ defmodule Bob.Artifacts do
             archs = EXCLUDED.archs,
             search = EXCLUDED.search,
             built_at = EXCLUDED.built_at,
+            last_pulled = COALESCE(EXCLUDED.last_pulled, docker_tags.last_pulled),
             updated_at = EXCLUDED.updated_at
           WHERE docker_tags.archs IS DISTINCT FROM EXCLUDED.archs
              OR docker_tags.search IS DISTINCT FROM EXCLUDED.search
              OR docker_tags.built_at IS DISTINCT FROM EXCLUDED.built_at
+             OR COALESCE(EXCLUDED.last_pulled, docker_tags.last_pulled)
+                IS DISTINCT FROM docker_tags.last_pulled
           """,
           [token, repo, now],
           timeout: @long_query_timeout
@@ -473,6 +508,128 @@ defmodule Bob.Artifacts do
     Enum.map(rows, fn [tag, archs] -> {tag, archs} end)
   end
 
+  def docker_cleanup_per_arch_repos(), do: @docker_cleanup_per_arch_repos
+  def docker_cleanup_manifest_repos(), do: @docker_cleanup_manifest_repos
+
+  @doc """
+  Counts, per repo, the tags the cleanup would delete: per-arch tags built
+  before `cutoff` and manifest tags neither pulled nor built since `cutoff`.
+  `GREATEST` keeps a tag whose last pull is ancient but that was re-pushed
+  recently — deleting it would 404 pullers only for the checker to rebuild the
+  manifest from its live per-arch tags. Reserved tags are excluded. Used by the
+  cleanup's dry run to report what live deletion would remove.
+  """
+  def count_stale_per_arch_tags(cutoff) do
+    count_stale(@docker_cleanup_per_arch_repos, "d.built_at < $2", cutoff)
+  end
+
+  def count_stale_manifest_tags(cutoff) do
+    count_stale(
+      @docker_cleanup_manifest_repos,
+      "GREATEST(d.last_pulled, d.built_at) < $2",
+      cutoff
+    )
+  end
+
+  defp count_stale(repos, age_predicate, cutoff) do
+    %{rows: rows} =
+      Repo.query!(
+        """
+        SELECT d.repo, count(*)
+        FROM docker_tags d
+        WHERE d.repo = ANY($1) AND #{age_predicate} AND NOT #{@reserved_predicate}
+        GROUP BY d.repo
+        """,
+        [repos, dump_utc_datetime(cutoff)],
+        timeout: @long_query_timeout
+      )
+
+    Map.new(rows, fn [repo, count] -> {repo, count} end)
+  end
+
+  @doc """
+  A batch of `{repo, tag}` the cleanup may delete, same predicates as
+  `count_stale_*`. `limit` bounds the batch so a single run deletes a manageable
+  slice and converges over successive runs.
+  """
+  def stale_per_arch_tags(cutoff, limit) do
+    stale(@docker_cleanup_per_arch_repos, "d.built_at < $2", cutoff, limit)
+  end
+
+  def stale_manifest_tags(cutoff, limit) do
+    stale(
+      @docker_cleanup_manifest_repos,
+      "GREATEST(d.last_pulled, d.built_at) < $2",
+      cutoff,
+      limit
+    )
+  end
+
+  defp stale(repos, age_predicate, cutoff, limit) do
+    %{rows: rows} =
+      Repo.query!(
+        """
+        SELECT d.repo, d.tag
+        FROM docker_tags d
+        WHERE d.repo = ANY($1) AND #{age_predicate} AND NOT #{@reserved_predicate}
+        LIMIT $3
+        """,
+        [repos, dump_utc_datetime(cutoff), limit],
+        timeout: @long_query_timeout
+      )
+
+    Enum.map(rows, fn [repo, tag] -> {repo, tag} end)
+  end
+
+  @doc "Whether a build request currently reserves `tag` in `repo`."
+  def docker_tag_reserved?(repo, tag) do
+    %{rows: [[reserved?]]} =
+      Repo.query!(
+        """
+        SELECT EXISTS(
+          SELECT 1 FROM docker_tags d
+          WHERE d.repo = $1 AND d.tag = $2 AND #{@reserved_predicate}
+        )
+        """,
+        [repo, tag]
+      )
+
+    reserved?
+  end
+
+  @doc """
+  The ids of the given `DockerTag` rows that a build request reserves, as a
+  MapSet. Lets the tags page flag reserved tags without re-deriving the match.
+  """
+  def reserved_docker_tag_ids([]), do: MapSet.new()
+
+  def reserved_docker_tag_ids(tags) do
+    ids = Enum.map(tags, & &1.id)
+
+    %{rows: rows} =
+      Repo.query!(
+        "SELECT d.id FROM docker_tags d WHERE d.id = ANY($1::bigint[]) AND #{@reserved_predicate}",
+        [ids]
+      )
+
+    MapSet.new(rows, fn [id] -> id end)
+  end
+
+  @doc "Drops `docker_tags` rows for the given `{repo, tag}` pairs, after Docker Hub deletion."
+  def delete_docker_tags([]), do: :ok
+
+  def delete_docker_tags(repo_tags) do
+    repo_tags
+    |> Enum.group_by(fn {repo, _tag} -> repo end, fn {_repo, tag} -> tag end)
+    |> Enum.each(fn {repo, tags} ->
+      Repo.query!("DELETE FROM docker_tags WHERE repo = $1 AND tag = ANY($2)", [repo, tags],
+        timeout: @long_query_timeout
+      )
+    end)
+
+    :ok
+  end
+
   def base_image_tags(repo) do
     Repo.all(
       from(b in BaseImageTag,

lib/bob/artifacts/docker_tag.ex

@@ -7,6 +7,7 @@ defmodule Bob.Artifacts.DockerTag do
     field(:archs, {:array, :string})
     field(:search, :map, default: %{})
     field(:built_at, :utc_datetime_usec)
+    field(:last_pulled, :utc_datetime_usec)
     timestamps(type: :utc_datetime_usec)
   end
 end

lib/bob/artifacts/docker_tag_staging.ex

@@ -9,6 +9,7 @@ defmodule Bob.Artifacts.DockerTagStaging do
     field(:archs, {:array, :string})
     field(:search, :map, default: %{})
     field(:built_at, :utc_datetime_usec)
+    field(:last_pulled, :utc_datetime_usec)
     field(:inserted_at, :naive_datetime_usec)
   end
 end