Skip to content

Commit 51b7b04

Browse files
authored
Merge pull request #3742 from PolicyEngine/feat/artifact-registry-image-cleanup
Automate Artifact Registry image cleanup + purge dead image repos
2 parents 5fe6b05 + 08176b4 commit 51b7b04

5 files changed

Lines changed: 218 additions & 1 deletion

File tree

.github/scripts/cleanup_app_engine_versions.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@
1818
# staging, and legacy timestamp-named versions), to stay under App Engine's
1919
# 210-versions-per-service limit. Prod is kept deeper because only prod
2020
# versions are meaningful rollback targets.
21+
# 3. DELETEs the Artifact Registry image for each deleted version so the
22+
# `gae-flexible` repo stays in lockstep with the retained versions (App
23+
# Engine Flex builds one image per version — `<service>.<version-id>` — and
24+
# nothing else prunes them, so the repo grows without bound). Best-effort
25+
# per image; set CLEANUP_APP_ENGINE_IMAGES=0 to skip.
2126
#
2227
# Stopping (not deleting) preserves rollback: a stopped version can be brought
2328
# back with `gcloud app versions start <v>` then `gcloud app services set-traffic
@@ -37,6 +42,13 @@ KEEP_STOPPED_PROD="${KEEP_STOPPED_PROD:-10}"
3742
KEEP_STOPPED_STAGING="${KEEP_STOPPED_STAGING:-3}"
3843
DRY_RUN="${DRY_RUN:-0}"
3944

45+
# Delete each deleted version's App Engine Flex image from Artifact Registry so
46+
# the image repo does not grow one image per version forever.
47+
CLEANUP_APP_ENGINE_IMAGES="${CLEANUP_APP_ENGINE_IMAGES:-1}"
48+
AR_LOCATION="${AR_LOCATION:-us-central1}"
49+
AR_IMAGE_PROJECT="${AR_IMAGE_PROJECT:-${APP_ENGINE_PROJECT:-policyengine-api}}"
50+
AR_IMAGE_REPO="${AR_IMAGE_REPO:-gae-flexible}"
51+
4052
common_args=("--service=${APP_ENGINE_SERVICE}")
4153
if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
4254
common_args+=("--project=${APP_ENGINE_PROJECT}")
@@ -156,3 +168,23 @@ if [[ "${#to_delete[@]}" -gt 0 ]]; then
156168
gcloud app versions delete "${to_delete[@]}" "${common_args[@]}" --quiet
157169
fi
158170
fi
171+
172+
# Delete each deleted version's App Engine Flex image so the image repo stays in
173+
# lockstep with the retained versions. Best-effort per image: a version may
174+
# predate this repo (older images live in the legacy us.gcr.io) or its image may
175+
# already be gone, so individual misses are ignored.
176+
if [[ "${CLEANUP_APP_ENGINE_IMAGES}" == "1" && "${#to_delete[@]}" -gt 0 ]]; then
177+
image_repo="${AR_LOCATION}-docker.pkg.dev/${AR_IMAGE_PROJECT}/${AR_IMAGE_REPO}"
178+
img_done=0
179+
for v in "${to_delete[@]}"; do
180+
[[ -z "${v}" ]] && continue
181+
image="${image_repo}/${APP_ENGINE_SERVICE}.${v}"
182+
if [[ "${DRY_RUN}" == "1" ]]; then
183+
echo "[dry-run] would delete image ${image}"
184+
elif gcloud artifacts docker images delete "${image}" --delete-tags --quiet \
185+
>/dev/null 2>&1; then
186+
img_done=$((img_done + 1))
187+
fi
188+
done
189+
[[ "${DRY_RUN}" == "1" ]] || echo "Deleted ${img_done}/${#to_delete[@]} gae-flexible image(s)."
190+
fi
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/usr/bin/env bash
2+
3+
# Keep only the most recent Cloud Run images in Artifact Registry.
4+
#
5+
# The Cloud Run deploy pushes a new image on every release and nothing prunes
6+
# them, so the repo grows without bound. This keeps the newest KEEP image
7+
# versions (by createTime) and deletes the rest.
8+
#
9+
# Note: deleting an image that an older Cloud Run *revision* still references
10+
# stops that revision from starting again. Cloud Run here is the non-primary
11+
# migration candidate, so losing rollback to revisions older than KEEP deploys
12+
# is acceptable; KEEP=15 leaves a generous window.
13+
#
14+
# Images are sorted in-script (by createTime) rather than trusting gcloud's
15+
# Artifact Registry `--sort-by`, which has proven unreliable. Set DRY_RUN=1 to
16+
# print the plan without changing anything. Written for bash 3.2.
17+
#
18+
# The target repo (region/project/repository) comes from cloud_run_env.sh — the
19+
# same config the deploy scripts source — so this can never prune a different
20+
# repo than the pipeline actually pushes to.
21+
22+
set -euo pipefail
23+
24+
source .github/scripts/cloud_run_env.sh
25+
cloud_run_set_defaults
26+
27+
KEEP="${KEEP:-15}"
28+
DRY_RUN="${DRY_RUN:-0}"
29+
AR_LOCATION="${AR_LOCATION:-${CLOUD_RUN_REGION}}"
30+
AR_PROJECT="${AR_PROJECT:-${CLOUD_RUN_PROJECT}}"
31+
AR_REPO="${AR_REPO:-${CLOUD_RUN_ARTIFACT_REPOSITORY}}"
32+
33+
image_repo="${AR_LOCATION}-docker.pkg.dev/${AR_PROJECT}/${AR_REPO}"
34+
35+
# Deletable refs (<image>@<digest>), newest first — sorted here by createTime
36+
# (ISO timestamps sort chronologically), one row per image version.
37+
refs=()
38+
while IFS='|' read -r _ctime pkg ver; do
39+
[[ -n "${ver}" ]] && refs+=("${pkg}@${ver}")
40+
done < <(
41+
gcloud artifacts docker images list "${image_repo}" \
42+
--format="value[separator='|'](createTime,package,version)" \
43+
| sort -r
44+
)
45+
46+
total="${#refs[@]}"
47+
to_delete=()
48+
if [[ "${total}" -gt "${KEEP}" ]]; then
49+
to_delete=("${refs[@]:${KEEP}}")
50+
fi
51+
52+
echo "Repo: ${image_repo}"
53+
echo "Images: ${total} (keeping newest ${KEEP})"
54+
echo "Deleting: ${#to_delete[@]}"
55+
56+
deleted=0
57+
for ref in "${to_delete[@]:-}"; do
58+
[[ -z "${ref}" ]] && continue
59+
if [[ "${DRY_RUN}" == "1" ]]; then
60+
echo "[dry-run] would delete ${ref}"
61+
elif gcloud artifacts docker images delete "${ref}" --delete-tags --quiet \
62+
>/dev/null 2>&1; then
63+
deleted=$((deleted + 1))
64+
fi
65+
done
66+
[[ "${DRY_RUN}" == "1" ]] || echo "Deleted ${deleted}/${#to_delete[@]} image(s)."

.github/workflows/push.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -607,6 +607,33 @@ jobs:
607607
- name: Wait for Cloud Run production service health
608608
run: bash .github/scripts/health_check.sh "${{ steps.cloud_run_service_url.outputs.url }}/readiness-check"
609609

610+
cleanup-cloud-run-images:
611+
name: Clean up old Cloud Run images
612+
runs-on: ubuntu-latest
613+
needs: deploy-cloud-run-candidate
614+
if: |
615+
(github.repository == 'PolicyEngine/policyengine-api')
616+
&& (github.event.head_commit.message == 'Update PolicyEngine API')
617+
environment: production
618+
permissions:
619+
contents: read
620+
id-token: write
621+
steps:
622+
- name: Checkout repo
623+
uses: actions/checkout@v4
624+
- name: GCP authentication
625+
uses: "google-github-actions/auth@v2"
626+
with:
627+
workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
628+
service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}"
629+
- name: Set up GCloud
630+
uses: "google-github-actions/setup-gcloud@v2"
631+
# The Cloud Run deploy pushes a new image every release and nothing prunes
632+
# them, so the repo grows without bound. Keep the 15 most recent image
633+
# versions (default in the script) and delete older ones.
634+
- name: Prune old Cloud Run images
635+
run: bash .github/scripts/cleanup_cloud_run_images.sh
636+
610637
docker:
611638
name: Docker
612639
runs-on: ubuntu-latest
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Prune Artifact Registry images automatically so the image repos stop growing without bound. After each deploy the pipeline keeps the 15 most recent Cloud Run images and deletes each removed App Engine version's Flex (`gae-flexible`) image. This bounds the ongoing growth of Artifact Registry storage (the repos had reached ~628 GiB / ~$45/mo).

tests/unit/test_app_engine_cleanup_scripts.py

Lines changed: 92 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
REPO = Path(__file__).resolve().parents[2]
1919
CLEANUP_SCRIPT = ".github/scripts/cleanup_app_engine_versions.sh"
2020
STOP_SCRIPT = ".github/scripts/stop_app_engine_version.sh"
21+
CR_IMAGES_SCRIPT = ".github/scripts/cleanup_cloud_run_images.sh"
2122

2223
# A stub `gcloud` that answers `app versions list` from MOCK_* env vars (each a
2324
# newline-joined, newest-first id list) and appends any stop/delete calls to
@@ -28,6 +29,8 @@
2829
case "$a" in --filter=*) filter="${a#--filter=}";; esac
2930
done
3031
case "$*" in
32+
*"artifacts docker images list"*) [ -n "${MOCK_AR_IMAGES:-}" ] && printf '%s\\n' "$MOCK_AR_IMAGES";;
33+
*"artifacts docker images delete"*) echo "IMG_DELETE $*" >> "$GCLOUD_CALLS";;
3134
*"versions list"*)
3235
case "$filter" in
3336
*"traffic_split>0"*) [ -n "${MOCK_TRAFFIC:-}" ] && printf '%s\\n' "$MOCK_TRAFFIC";;
@@ -48,6 +51,7 @@ def _run(
4851
serving: list[str] | None = None,
4952
traffic: list[str] | None = None,
5053
stopped: list[str] | None = None,
54+
ar_images: list[str] | None = None,
5155
extra_env: dict[str, str] | None = None,
5256
) -> tuple[subprocess.CompletedProcess[str], Path]:
5357
stub_dir = tmp_path / "bin"
@@ -68,6 +72,7 @@ def _run(
6872
"MOCK_SERVING": "\n".join(serving or []),
6973
"MOCK_TRAFFIC": "\n".join(traffic or []),
7074
"MOCK_STOPPED": "\n".join(stopped or []),
75+
"MOCK_AR_IMAGES": "\n".join(ar_images or []),
7176
}
7277
env.update(extra_env or {})
7378

@@ -94,13 +99,99 @@ def _list_after(label: str, stdout: str) -> list[str]:
9499

95100

96101
def test_cleanup_scripts_are_shell_syntax_valid():
97-
for script in (CLEANUP_SCRIPT, STOP_SCRIPT):
102+
for script in (CLEANUP_SCRIPT, STOP_SCRIPT, CR_IMAGES_SCRIPT):
98103
result = subprocess.run(
99104
["bash", "-n", script], cwd=REPO, capture_output=True, text=True
100105
)
101106
assert result.returncode == 0, f"{script}: {result.stderr}"
102107

103108

109+
# --- cloud run image cleanup (keep 15) -----------------------------------
110+
111+
_CR_PKG = (
112+
"us-central1-docker.pkg.dev/policyengine-api/policyengine-api/policyengine-api"
113+
)
114+
115+
116+
def _ar_rows(digests: list[str], package: str = _CR_PKG) -> list[str]:
117+
"""`createTime|package|version` rows, oldest first (day 01 = oldest). The
118+
script sorts by createTime descending itself, so order here is irrelevant."""
119+
return [
120+
f"2026-01-{i + 1:02d}T00:00:00|{package}|sha256:{d}"
121+
for i, d in enumerate(digests)
122+
]
123+
124+
125+
def test_cloud_run_images_keeps_newest_15(tmp_path):
126+
digests = [f"d{i:02d}" for i in range(18)] # d00 oldest ... d17 newest
127+
result, calls = _run(CR_IMAGES_SCRIPT, tmp_path, ar_images=_ar_rows(digests))
128+
assert result.returncode == 0, result.stderr
129+
assert "Images: 18 (keeping newest 15)" in result.stdout
130+
assert "Deleting: 3" in result.stdout
131+
# The 3 oldest digests are the ones deleted; the newest is kept.
132+
for old in ("sha256:d00", "sha256:d01", "sha256:d02"):
133+
assert f"would delete {_CR_PKG}@{old}" in result.stdout
134+
assert "sha256:d17" not in result.stdout.split("Deleting:")[1]
135+
assert calls.read_text() == "" # DRY_RUN performs no mutations
136+
137+
138+
def test_cloud_run_images_no_delete_when_within_keep(tmp_path):
139+
result, _ = _run(
140+
CR_IMAGES_SCRIPT, tmp_path, ar_images=_ar_rows([f"d{i}" for i in range(10)])
141+
)
142+
assert result.returncode == 0, result.stderr
143+
assert "Deleting: 0" in result.stdout
144+
assert "would delete" not in result.stdout
145+
146+
147+
def test_cloud_run_images_respects_keep_override(tmp_path):
148+
result, _ = _run(
149+
CR_IMAGES_SCRIPT,
150+
tmp_path,
151+
ar_images=_ar_rows([f"d{i}" for i in range(10)]),
152+
extra_env={"KEEP": "5"},
153+
)
154+
assert "keeping newest 5" in result.stdout
155+
assert "Deleting: 5" in result.stdout
156+
157+
158+
# --- app engine cleanup also prunes each deleted version's image ----------
159+
160+
161+
def test_app_engine_cleanup_deletes_gae_flexible_image_per_deleted_version(tmp_path):
162+
result, _ = _run(
163+
CLEANUP_SCRIPT,
164+
tmp_path,
165+
serving=["prod-110", "prod-108"],
166+
traffic=["prod-110"],
167+
stopped=[f"prod-{n}" for n in range(106, 84, -2)] # 11 prod -> 1 deleted
168+
+ ["staging-90", "staging-88", "staging-86", "staging-84"], # 1 deleted
169+
)
170+
assert result.returncode == 0, result.stderr
171+
deleted = _list_after("Deleting", result.stdout)
172+
assert deleted, "expected some versions to be deleted"
173+
# AR_IMAGE_PROJECT defaults to APP_ENGINE_PROJECT, which _run sets.
174+
repo = "us-central1-docker.pkg.dev/test-project/gae-flexible/default."
175+
# Every deleted version has a matching gae-flexible image deletion line.
176+
for v in deleted:
177+
assert f"would delete image {repo}{v}" in result.stdout
178+
# No image deletion for a kept (live/warm) version.
179+
assert f"{repo}prod-110" not in result.stdout
180+
181+
182+
def test_app_engine_cleanup_image_pruning_can_be_disabled(tmp_path):
183+
result, _ = _run(
184+
CLEANUP_SCRIPT,
185+
tmp_path,
186+
serving=["prod-110", "prod-108"],
187+
traffic=["prod-110"],
188+
stopped=[f"prod-{n}" for n in range(106, 84, -2)],
189+
extra_env={"CLEANUP_APP_ENGINE_IMAGES": "0"},
190+
)
191+
assert result.returncode == 0, result.stderr
192+
assert "would delete image" not in result.stdout
193+
194+
104195
# --- cleanup: stop selection ---------------------------------------------
105196

106197

0 commit comments

Comments
 (0)