Skip to content

Commit 9ed1701

Browse files
authored
Merge pull request #3733 from PolicyEngine/feat/app-engine-idle-version-cleanup
Stop idle App Engine versions after deploy
2 parents c76856e + 4407075 commit 9ed1701

5 files changed

Lines changed: 503 additions & 0 deletions

File tree

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env bash
2+
3+
# Deactivate idle App Engine versions after a production promote.
4+
#
5+
# App Engine Flexible keeps at least one always-on VM for every SERVING version,
6+
# regardless of traffic, so old versions left SERVING after a promote keep
7+
# costing money even at 0 traffic (~$212/mo per 4 vCPU / 24 GB version). The
8+
# deploy pipeline promotes traffic with set-traffic but never deactivates the
9+
# versions it supersedes, so they pile up.
10+
#
11+
# This script, run after promote-production:
12+
# 1. Keeps the version currently receiving traffic PLUS the newest
13+
# KEEP_WARM_PROD prod-* versions SERVING (a warm rollback window), and STOPs
14+
# every other SERVING version (older prod + any leftover staging).
15+
# 2. DELETEs stopped versions beyond the retention window: keeps the newest
16+
# KEEP_STOPPED_PROD prod-* and KEEP_STOPPED_STAGING staging-* stopped
17+
# versions, and deletes every other stopped version (older prod, older
18+
# staging, and legacy timestamp-named versions), to stay under App Engine's
19+
# 210-versions-per-service limit. Prod is kept deeper because only prod
20+
# versions are meaningful rollback targets.
21+
#
22+
# Stopping (not deleting) preserves rollback: a stopped version can be brought
23+
# back with `gcloud app versions start <v>` then `gcloud app services set-traffic
24+
# ${SERVICE} --splits=<v>=1`. The version currently serving traffic is never
25+
# stopped or deleted.
26+
#
27+
# Set DRY_RUN=1 to print the plan without changing anything.
28+
#
29+
# Written to run on bash 3.2 (no mapfile / associative arrays) so it can be
30+
# dry-run locally on macOS as well as in CI.
31+
32+
set -euo pipefail
33+
34+
APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}"
35+
KEEP_WARM_PROD="${KEEP_WARM_PROD:-2}"
36+
KEEP_STOPPED_PROD="${KEEP_STOPPED_PROD:-10}"
37+
KEEP_STOPPED_STAGING="${KEEP_STOPPED_STAGING:-3}"
38+
DRY_RUN="${DRY_RUN:-0}"
39+
40+
common_args=("--service=${APP_ENGINE_SERVICE}")
41+
if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
42+
common_args+=("--project=${APP_ENGINE_PROJECT}")
43+
fi
44+
45+
contains() {
46+
local needle="$1"
47+
shift
48+
local item
49+
for item in "$@"; do
50+
[[ "${item}" == "${needle}" ]] && return 0
51+
done
52+
return 1
53+
}
54+
55+
read_versions() {
56+
# Args: extra gcloud filter. Emits version ids newest-first, one per line.
57+
local filter="$1"
58+
gcloud app versions list "${common_args[@]}" \
59+
--filter="${filter}" \
60+
--sort-by="~version.createTime" \
61+
--format="value(id)"
62+
}
63+
64+
# Versions currently receiving any traffic — never touched.
65+
serving_traffic=()
66+
while IFS= read -r v; do
67+
[[ -n "${v}" ]] && serving_traffic+=("${v}")
68+
done < <(read_versions "version.servingStatus=SERVING AND traffic_split>0")
69+
70+
# All SERVING versions, newest first.
71+
serving_all=()
72+
while IFS= read -r v; do
73+
[[ -n "${v}" ]] && serving_all+=("${v}")
74+
done < <(read_versions "version.servingStatus=SERVING")
75+
76+
# Fail safe: if there are SERVING versions but none report traffic, we cannot
77+
# identify the live service. Refuse to stop anything rather than risk
78+
# deactivating production (e.g. if the traffic query returns nothing). The
79+
# delete step never runs either, since we exit here.
80+
if [[ "${#serving_all[@]}" -gt 0 && "${#serving_traffic[@]}" -eq 0 ]]; then
81+
echo "ERROR: ${#serving_all[@]} SERVING version(s) but none are receiving traffic;" >&2
82+
echo "refusing to stop versions (cannot identify the live service). Aborting." >&2
83+
exit 1
84+
fi
85+
86+
# Newest KEEP_WARM_PROD prod-* versions to keep warm for instant rollback.
87+
warm_prod=()
88+
while IFS= read -r v; do
89+
[[ -n "${v}" ]] && warm_prod+=("${v}")
90+
done < <(printf '%s\n' "${serving_all[@]:-}" | grep '^prod-' | head -n "${KEEP_WARM_PROD}")
91+
92+
# Keep set = traffic-serving versions + warm prod versions.
93+
keep=("${serving_traffic[@]:-}" "${warm_prod[@]:-}")
94+
95+
# Stop every SERVING version not in the keep set.
96+
to_stop=()
97+
for v in "${serving_all[@]:-}"; do
98+
[[ -z "${v}" ]] && continue
99+
contains "${v}" "${keep[@]:-}" && continue
100+
to_stop+=("${v}")
101+
done
102+
103+
echo "Service: ${APP_ENGINE_SERVICE}"
104+
echo "Serving traffic (never touched): ${serving_traffic[*]:-<none>}"
105+
echo "Keeping warm (rollback window): ${warm_prod[*]:-<none>}"
106+
echo "Stopping (${#to_stop[@]}): ${to_stop[*]:-<none>}"
107+
108+
if [[ "${#to_stop[@]}" -gt 0 ]]; then
109+
if [[ "${DRY_RUN}" == "1" ]]; then
110+
echo "[dry-run] would stop ${#to_stop[@]} version(s)"
111+
else
112+
gcloud app versions stop "${to_stop[@]}" "${common_args[@]}" --quiet
113+
fi
114+
fi
115+
116+
# Delete stopped versions beyond the retention window (version-quota hygiene).
117+
# Keep the newest KEEP_STOPPED_PROD stopped prod-* versions (cold rollback
118+
# targets) and the newest KEEP_STOPPED_STAGING stopped staging-* versions; delete
119+
# everything else stopped — older prod, older staging, and legacy timestamp-named
120+
# versions. Serving/warm versions are SERVING, so never appear in this set.
121+
stopped_all=()
122+
while IFS= read -r v; do
123+
[[ -n "${v}" ]] && stopped_all+=("${v}")
124+
done < <(read_versions "version.servingStatus=STOPPED")
125+
126+
# Newest KEEP_STOPPED_PROD stopped prod-* versions to keep.
127+
keep_stopped_prod=()
128+
while IFS= read -r v; do
129+
[[ -n "${v}" ]] && keep_stopped_prod+=("${v}")
130+
done < <(printf '%s\n' "${stopped_all[@]:-}" | grep '^prod-' | head -n "${KEEP_STOPPED_PROD}")
131+
132+
# Newest KEEP_STOPPED_STAGING stopped staging-* versions to keep.
133+
keep_stopped_staging=()
134+
while IFS= read -r v; do
135+
[[ -n "${v}" ]] && keep_stopped_staging+=("${v}")
136+
done < <(printf '%s\n' "${stopped_all[@]:-}" | grep '^staging-' | head -n "${KEEP_STOPPED_STAGING}")
137+
138+
keep_stopped=("${keep_stopped_prod[@]:-}" "${keep_stopped_staging[@]:-}")
139+
140+
to_delete=()
141+
for v in "${stopped_all[@]:-}"; do
142+
[[ -z "${v}" ]] && continue
143+
contains "${v}" "${keep_stopped[@]:-}" && continue
144+
to_delete+=("${v}")
145+
done
146+
147+
echo "Stopped versions: ${#stopped_all[@]}"
148+
echo "Keeping stopped prod (${#keep_stopped_prod[@]}): ${keep_stopped_prod[*]:-<none>}"
149+
echo "Keeping stopped staging (${#keep_stopped_staging[@]}): ${keep_stopped_staging[*]:-<none>}"
150+
echo "Deleting (${#to_delete[@]}): ${to_delete[*]:-<none>}"
151+
152+
if [[ "${#to_delete[@]}" -gt 0 ]]; then
153+
if [[ "${DRY_RUN}" == "1" ]]; then
154+
echo "[dry-run] would delete ${#to_delete[@]} version(s)"
155+
else
156+
gcloud app versions delete "${to_delete[@]}" "${common_args[@]}" --quiet
157+
fi
158+
fi
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
# Stop (deactivate) a single App Engine version.
4+
#
5+
# App Engine Flexible keeps at least one VM running for every SERVING version
6+
# regardless of traffic, so a version left SERVING after its tests keeps costing
7+
# money. Stopping it drops its instances to zero (no billing) while keeping the
8+
# version deployed and restartable (`gcloud app versions start`) for rollback.
9+
10+
set -euo pipefail
11+
12+
: "${APP_ENGINE_VERSION:?APP_ENGINE_VERSION is required}"
13+
14+
APP_ENGINE_SERVICE="${APP_ENGINE_SERVICE:-default}"
15+
16+
stop_args=(
17+
"${APP_ENGINE_VERSION}"
18+
"--service=${APP_ENGINE_SERVICE}"
19+
"--quiet"
20+
)
21+
22+
if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
23+
stop_args+=("--project=${APP_ENGINE_PROJECT}")
24+
fi
25+
26+
echo "Stopping App Engine version ${APP_ENGINE_VERSION} (service ${APP_ENGINE_SERVICE})"
27+
gcloud app versions stop "${stop_args[@]}"

.github/workflows/push.yml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,37 @@ jobs:
286286
API_BASE_URL: ${{ needs.deploy-staging.outputs.url }}
287287
STAGING_API_TEST_PROBE_ID: ${{ needs.deploy-staging.outputs.version }}
288288

289+
stop-staging-app-engine-version:
290+
name: Stop staging App Engine version
291+
runs-on: ubuntu-latest
292+
needs:
293+
- deploy-staging
294+
- integration-tests-staging
295+
if: |
296+
(github.repository == 'PolicyEngine/policyengine-api')
297+
&& (github.event.head_commit.message == 'Update PolicyEngine API')
298+
environment: staging
299+
permissions:
300+
contents: read
301+
id-token: write
302+
steps:
303+
- name: Checkout repo
304+
uses: actions/checkout@v4
305+
- name: GCP authentication
306+
uses: "google-github-actions/auth@v2"
307+
with:
308+
workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
309+
service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}"
310+
- name: Set up GCloud
311+
uses: "google-github-actions/setup-gcloud@v2"
312+
# Flex keeps a VM running for every SERVING version regardless of traffic,
313+
# so the just-tested staging version would keep costing money. Stop it now
314+
# that its integration tests have passed; the next deploy makes a fresh one.
315+
- name: Stop staging App Engine version
316+
run: bash .github/scripts/stop_app_engine_version.sh
317+
env:
318+
APP_ENGINE_VERSION: ${{ needs.deploy-staging.outputs.version }}
319+
289320
integration-tests-staging-cloud-run:
290321
name: Run Cloud Run staging integration tests
291322
runs-on: ubuntu-latest
@@ -468,6 +499,35 @@ jobs:
468499
env:
469500
APP_ENGINE_VERSION: ${{ needs.deploy-production-candidate.outputs.version }}
470501

502+
cleanup-prod-app-engine-versions:
503+
name: Clean up idle production App Engine versions
504+
runs-on: ubuntu-latest
505+
needs: promote-production
506+
if: |
507+
(github.repository == 'PolicyEngine/policyengine-api')
508+
&& (github.event.head_commit.message == 'Update PolicyEngine API')
509+
environment: production
510+
permissions:
511+
contents: read
512+
id-token: write
513+
steps:
514+
- name: Checkout repo
515+
uses: actions/checkout@v4
516+
- name: GCP authentication
517+
uses: "google-github-actions/auth@v2"
518+
with:
519+
workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
520+
service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}"
521+
- name: Set up GCloud
522+
uses: "google-github-actions/setup-gcloud@v2"
523+
# Keep the live version plus the 2 newest prod versions warm for instant
524+
# rollback; stop every other SERVING version (older prod + leftover staging)
525+
# so idle Flex VMs stop billing; keep the newest 10 stopped prod and 3
526+
# stopped staging versions and delete the rest to stay under the
527+
# 210-versions-per-service limit. Retention counts are defaults in the script.
528+
- name: Stop superseded versions and prune old stopped versions
529+
run: bash .github/scripts/cleanup_app_engine_versions.sh
530+
471531
deploy-cloud-run-candidate:
472532
name: Deploy production Cloud Run candidate
473533
runs-on: ubuntu-latest
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Deploys now stop idle App Engine versions automatically. The staging version is stopped after its integration tests pass, and after a production promote only the two newest production versions stay warm while every other superseded version is stopped. Stopped versions are then pruned to the newest 10 production and newest 3 staging (older prod, older staging, and legacy timestamp-named versions are deleted). This removes always-on Flexible-environment VM cost from versions that serve no traffic, while keeping recent production versions available for rollback via `gcloud app versions start`.

0 commit comments

Comments
 (0)