|
5 | 5 | types: |
6 | 6 | - deploy-docker-image |
7 | 7 | workflow_dispatch: |
| 8 | + inputs: |
| 9 | + image_tag: |
| 10 | + description: "Immutable image tag to deploy (e.g. sha-abc1234). 'latest' is rejected - it can't be verified by /version and is not a safe rollback target." |
| 11 | + required: true |
| 12 | + |
| 13 | +concurrency: |
| 14 | + group: deploy-self-hosted |
| 15 | + cancel-in-progress: false |
| 16 | + |
| 17 | +env: |
| 18 | + IMAGE: ghcr.io/butr/nmstats |
| 19 | + HEALTH_URL: https://nmstats.butr.link/healthz |
| 20 | + VERSION_URL: https://nmstats.butr.link/version |
| 21 | + STACK: ${{ vars.DEPLOY_STACK }} |
| 22 | + SERVICE: ${{ vars.DEPLOY_SERVICE }} |
| 23 | + COMPOSE: ${{ vars.DEPLOY_COMPOSE }} |
| 24 | + IMAGE_TAG: ${{ github.event.client_payload.image_tag || inputs.image_tag }} |
| 25 | + STATE_DIR: /tmp/nmstats-deploy |
8 | 26 |
|
9 | 27 | jobs: |
10 | 28 | deploy: |
11 | 29 | name: Deploy |
12 | 30 | runs-on: ubuntu-latest |
13 | 31 | environment: |
14 | | - name: 'self-hosted-backend' |
15 | | - url: 'https://nmstats.butr.link' |
| 32 | + name: "self-hosted-backend" |
| 33 | + url: "https://nmstats.butr.link" |
16 | 34 | steps: |
17 | | - - name: multiple command |
18 | | - uses: appleboy/ssh-action@master |
19 | | - with: |
20 | | - host: ${{ secrets.HOST }} |
21 | | - username: ${{ secrets.USERNAME }} |
22 | | - password: ${{ secrets.PASSWORD }} |
23 | | - port: ${{ secrets.PORT }} |
24 | | - script: | |
25 | | - docker image pull ghcr.io/butr/nmstats; |
26 | | - docker stack deploy --prune --resolve-image always -c /deploy/nmstats/docker-compose.yml nmstats; |
| 35 | + - name: Mask deploy topology in logs |
| 36 | + timeout-minutes: 1 |
| 37 | + run: | |
| 38 | + echo "::add-mask::${{ vars.DEPLOY_STACK }}" |
| 39 | + echo "::add-mask::${{ vars.DEPLOY_SERVICE }}" |
| 40 | + echo "::add-mask::${{ vars.DEPLOY_COMPOSE }}" |
| 41 | +
|
| 42 | + - name: Validate image tag |
| 43 | + timeout-minutes: 1 |
| 44 | + run: | |
| 45 | + set -euo pipefail |
| 46 | +
|
| 47 | + if [ -z "${IMAGE_TAG}" ] || [ "${IMAGE_TAG}" = "latest" ]; then |
| 48 | + echo "::error::IMAGE_TAG must be an immutable tag (e.g. sha-abc1234); got '${IMAGE_TAG:-<empty>}'. 'latest' is rejected because /version cannot confirm it and it is not a safe rollback target." |
| 49 | + exit 1 |
| 50 | + fi |
| 51 | + echo "Deploying tag: ${IMAGE_TAG}" |
| 52 | +
|
| 53 | + - name: Pull image and capture rollback state |
| 54 | + timeout-minutes: 10 |
| 55 | + uses: appleboy/ssh-action@v1 |
| 56 | + with: |
| 57 | + host: ${{ secrets.HOST }} |
| 58 | + username: ${{ secrets.USERNAME }} |
| 59 | + password: ${{ secrets.PASSWORD }} |
| 60 | + port: ${{ secrets.PORT }} |
| 61 | + envs: IMAGE,SERVICE,IMAGE_TAG,STATE_DIR |
| 62 | + script: | |
| 63 | + set -eu |
| 64 | +
|
| 65 | + # Start from a clean slate so a previous deploy's stale state can never be read as this deploy's. |
| 66 | + rm -rf "${STATE_DIR}"; mkdir -p "${STATE_DIR}" |
| 67 | + echo "Deploying ${IMAGE}:${IMAGE_TAG}" |
| 68 | + docker pull "${IMAGE}:${IMAGE_TAG}" |
| 69 | +
|
| 70 | + # Capture the EXACT currently-running image WITH its @sha256 digest, plus the replica count, as the |
| 71 | + # rollback target. Pinning by digest makes rollback immune to tag movement - 'latest' (and even this |
| 72 | + # commit's sha tag) may already point at the very build we're about to deploy. |
| 73 | + OLD_IMAGE_REF="$(docker service inspect --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' "${SERVICE}" 2>/dev/null || true)" |
| 74 | + OLD_REPLICAS="$(docker service inspect --format '{{.Spec.Mode.Replicated.Replicas}}' "${SERVICE}" 2>/dev/null || true)" |
| 75 | + if [ -n "${OLD_IMAGE_REF}" ]; then |
| 76 | + echo "${OLD_IMAGE_REF}" > "${STATE_DIR}/old_image_ref" |
| 77 | + echo "${OLD_REPLICAS:-1}" > "${STATE_DIR}/old_replicas" |
| 78 | + # Tag (no repo, no digest) is only used by the prune step's keep-list. |
| 79 | + OLD_TAG="${OLD_IMAGE_REF%@*}"; OLD_TAG="${OLD_TAG##*:}" |
| 80 | + echo "${OLD_TAG}" > "${STATE_DIR}/old_tag" |
| 81 | + echo "Previous image: ${OLD_IMAGE_REF} (replicas ${OLD_REPLICAS:-1})" |
| 82 | + else |
| 83 | + echo "::warning::No running service found; automatic image rollback will be unavailable" |
| 84 | + fi |
| 85 | +
|
| 86 | + - name: Deploy new tag |
| 87 | + timeout-minutes: 5 |
| 88 | + uses: appleboy/ssh-action@v1 |
| 89 | + with: |
| 90 | + host: ${{ secrets.HOST }} |
| 91 | + username: ${{ secrets.USERNAME }} |
| 92 | + password: ${{ secrets.PASSWORD }} |
| 93 | + port: ${{ secrets.PORT }} |
| 94 | + envs: STACK,COMPOSE,IMAGE_TAG,STATE_DIR |
| 95 | + script: | |
| 96 | + set -eu |
| 97 | +
|
| 98 | + # Marker: from here on the running deployment has been touched, so rollback is warranted on failure. |
| 99 | + # If a step before this fails, the service is still healthy and rollback must NOT touch it. |
| 100 | + touch "${STATE_DIR}/deploy_started" |
| 101 | + IMAGE_TAG="${IMAGE_TAG}" docker stack deploy \ |
| 102 | + --resolve-image always --prune \ |
| 103 | + -c "${COMPOSE}" "${STACK}" |
| 104 | +
|
| 105 | + - name: Health check |
| 106 | + timeout-minutes: 5 |
| 107 | + run: | |
| 108 | + set -euo pipefail |
| 109 | +
|
| 110 | + for i in $(seq 1 30); do |
| 111 | + if curl -fsS --max-time 5 "${HEALTH_URL}" >/dev/null; then |
| 112 | + echo "Healthy after ${i} check(s)"; exit 0 |
| 113 | + fi |
| 114 | + echo "Health check ${i}/30 not ready; waiting 5s" |
| 115 | + sleep 5 |
| 116 | + done |
| 117 | + echo "::error::Service did not become healthy" |
| 118 | + exit 1 |
| 119 | +
|
| 120 | + - name: Verify deployed version |
| 121 | + timeout-minutes: 5 |
| 122 | + run: | |
| 123 | + set -euo pipefail |
| 124 | +
|
| 125 | + for i in $(seq 1 30); do |
| 126 | + LIVE="$(curl -fsS --max-time 5 "${VERSION_URL}" || true)" |
| 127 | + echo "Attempt ${i}/30: /version -> '${LIVE:-<unreachable>}' (expected '${IMAGE_TAG}')" |
| 128 | + if [ "${LIVE}" = "${IMAGE_TAG}" ]; then |
| 129 | + echo "Confirmed live version: ${LIVE}"; exit 0 |
| 130 | + fi |
| 131 | + sleep 5 |
| 132 | + done |
| 133 | + echo "::error::Live /version never matched '${IMAGE_TAG}'" |
| 134 | + exit 1 |
| 135 | +
|
| 136 | + - name: Prune old service images |
| 137 | + if: success() |
| 138 | + timeout-minutes: 5 |
| 139 | + uses: appleboy/ssh-action@v1 |
| 140 | + with: |
| 141 | + host: ${{ secrets.HOST }} |
| 142 | + username: ${{ secrets.USERNAME }} |
| 143 | + password: ${{ secrets.PASSWORD }} |
| 144 | + port: ${{ secrets.PORT }} |
| 145 | + envs: IMAGE,IMAGE_TAG,STATE_DIR |
| 146 | + script: | |
| 147 | + set -eu |
| 148 | +
|
| 149 | + OLD_TAG="$(cat "${STATE_DIR}/old_tag" 2>/dev/null || true)" |
| 150 | + KEEP="${IMAGE_TAG} latest ${OLD_TAG}" |
| 151 | + echo "Keeping tags: ${KEEP}" |
| 152 | + docker images --format '{{.Repository}}:{{.Tag}}' \ |
| 153 | + | awk -v repo="${IMAGE}:" 'index($0, repo) == 1' \ |
| 154 | + | while read -r ref; do |
| 155 | + tag="${ref##*:}" |
| 156 | + keep=0 |
| 157 | + for k in ${KEEP}; do [ "${tag}" = "${k}" ] && keep=1 && break; done |
| 158 | + if [ "${keep}" = "1" ]; then |
| 159 | + echo "keep ${ref}" |
| 160 | + else |
| 161 | + echo "remove ${ref}" |
| 162 | + docker rmi "${ref}" || echo "::warning::could not remove ${ref} (still in use?)" |
| 163 | + fi |
| 164 | + done |
| 165 | + # Mop up layers left dangling by the removed/replaced tags. |
| 166 | + docker image prune -f >/dev/null 2>&1 || true |
| 167 | +
|
| 168 | + - name: Rollback on failure |
| 169 | + if: failure() |
| 170 | + timeout-minutes: 10 |
| 171 | + uses: appleboy/ssh-action@v1 |
| 172 | + with: |
| 173 | + host: ${{ secrets.HOST }} |
| 174 | + username: ${{ secrets.USERNAME }} |
| 175 | + password: ${{ secrets.PASSWORD }} |
| 176 | + port: ${{ secrets.PORT }} |
| 177 | + envs: IMAGE,SERVICE,IMAGE_TAG,STATE_DIR,HEALTH_URL,VERSION_URL |
| 178 | + script: | |
| 179 | + set -eu |
| 180 | +
|
| 181 | + # Only roll back if we actually touched the deployment. If a step before "Deploy new tag" failed, |
| 182 | + # the original service is still running and healthy - touching it would cause a needless outage. |
| 183 | + if [ ! -f "${STATE_DIR}/deploy_started" ]; then |
| 184 | + echo "Deploy was never started; deployment unchanged. Skipping rollback." |
| 185 | + exit 0 |
| 186 | + fi |
| 187 | +
|
| 188 | + OLD_IMAGE_REF="$(cat "${STATE_DIR}/old_image_ref" 2>/dev/null || true)" |
| 189 | + OLD_REPLICAS="$(cat "${STATE_DIR}/old_replicas" 2>/dev/null || true)" |
| 190 | + echo "::error::Deploy of ${IMAGE}:${IMAGE_TAG} failed - rolling back to ${OLD_IMAGE_REF:-<unknown>}" |
| 191 | +
|
| 192 | + # Restore the EXACT previous image by digest (never a tag that may now resolve to the broken build), |
| 193 | + # then scale back to the captured replica count. |
| 194 | + if [ -z "${OLD_IMAGE_REF}" ]; then |
| 195 | + echo "::error::No previous image captured; cannot auto-restore the image. Manual intervention required." |
| 196 | + exit 1 |
| 197 | + fi |
| 198 | + docker pull "${OLD_IMAGE_REF}" || echo "::warning::could not pull ${OLD_IMAGE_REF}; relying on local copy" |
| 199 | + docker service update --force --image "${OLD_IMAGE_REF}" "${SERVICE}" |
| 200 | + docker service scale "${SERVICE}=${OLD_REPLICAS:-1}" |
| 201 | +
|
| 202 | + # Verify the restored image actually came back healthy. A failed rollback must be distinguishable from a |
| 203 | + # failed deploy - never leave the service silently down believing we recovered. |
| 204 | + if ! command -v curl >/dev/null 2>&1; then |
| 205 | + echo "::warning::curl unavailable on host; cannot verify rollback health. Restored image=${OLD_IMAGE_REF}. Confirm /healthz manually." |
| 206 | + exit 0 |
| 207 | + fi |
| 208 | + echo "Verifying rollback health at ${HEALTH_URL}" |
| 209 | + for i in $(seq 1 30); do |
| 210 | + if curl -fsS --max-time 5 "${HEALTH_URL}" >/dev/null 2>&1; then |
| 211 | + LIVE="$(curl -fsS --max-time 5 "${VERSION_URL}" 2>/dev/null || true)" |
| 212 | + echo "Rollback healthy after ${i} check(s); live /version='${LIVE:-<unknown>}'" |
| 213 | + exit 0 |
| 214 | + fi |
| 215 | + echo "Rollback health ${i}/30 not ready; waiting 5s" |
| 216 | + sleep 5 |
| 217 | + done |
| 218 | + echo "::error::Service did NOT become healthy after rollback. Manual intervention required." |
| 219 | + exit 1 |
0 commit comments