Better health checks

Author: isTravis

.github/workflows/deploy.yml

@@ -169,18 +169,27 @@ jobs:
             local svc="$1" timeout="${2:-300}"
             local end=$((SECONDS + timeout))
             while (( SECONDS < end )); do
-              local desired running state
-              desired="$(sudo docker service inspect "$svc" --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo "")"
-              running="$(sudo docker service ps "$svc" --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running' || true)"
+              local state
               state="$(sudo docker service inspect "$svc" --format '{{if .UpdateStatus}}{{.UpdateStatus.State}}{{end}}' 2>/dev/null || echo "")"
-              echo "  $svc: desired=$desired running=$running state=$state"
-              if [[ -n "$desired" && "$running" == "$desired" ]] && { [[ -z "$state" ]] || [[ "$state" == "completed" ]]; }; then
-                echo "  $svc rollout complete"
-                return 0
+              echo "  $svc: update_state=$state"
+              if [[ "$state" == "rollback_started" || "$state" == "rollback_completed" ]]; then
+                echo "  ERROR: $svc rolled back!"
+                sudo docker service ps "$svc" --no-trunc --format '{{.Name}} {{.CurrentState}} {{.Error}}' | head -10
+                return 1
               fi
-              sleep 5
+              if [[ "$state" == "completed" ]] || [[ -z "$state" ]]; then
+                # Verify all running tasks are healthy (not just started)
+                local unhealthy
+                unhealthy="$(sudo docker service ps "$svc" --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -cv '^Running' || true)"
+                if [[ "$unhealthy" == "0" ]]; then
+                  echo "  $svc rollout complete"
+                  return 0
+                fi
+              fi
+              sleep 10
             done
             echo "Rollout timeout for $svc"
+            sudo docker service ps "$svc" --no-trunc --format '{{.Name}} {{.CurrentState}} {{.Error}}' | head -10
             return 1
           }
 

docker-compose.yml

@@ -38,11 +38,11 @@ services:
     tmpfs:
       - /tmp:size=64m
     healthcheck:
-      test: ["CMD-SHELL", "node -e \"require('http').get('http://127.0.0.1:3000/api/health', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1));\""]
+      test: ["CMD-SHELL", "node -e \"const h=require('http'),check=(port,path)=>new Promise((res,rej)=>{h.get('http://127.0.0.1:'+port+path,r=>r.statusCode<400?res():rej()).on('error',rej)});Promise.all([check(3000,'/api/health'),check(4321,'/')]).then(()=>process.exit(0)).catch(()=>process.exit(1));\""]
       interval: 30s
       timeout: 5s
       retries: 3
-      start_period: 15s
+      start_period: 30s
     logging:
       driver: json-file
       options: