fix(services/keycloak): refresh bootstrap credentials when secret rotates server-side

Author: kmein

services/keycloak/README.md

@@ -54,9 +54,12 @@ realm). The pairing offers two paths:
    `admin` composite role, and persists the
    resulting `client_id`/`client_secret` 0600 under
    `/var/lib/declarative-keycloak-bootstrap/`. The reconciler picks them up
-   via systemd `LoadCredential=` on every run; the file pair is the
-   "already bootstrapped" marker, so the oneshot is a no-op on subsequent
-   boots.
+   via systemd `LoadCredential=` on every run. On subsequent boots the
+   oneshot probes the saved pair against keycloak's token endpoint --
+   if the secret has been rotated server-side (e.g. through the admin
+   console) the pair is refreshed in place from the existing client.
+   Recovery is therefore a `systemctl restart declarative-keycloak-bootstrap`
+   away.
 
 2. **Operator-supplied.** Set both `clientIdFile` and `clientSecretFile` to
    host paths for a service-account client you've created externally. The

services/keycloak/module.nix

@@ -182,12 +182,6 @@ in
           client_id_file="$STATE_DIRECTORY/client_id"
           client_secret_file="$STATE_DIRECTORY/client_secret"
 
-          # the saved credential pair is the "already bootstrapped"
-          # marker. /var/lib persists across reboots.
-          if [ -s "$client_id_file" ] && [ -s "$client_secret_file" ]; then
-            exit 0
-          fi
-
           # wait up to 3 minutes for keycloak to come up.
           for _ in $(seq 1 90); do
             if curl -fsS -o /dev/null "${cfg.baseUrl}/realms/master"; then
@@ -196,6 +190,23 @@ in
             sleep 2
           done
 
+          # saved credential pair exists -- probe it against the token
+          # endpoint. valid means already-bootstrapped; otherwise the
+          # secret has been rotated server-side and we fall through to
+          # re-fetch the current one (same client, same id).
+          if [ -s "$client_id_file" ] && [ -s "$client_secret_file" ]; then
+            saved_id="$(cat "$client_id_file")"
+            saved_secret="$(cat "$client_secret_file")"
+            if curl -fsS -o /dev/null -X POST \
+                 ${lib.escapeShellArg "${cfg.baseUrl}/realms/master/protocol/openid-connect/token"} \
+                 --data-urlencode 'grant_type=client_credentials' \
+                 --data-urlencode "client_id=$saved_id" \
+                 --data-urlencode "client_secret=$saved_secret"; then
+              exit 0
+            fi
+            echo "saved service-account credentials no longer accepted; refreshing from keycloak admin." >&2
+          fi
+
           kcadm.sh config credentials \
             --server ${lib.escapeShellArg cfg.baseUrl} \
             --realm master \