Add dynamic Job naming and TTL increase

Add dynamic Job name suffixes to Kubernetes Jobs due to
immutable jobs causing Helm upgrade failures.
Inrease ttlSecondsAfterFinished to 43200s to allow for troubleshooting now that
Jobs no longer require deleting.
Adds a CHANGELOG entries under "New Features/Updated functionality" describing
the Release.Revision suffix applied to all three Helm Job names and the
ttlSecondsAfterFinished increase to 43200 s (12 h).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: lee-baines

CHANGELOG.md

@@ -43,7 +43,18 @@ You can now pass a values file to `forgeops prereqs` for cert-manager and
 ingress. This allows users to provide extra values as needed. See `forgeops
 prereqs -h` for more info.
 
+### Increased TTL for Kubernetes jobs (Helm only)
+
+The default `ttlSecondsAfterFinished` for all three jobs has been increased from 7200s
+(2 hours) to 43200s (12 hours), giving more time to inspect completed or failed jobs
+before they are automatically cleaned up.
+
 ## Bugfixes
 
-## How-tos
+### Dynamic Kubernetes job naming (Helm only)
 
+The `amster`, `ds-set-passwords`, and `keystore-create` Kubernetes job names in the
+`identity-platform` Helm chart now include the Helm release revision as a suffix
+(e.g. `amster-3`, `ds-set-passwords-3`, `keystore-create-3`). This ensures each
+`helm upgrade` produces a distinct job name, preventing failures caused by Kubernetes'
+immutability constraint on existing jobs.

bin/commands/wait

@@ -102,6 +102,31 @@ waitResourceExists() {
   timeoutExit
 }
 
+# Returns the name of the most recently created amster Job, polling until one exists.
+# Progress messages go to stderr so stdout carries only the job name for capture.
+getLatestAmsterJob() {
+  local start_time=$(date +%s)
+  local now=$(date +%s)
+  local job_name
+
+  echo -n "Waiting for amster Job to exist..." >&2
+  while [ $(($now - $start_time)) -le $TIMEOUT ] ; do
+    job_name=$($K_GET jobs -l app.kubernetes.io/component=amster \
+      --sort-by=.metadata.creationTimestamp \
+      -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null)
+    if [ -n "$job_name" ] ; then
+      echo " $job_name" >&2
+      echo "$job_name"
+      return 0
+    fi
+    echo -n '.' >&2
+    sleep 1
+    now=$(date +%s)
+  done
+
+  timeoutExit
+}
+
 waitForDS() {
   local ds=$1
 
@@ -160,12 +185,14 @@ for c in ${COMPONENTS[@]} ; do
       ;;
 
     amster)
-      waitForResource 'complete' job/amster
+      amster_job=$(getLatestAmsterJob)
+      waitForResource 'complete' "job/${amster_job}"
       ;;
 
     apps)
       waitForResource ready "pod -l app=am"
-      waitForResource 'complete' job/amster
+      amster_job=$(getLatestAmsterJob)
+      waitForResource 'complete' "job/${amster_job}"
       waitForResource ready "pod -l app=idm"
       ;;
 

charts/README.md

@@ -57,4 +57,3 @@ $ helm upgrade identity-platform . \
 ```bash
 $ helm delete identity-platform -n identity-platform
 ```
-

charts/identity-platform/templates/amster-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: amster
+  name: amster-{{ .Release.Revision }}
   annotations:
     "helm.sh/hook": post-install,post-upgrade
     {{- if .Values.amster.deleteOnSuccess }}

charts/identity-platform/templates/ds-set-passwords-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: ds-set-passwords
+  name: ds-set-passwords-{{ .Release.Revision }}
   labels:
     {{- include "identity-platform.labels" . | nindent 4 }}
     {{- with .Values.ds_set_passwords.podLabels }}

charts/identity-platform/templates/keystore-create-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: keystore-create
+  name: keystore-create-{{ .Release.Revision }}
   labels:
     {{- include "identity-platform.labels" . | nindent 4 }}
     {{- with .Values.keystore_create.podLabels }}

charts/identity-platform/values.yaml

@@ -301,7 +301,7 @@ amster:
   backoffLimit: 6
   restartPolicy: OnFailure
   deleteOnSuccess: false
-  ttlSecondsAfterFinished: 7200
+  ttlSecondsAfterFinished: 43200
 
   image:
     repository: us-docker.pkg.dev/forgeops-public/images/amster
@@ -350,7 +350,7 @@ ds_set_passwords:
   force: false
   backoffLimit: 6
   restartPolicy: OnFailure
-  ttlSecondsAfterFinished: 7200
+  ttlSecondsAfterFinished: 43200
 
   image:
     repository: us-docker.pkg.dev/forgeops-public/images/ds
@@ -499,7 +499,7 @@ keystore_create:
   backoffLimit: 6
   restartPolicy: OnFailure
   deleteOnSuccess: false
-  ttlSecondsAfterFinished: 7200
+  ttlSecondsAfterFinished: 43200
   clusterRoleName: identity-platform-keystore-create
   appendNSToRole: true
 

lib/python/utils.py

@@ -503,7 +503,7 @@ def get_secret_value(ns, secret, key):
 # Clean up amster resources.
 def clean_amster_job(ns):
     message(f'Cleaning up amster components.')
-    run('kubectl', f'-n {ns} delete --ignore-not-found=true job amster')
+    run('kubectl', f'-n {ns} delete --ignore-not-found=true job -l app.kubernetes.io/component=amster')
     run('kubectl', f'-n {ns} delete --wait=true --ignore-not-found=true cm amster-scripts')
     run('kubectl', f'-n {ns} delete --wait=true --ignore-not-found=true cm amster-function')
     run('kubectl', f'-n {ns} delete --wait=true --ignore-not-found=true cm amster-config')