feat(helm): add support for user-managed SASL secrets and dynamic secret handling for Kafka, topic initialization, and MirrorMaker

CoderYellow · CoderYellow · commit b6099d86f3dd · 2026-05-01T15:39:52.000+08:00
diff --git a/kafka/README.md b/kafka/README.md
@@ -43,6 +43,77 @@ kafka:
 
 With persistence disabled, the broker data path is still mounted, but it uses `emptyDir` and is lost when the pod is deleted.
 
+## User-managed SASL secrets
+
+For SASL/PLAIN, keep credentials in Kubernetes Secrets and point the chart at those secrets instead of putting JAAS strings in values files.
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kafka-auth
+type: Opaque
+stringData:
+  kafka-server-jaas.conf: |
+    KafkaServer {
+      org.apache.kafka.common.security.plain.PlainLoginModule required
+      username="admin"
+      password="admin-secret"
+      user_admin="admin-secret"
+      user_kafbat-ui="ui-secret"
+      user_schema-registry="schema-secret"
+      user_mm2="mm2-secret";
+    };
+  schema-registry-jaas.conf: |
+    org.apache.kafka.common.security.plain.PlainLoginModule required username="schema-registry" password="schema-secret";
+  kafbat-ui-jaas.conf: |
+    org.apache.kafka.common.security.plain.PlainLoginModule required username="kafbat-ui" password="ui-secret";
+  client-secret.properties: |
+    sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret";
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kafka-mm2
+type: Opaque
+stringData:
+  mm2-secret.properties: |
+    primary.sasl.jaas.config = org.apache.kafka.common.security.plain.PlainLoginModule required username="mm2" password="mm2-secret";
+    standby.sasl.jaas.config = org.apache.kafka.common.security.plain.PlainLoginModule required username="mm2" password="mm2-secret";
+```
+
+Then reference them from values:
+
+```yaml
+kafka:
+  jaas:
+    enabled: true
+    existingSecret: kafka-auth
+    existingSecretKey: kafka-server-jaas.conf
+
+topicInit:
+  commandConfig: |
+    security.protocol=SASL_PLAINTEXT
+    sasl.mechanism=PLAIN
+  commandConfigSecretExistingSecret: kafka-auth
+  commandConfigSecretExistingSecretKey: client-secret.properties
+
+mirrorMaker:
+  properties: |
+    clusters = primary, standby
+    primary.bootstrap.servers = kafka-kafka-primary-0.kafka-kafka-primary-headless.kafka.svc.cluster.local:9092,kafka-kafka-primary-1.kafka-kafka-primary-headless.kafka.svc.cluster.local:9092
+    standby.bootstrap.servers = kafka-kafka-standby-0.kafka-kafka-standby-headless.kafka.svc.cluster.local:9092,kafka-kafka-standby-1.kafka-kafka-standby-headless.kafka.svc.cluster.local:9092
+    primary.security.protocol = SASL_PLAINTEXT
+    primary.sasl.mechanism = PLAIN
+    standby.security.protocol = SASL_PLAINTEXT
+    standby.sasl.mechanism = PLAIN
+    primary->standby.enabled = true
+    primary->standby.topics = .*
+    replication.policy.class = org.apache.kafka.connect.mirror.IdentityReplicationPolicy
+  secretPropertiesExistingSecret: kafka-mm2
+  secretPropertiesExistingSecretKey: mm2-secret.properties
+```
+
 ## Multi-cluster example
 
 ```yaml
@@ -83,6 +154,7 @@ mirrorMaker:
 
     primary->standby.enabled = true
     primary->standby.topics = .*
+    replication.policy.class = org.apache.kafka.connect.mirror.IdentityReplicationPolicy
     replication.factor = 1
     checkpoints.topic.replication.factor = 1
     heartbeats.topic.replication.factor = 1
diff --git a/kafka/templates/kafka-statefulset.yaml b/kafka/templates/kafka-statefulset.yaml
@@ -170,11 +170,15 @@ spec:
               export KAFKA_OPTS="-Djava.security.krb5.conf={{ $root.Values.kerberos.krb5ConfigPath }} -Djava.security.auth.login.config={{ default $root.Values.kafka.kerberos.jaasPath (dig "kerberos" "jaasPath" nil $cluster) }} ${KAFKA_OPTS:-}"
               {{- end }}
               {{- if (default $root.Values.kafka.jaas.enabled (dig "jaas" "enabled" nil $cluster)) }}
+              {{- if (default $root.Values.kafka.jaas.existingSecret (dig "jaas" "existingSecret" nil $cluster)) }}
+              export KAFKA_OPTS="-Djava.security.auth.login.config={{ default $root.Values.kafka.jaas.path (dig "jaas" "path" nil $cluster) }} ${KAFKA_OPTS:-}"
+              {{- else }}
               cat > {{ default $root.Values.kafka.jaas.path (dig "jaas" "path" nil $cluster) }} <<'EOF'
               {{- default $root.Values.kafka.jaas.config (dig "jaas" "config" nil $cluster) | nindent 14 }}
               EOF
               export KAFKA_OPTS="-Djava.security.auth.login.config={{ default $root.Values.kafka.jaas.path (dig "jaas" "path" nil $cluster) }} ${KAFKA_OPTS:-}"
               {{- end }}
+              {{- end }}
               exec /etc/kafka/docker/run
           ports:
             - name: {{ $clientName }}
@@ -204,6 +208,12 @@ spec:
               mountPath: {{ $root.Values.kerberos.mountPath }}
               readOnly: true
             {{- end }}
+            {{- if and (default $root.Values.kafka.jaas.enabled (dig "jaas" "enabled" nil $cluster)) (default $root.Values.kafka.jaas.existingSecret (dig "jaas" "existingSecret" nil $cluster)) }}
+            - name: kafka-jaas
+              mountPath: {{ default $root.Values.kafka.jaas.path (dig "jaas" "path" nil $cluster) }}
+              subPath: {{ default $root.Values.kafka.jaas.existingSecretKey (dig "jaas" "existingSecretKey" nil $cluster) }}
+              readOnly: true
+            {{- end }}
             {{- with (concat (default list $root.Values.kafka.extraVolumeMounts) (default list $cluster.extraVolumeMounts)) }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -233,6 +243,14 @@ spec:
           emptyDir:
             {{- toYaml (default $root.Values.kafka.persistence.emptyDir (dig "persistence" "emptyDir" nil $cluster)) | nindent 12 }}
         {{- end }}
+        {{- if and (default $root.Values.kafka.jaas.enabled (dig "jaas" "enabled" nil $cluster)) (default $root.Values.kafka.jaas.existingSecret (dig "jaas" "existingSecret" nil $cluster)) }}
+        - name: kafka-jaas
+          secret:
+            secretName: {{ default $root.Values.kafka.jaas.existingSecret (dig "jaas" "existingSecret" nil $cluster) }}
+            items:
+              - key: {{ default $root.Values.kafka.jaas.existingSecretKey (dig "jaas" "existingSecretKey" nil $cluster) }}
+                path: {{ default $root.Values.kafka.jaas.existingSecretKey (dig "jaas" "existingSecretKey" nil $cluster) }}
+        {{- end }}
         {{- if $root.Values.kerberos.enabled }}
         - name: kerberos
           {{- if and $root.Values.kerberos.kdc.enabled (not $root.Values.kerberos.existingSecret) }}
diff --git a/kafka/templates/mirror-maker.yaml b/kafka/templates/mirror-maker.yaml
@@ -1,4 +1,5 @@
 {{- if .Values.mirrorMaker.enabled }}
+{{- if not .Values.mirrorMaker.propertiesExistingSecret }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -10,6 +11,7 @@ data:
   mm2.properties: |
     {{- .Values.mirrorMaker.properties | nindent 4 }}
 ---
+{{- end }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -42,12 +44,28 @@ spec:
             - -ec
             - |
               export KAFKA_OPTS="-Djava.security.krb5.conf={{ .Values.kerberos.krb5ConfigPath }} ${KAFKA_OPTS:-}"
+              {{- if .Values.mirrorMaker.secretPropertiesExistingSecret }}
+              cat /etc/kafka/mm2-common.properties /etc/kafka/mm2-secret.properties > /tmp/mm2.properties
+              exec /opt/kafka/bin/connect-mirror-maker.sh /tmp/mm2.properties
+              {{- else }}
               exec /opt/kafka/bin/connect-mirror-maker.sh /etc/kafka/mm2.properties
+              {{- end }}
           volumeMounts:
             - name: mirror-maker-config
+              {{- if .Values.mirrorMaker.secretPropertiesExistingSecret }}
+              mountPath: /etc/kafka/mm2-common.properties
+              subPath: mm2.properties
+              {{- else }}
               mountPath: /etc/kafka/mm2.properties
               subPath: mm2.properties
+              {{- end }}
+              readOnly: true
+            {{- if .Values.mirrorMaker.secretPropertiesExistingSecret }}
+            - name: mirror-maker-secret-config
+              mountPath: /etc/kafka/mm2-secret.properties
+              subPath: mm2-secret.properties
               readOnly: true
+            {{- end }}
             {{- if .Values.kerberos.enabled }}
             - name: kerberos
               mountPath: {{ .Values.kerberos.mountPath }}
@@ -59,8 +77,24 @@ spec:
           {{- end }}
       volumes:
         - name: mirror-maker-config
+          {{- if .Values.mirrorMaker.propertiesExistingSecret }}
+          secret:
+            secretName: {{ .Values.mirrorMaker.propertiesExistingSecret }}
+            items:
+              - key: {{ .Values.mirrorMaker.propertiesExistingSecretKey }}
+                path: mm2.properties
+          {{- else }}
           configMap:
             name: {{ include "kafka.fullname" . }}-mirror-maker
+          {{- end }}
+        {{- if .Values.mirrorMaker.secretPropertiesExistingSecret }}
+        - name: mirror-maker-secret-config
+          secret:
+            secretName: {{ .Values.mirrorMaker.secretPropertiesExistingSecret }}
+            items:
+              - key: {{ .Values.mirrorMaker.secretPropertiesExistingSecretKey }}
+                path: mm2-secret.properties
+        {{- end }}
         {{- if .Values.kerberos.enabled }}
         - name: kerberos
           {{- if and .Values.kerberos.kdc.enabled (not .Values.kerberos.existingSecret) }}
diff --git a/kafka/templates/topic-init.yaml b/kafka/templates/topic-init.yaml
@@ -8,6 +8,7 @@
 {{- end }}
 {{- end }}
 {{- end }}
+{{- if not .Values.topicInit.commandConfigExistingSecret }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -19,6 +20,7 @@ data:
   client.properties: |
     {{- .Values.topicInit.commandConfig | nindent 4 }}
 ---
+{{- end }}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -48,18 +50,35 @@ spec:
             - -ec
             - |
               export KAFKA_OPTS="-Djava.security.krb5.conf={{ .Values.kerberos.krb5ConfigPath }} ${KAFKA_OPTS:-}"
+              {{- if .Values.topicInit.commandConfigSecretExistingSecret }}
+              cat /etc/kafka/client-common.properties /etc/kafka/client-secret.properties > /tmp/client.properties
+              COMMAND_CONFIG=/tmp/client.properties
+              {{- else }}
+              COMMAND_CONFIG=/etc/kafka/client.properties
+              {{- end }}
               for i in $(seq 1 {{ int .Values.topicInit.waitAttempts }}); do
-                /opt/kafka/bin/kafka-topics.sh --bootstrap-server {{ include "kafka.cluster.bootstrapServers" (dict "root" . "cluster" $targetCluster) }} --command-config /etc/kafka/client.properties --list >/dev/null && break
+                /opt/kafka/bin/kafka-topics.sh --bootstrap-server {{ include "kafka.cluster.bootstrapServers" (dict "root" . "cluster" $targetCluster) }} --command-config "${COMMAND_CONFIG}" --list >/dev/null && break
                 sleep {{ int .Values.topicInit.waitSeconds }}
               done
               {{- range .Values.topicInit.topics }}
-              /opt/kafka/bin/kafka-topics.sh --bootstrap-server {{ include "kafka.cluster.bootstrapServers" (dict "root" $ "cluster" $targetCluster) }} --command-config /etc/kafka/client.properties --create --topic {{ .name | quote }} --partitions {{ .partitions }} --replication-factor {{ .replicationFactor }} --if-not-exists
+              /opt/kafka/bin/kafka-topics.sh --bootstrap-server {{ include "kafka.cluster.bootstrapServers" (dict "root" $ "cluster" $targetCluster) }} --command-config "${COMMAND_CONFIG}" --create --topic {{ .name | quote }} --partitions {{ .partitions }} --replication-factor {{ .replicationFactor }} --if-not-exists
               {{- end }}
           volumeMounts:
             - name: topic-init-config
+              {{- if .Values.topicInit.commandConfigSecretExistingSecret }}
+              mountPath: /etc/kafka/client-common.properties
+              subPath: client.properties
+              {{- else }}
               mountPath: /etc/kafka/client.properties
               subPath: client.properties
+              {{- end }}
               readOnly: true
+            {{- if .Values.topicInit.commandConfigSecretExistingSecret }}
+            - name: topic-init-secret-config
+              mountPath: /etc/kafka/client-secret.properties
+              subPath: client-secret.properties
+              readOnly: true
+            {{- end }}
             {{- if .Values.kerberos.enabled }}
             - name: kerberos
               mountPath: {{ .Values.kerberos.mountPath }}
@@ -71,8 +90,24 @@ spec:
           {{- end }}
       volumes:
         - name: topic-init-config
+          {{- if .Values.topicInit.commandConfigExistingSecret }}
+          secret:
+            secretName: {{ .Values.topicInit.commandConfigExistingSecret }}
+            items:
+              - key: {{ .Values.topicInit.commandConfigExistingSecretKey }}
+                path: client.properties
+          {{- else }}
           configMap:
             name: {{ include "kafka.fullname" . }}-topic-init
+          {{- end }}
+        {{- if .Values.topicInit.commandConfigSecretExistingSecret }}
+        - name: topic-init-secret-config
+          secret:
+            secretName: {{ .Values.topicInit.commandConfigSecretExistingSecret }}
+            items:
+              - key: {{ .Values.topicInit.commandConfigSecretExistingSecretKey }}
+                path: client-secret.properties
+        {{- end }}
         {{- if .Values.kerberos.enabled }}
         - name: kerberos
           {{- if and .Values.kerberos.kdc.enabled (not .Values.kerberos.existingSecret) }}
diff --git a/kafka/values-sample.yaml b/kafka/values-sample.yaml
@@ -16,27 +16,10 @@ kafka:
     - User:schema-registry
     - User:mm2
 
-  config:
-    KAFKA_LISTENER_NAME_SASL_PLAINTEXT_PLAIN_SASL_JAAS_CONFIG: >-
-      org.apache.kafka.common.security.plain.PlainLoginModule required
-      username="admin"
-      password="admin-secret"
-      user_admin="admin-secret"
-      user_kafbat-ui="ui-secret"
-      user_schema-registry="schema-secret"
-      user_mm2="mm2-secret";
   jaas:
     enabled: true
-    config: |
-      KafkaServer {
-        org.apache.kafka.common.security.plain.PlainLoginModule required
-        username="admin"
-        password="admin-secret"
-        user_admin="admin-secret"
-        user_kafbat-ui="ui-secret"
-        user_schema-registry="schema-secret"
-        user_mm2="mm2-secret";
-      };
+    existingSecret: kafka-auth
+    existingSecretKey: kafka-server-jaas.conf
 
   clusters:
     - name: primary
@@ -55,12 +38,18 @@ schemaRegistry:
   env:
     SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
     SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
-    SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >-
-      org.apache.kafka.common.security.plain.PlainLoginModule required
-      username="schema-registry"
-      password="schema-secret";
+  extraEnv:
+    - name: SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG
+      valueFrom:
+        secretKeyRef:
+          name: kafka-auth
+          key: schema-registry-jaas.conf
 
 kafbatUi:
+  image:
+    repository: ghcr.io/openprojectx/kafka-ui
+    tag: latest
+    pullPolicy: Always
   enabled: true
   autoClusters: true
   clusters: []
@@ -69,43 +58,49 @@ kafbatUi:
   env:
     KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT
     KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM: PLAIN
-    KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG: >-
-      org.apache.kafka.common.security.plain.PlainLoginModule required
-      username="kafbat-ui"
-      password="ui-secret";
     KAFKA_CLUSTERS_1_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT
     KAFKA_CLUSTERS_1_PROPERTIES_SASL_MECHANISM: PLAIN
-    KAFKA_CLUSTERS_1_PROPERTIES_SASL_JAAS_CONFIG: >-
-      org.apache.kafka.common.security.plain.PlainLoginModule required
-      username="kafbat-ui"
-      password="ui-secret";
+  extraEnv:
+    - name: KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG
+      valueFrom:
+        secretKeyRef:
+          name: kafka-auth
+          key: kafbat-ui-jaas.conf
+    - name: KAFKA_CLUSTERS_1_PROPERTIES_SASL_JAAS_CONFIG
+      valueFrom:
+        secretKeyRef:
+          name: kafka-auth
+          key: kafbat-ui-jaas.conf
 
 topicInit:
   enabled: true
   kafkaCluster: primary
+  commandConfigSecretExistingSecret: kafka-auth
+  commandConfigSecretExistingSecretKey: client-secret.properties
   commandConfig: |
     security.protocol=SASL_PLAINTEXT
     sasl.mechanism=PLAIN
-    sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret";
 
 mirrorMaker:
   enabled: true
+  secretPropertiesExistingSecret: kafka-mm2
+  secretPropertiesExistingSecretKey: mm2-secret.properties
   properties: |
     clusters = primary, standby
-    
+
     primary.bootstrap.servers = kafka-kafka-primary-0.kafka-kafka-primary-headless.kafka.svc.cluster.local:9092,kafka-kafka-primary-1.kafka-kafka-primary-headless.kafka.svc.cluster.local:9092
     standby.bootstrap.servers = kafka-kafka-standby-0.kafka-kafka-standby-headless.kafka.svc.cluster.local:9092,kafka-kafka-standby-1.kafka-kafka-standby-headless.kafka.svc.cluster.local:9092
-    
+
     primary.security.protocol = SASL_PLAINTEXT
     primary.sasl.mechanism = PLAIN
-    primary.sasl.jaas.config = org.apache.kafka.common.security.plain.PlainLoginModule required username="mm2" password="mm2-secret";
-    
     standby.security.protocol = SASL_PLAINTEXT
     standby.sasl.mechanism = PLAIN
-    standby.sasl.jaas.config = org.apache.kafka.common.security.plain.PlainLoginModule required username="mm2" password="mm2-secret";
-    
+
     primary->standby.enabled = true
     primary->standby.topics = .*
+    replication.policy.class = org.apache.kafka.connect.mirror.IdentityReplicationPolicy
+    refresh.topics.interval.seconds = 10
+    sync.topic.configs.interval.seconds = 10
 
     standby->primary.enabled = false
     standby->primary.topics = $^
diff --git a/kafka/values.yaml b/kafka/values.yaml
