Replace Bitnami Redis with Dandydev Redis-HA

.github/workflows/helm-lint-test.yaml

@@ -20,7 +20,7 @@ permissions:
   contents: read
 
 jobs:
-  lint-test:
+  lint-serverside-dryrun:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

Chart.yaml

@@ -5,19 +5,20 @@ icon: https://ctfd.io/static/img/ctfd.svg
 
 type: application
 
+# Dev note: trigger a helm chart release by bumping the version
 version: 0.2.0
 
 # Redis, MySQL/MariaDB and Seaweedfs
 dependencies:
-  - name: redis
-    version: 20.0.5
-    repository: https://charts.bitnami.com/bitnami
-    condition: redis.enabled
+  - name: redis-ha
+    version: 4.27.6
+    repository: https://dandydeveloper.github.io/charts/
+    condition: redis-ha.enabled
   - name: mariadb-galera
     version: 14.0.12
     repository: https://charts.bitnami.com/bitnami
     condition: mariadb-galera.enabled
   - name: seaweedfs
-    version: 4.0.0
+    version: 4.0.380
     repository: https://seaweedfs.github.io/seaweedfs/helm
     condition: seaweedfs.enabled

README.md

@@ -154,7 +154,7 @@ ctfd:
 | Repository | Name | Version |
 |------------|------|---------|
 | https://charts.bitnami.com/bitnami | mariadb-galera | 14.0.12 |
-| https://charts.bitnami.com/bitnami | redis | 20.0.5 |
+| https://dandydeveloper.github.io/charts/ | redis-ha | 4.27.6 |
 | https://seaweedfs.github.io/seaweedfs/helm | seaweedfs | 4.0.0 |
 
 ## Values
@@ -163,8 +163,8 @@ ctfd:
 |-----|------|---------|-------------|
 | ctfd.affinity | object | `{}` | CTFd affinity |
 | ctfd.autoscaling.enabled | bool | `true` | Enables HPA autoscaling |
-| ctfd.autoscaling.maxReplicas | int | `10` | Autoscaling max replicas |
-| ctfd.autoscaling.minReplicas | int | `2` | Autoscaling min replicas |
+| ctfd.autoscaling.maxReplicas | int | `20` | Autoscaling max replicas |
+| ctfd.autoscaling.minReplicas | int | `3` | Autoscaling min replicas |
 | ctfd.autoscaling.targetCPUUtilizationPercentage | int | `80` | Autoscaling target CPU utilization percentage |
 | ctfd.autoscaling.targetMemoryUtilizationPercentage | int | `80` | Autoscaling target memory utilization percentage |
 | ctfd.fullnameOverride | string | `""` | Chart fullname override |
@@ -186,7 +186,7 @@ ctfd:
 | ctfd.podLabels | object | `{}` | CTFd pod labels |
 | ctfd.podSecurityContext | object | `{}` | CTFd pod security context |
 | ctfd.readinessProbe | object | Check `values.yaml` | CTFd readiness probe |
-| ctfd.replicaCount | int | `2` | CTFd replica count (If autoscaling is enabled, this value is ignored) |
+| ctfd.replicaCount | int | `3` | CTFd replica count (If autoscaling is enabled, this value is ignored) |
 | ctfd.resources.limits.cpu | string | `"2"` | CTFd pod CPU limit |
 | ctfd.resources.limits.memory | string | `"2Gi"` | CTFd pod memory limit |
 | ctfd.resources.requests.cpu | string | `"1"` | CTFd pod CPU request |
@@ -206,43 +206,63 @@ ctfd:
 | ctfd.uploadprovider.s3.secret_access_key | string | `""` | AWS S3 bucket access key |
 | ctfd.volumeMounts | list | `[]` | CTFd volumeMounts |
 | ctfd.volumes | list | `[]` | CTFd volumes |
-| mariadb-galera.db.name | string | `"ctfd"` |  |
-| mariadb-galera.db.password | string | `"ctfd"` |  |
-| mariadb-galera.db.user | string | `"ctfd"` |  |
+| mariadb-galera.db.name | string | `"ctfd"` | ctfd database name |
+| mariadb-galera.db.password | string | `"ctfd"` | ctfd database password |
+| mariadb-galera.db.user | string | `"ctfd"` | ctfd database user |
 | mariadb-galera.enabled | bool | `true` | Deploys bitnami's mariadb-galera (set to false if you want to use an external database) |
 | mariadb-galera.external | object | ignored | External database connection details. Takes effect if `mariadb.enabled` is set to false |
-| mariadb-galera.extraFlags | string | Check `values.yaml`. Used by official CTFd `docker-compose.yml` | MariaDB primary entrypoint extra flags |
-| mariadb-galera.galera.mariabackup.password | string | `"ctfd"` |  |
-| mariadb-galera.metrics.enabled | bool | `true` |  |
+| mariadb-galera.extraFlags | string | Check `values.yaml`. Used by official CTFd `docker-compose.yml` | primary entrypoint extra flags |
+| mariadb-galera.galera | object | `{"mariabackup":{"password":"ctfd"}}` | backup user (This is required by the subchart to do helm upgrades) |
+| mariadb-galera.galera.mariabackup.password | string | `"ctfd"` | backup user (This is required by the subchart to do helm upgrades) |
+| mariadb-galera.metrics.enabled | bool | `false` |  |
 | mariadb-galera.persistence.enabled | bool | `true` |  |
-| mariadb-galera.persistence.size | string | `"2Gi"` |  |
-| mariadb-galera.resourcesPreset | string | `"large"` |  |
-| mariadb-galera.rootUser.password | string | `"ctfd"` |  |
-| redis.auth.enabled | bool | `false` |  |
-| redis.enabled | bool | `true` | Deploys bitnami's redis (set to false if you want to use an external cache) |
-| redis.external | object | ignored | External redis cache connection details. Takes effect if `redis.enabled` is set to false |
-| redis.master.count | int | `1` |  |
-| redis.master.persistence.enabled | bool | `false` |  |
-| redis.master.resourcesPreset | string | `"micro"` | Check Bintami's documentation |
-| redis.metrics.enabled | bool | `true` |  |
-| redis.replica.autoscaling.enabled | bool | `true` |  |
-| redis.replica.autoscaling.targetCPU | string | `"80"` |  |
-| redis.replica.persistence.enabled | bool | `false` |  |
-| redis.replica.resourcesPreset | string | `"micro"` | Check Bintami's documentation |
-| redis.sysctl.enabled | bool | `true` |  |
-| redis.volumePermissions.enabled | bool | `true` |  |
+| mariadb-galera.persistence.size | string | `"2Gi"` | PVC size |
+| mariadb-galera.replicaCount | int | `3` | Number of primary nodes replicas |
+| mariadb-galera.resourcesPreset | string | `"large"` | request and limits preset (check bitnami's mariadb-galera chart for details) |
+| mariadb-galera.rootUser.password | string | `"ctfd"` | root user |
+| redis-ha.additionalAffinities | object | `{}` | Additional affinities to add to the Redis server pods. |
+| redis-ha.affinity | string | `""` | Assign custom [affinity] rules to the Redis pods. |
+| redis-ha.auth | bool | `false` | Configures redis-ha with AUTH |
+| redis-ha.containerSecurityContext | object | See [values.yaml] | Redis HA statefulset container-level security context |
+| redis-ha.enabled | bool | `true` | Enables the Redis HA subchart and disables the custom Redis single node deployment |
+| redis-ha.exporter.enabled | bool | `false` | Enable Prometheus redis-exporter sidecar |
+| redis-ha.exporter.image | string | `"public.ecr.aws/bitnami/redis-exporter"` | Repository to use for the redis-exporter |
+| redis-ha.exporter.tag | string | `"1.58.0"` | Tag to use for the redis-exporter |
+| redis-ha.external | object | ignored | External redis cache connection details. Takes effect if `redis-ha.enabled` is set to false |
+| redis-ha.haproxy.additionalAffinities | object | `{}` | Additional affinities to add to the haproxy pods. |
+| redis-ha.haproxy.affinity | string | `""` | Assign custom [affinity] rules to the haproxy pods. |
+| redis-ha.haproxy.containerSecurityContext | object | See [values.yaml] | HAProxy container-level security context |
+| redis-ha.haproxy.enabled | bool | `true` | Enabled HAProxy LoadBalancing/Proxy |
+| redis-ha.haproxy.hardAntiAffinity | bool | `true` | Whether the haproxy pods should be forced to run on separate nodes. |
+| redis-ha.haproxy.metrics.enabled | bool | `true` | HAProxy enable prometheus metric scraping |
+| redis-ha.haproxy.replicas | int | `3` | HAProxy replicas |
+| redis-ha.haproxy.tolerations | list | `[]` | [Tolerations] for use with node taints for haproxy pods. |
+| redis-ha.hardAntiAffinity | bool | `true` | Whether the Redis server pods should be forced to run on separate nodes. |
+| redis-ha.image.repository | string | `"public.ecr.aws/docker/library/redis"` | Redis repository |
+| redis-ha.image.tag | string | `"7.4.1-alpine"` | Redis tag |
+| redis-ha.persistentVolume.enabled | bool | `false` | Configures persistence on Redis nodes |
+| redis-ha.redis.config | object | See [values.yaml] | Any valid redis config options in this section will be applied to each server (see `redis-ha` chart) |
+| redis-ha.redis.config.save | string | `'""'` | Will save the DB if both the given number of seconds and the given number of write operations against the DB occurred. `""`  is disabled |
+| redis-ha.redis.masterGroupName | string | `"ctfd"` | Redis convention for naming the cluster group: must match `^[\\w-\\.]+$` and can be templated |
+| redis-ha.replicas | int | `3` | Redis-HA replicas |
+| redis-ha.tolerations | list | `[]` | [Tolerations] for use with node taints for Redis pods. |
+| redis-ha.topologySpreadConstraints | object | `{"enabled":false,"maxSkew":"","topologyKey":"","whenUnsatisfiable":""}` | Assign custom [TopologySpreadConstraints] rules to the Redis pods. # https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ |
+| redis-ha.topologySpreadConstraints.enabled | bool | `false` | Enable Redis HA topology spread constraints |
+| redis-ha.topologySpreadConstraints.maxSkew | string | `""` (defaults to `1`) | Max skew of pods tolerated |
+| redis-ha.topologySpreadConstraints.topologyKey | string | `""` (defaults to `topology.kubernetes.io/zone`) | Topology key for spread |
+| redis-ha.topologySpreadConstraints.whenUnsatisfiable | string | `""` (defaults to `ScheduleAnyway`) | Enforcement policy, hard or soft |
 | seaweedfs.enabled | bool | `true` | Deploys seaweedfs (set to false if you want to use an bucket) |
 | seaweedfs.filer.data.size | string | `"5Gi"` | seaweedfs-filer storage size |
 | seaweedfs.filer.data.type | string | `"persistentVolumeClaim"` | seaweedfs-filer data storage type |
 | seaweedfs.filer.enablePVC | bool | `true` | seaweedfs-filer enable PVC for data persistence |
-| seaweedfs.filer.replicas | int | `1` | seaweedfs-filer replicas |
+| seaweedfs.filer.replicas | int | `3` | seaweedfs-filer replicas |
 | seaweedfs.filer.s3.createBuckets | list | `[{"name":"ctfd-bucket"}]` | seaweedfs-s3 create bucket upon deploying |
 | seaweedfs.filer.s3.enableAuth | bool | `false` | seaweedfs-s3 enable authentication (no need since seaweed is private to the cluster) |
 | seaweedfs.filer.s3.enabled | bool | `true` | seaweedfs-s3 enable. This enables S3 API (Should be left to `true`) |
 | seaweedfs.filer.storage | string | `"5Gi"` | seaweedfs-filer PVC storage size |
 | seaweedfs.master.data.size | string | `"5Gi"` | seaweedfs storage size |
 | seaweedfs.master.data.type | string | `"persistentVolumeClaim"` | seaweedfs data storage type |
-| seaweedfs.master.replicas | int | `1` | seaweedfs-master replicas |
-| seaweedfs.volume.replicas | int | `1` | seaweedfs-volume replicas |
+| seaweedfs.master.replicas | int | `3` | seaweedfs-master replicas |
+| seaweedfs.volume.replicas | int | `3` | seaweedfs-volume replicas |
 
 Autogenerated from chart metadata using [helm-docs](https://github.com/norwoodj/helm-docs)

templates/NOTES.txt

@@ -20,8 +20,8 @@
                                                                      version: {{ .Values.ctfd.image.tag | default .Chart.AppVersion }}
 
 
-{{ if or (index .Values "mariadb-galera" "enabled") (.Values.redis.enabled) -}}
-** Please be patient while MariaDB or Redis are being deployed **
+{{ if or (index .Values "mariadb-galera" "enabled") (index .Values "redis-ha" "enabled") -}}
+** Please be patient while MariaDB and/or Redis are being deployed **
 {{ end }}
 
 Get the list of pods by executing:

templates/_helpers.tpl

@@ -78,9 +78,9 @@ mysql+pymysql://{{ index .Values "mariadb-galera" "external" "username" }}:{{ in
    Generate CTFd REDIS_URL (internal bitnami redis or external self managed redis)
 */}}
 {{- define "ctfd.REDIS_URL" -}}
-{{- if .Values.redis.enabled -}}
-redis://{{ .Release.Name }}-redis-master:6379
+{{- if index .Values "redis-ha" "enabled" -}}
+redis://{{ .Release.Name }}-redis-ha-haproxy:6379
 {{- else -}}
-redis://{{ .Values.redis.external.username }}:{{ .Values.redis.external.password }}@{{ .Values.redis.external.host }}:{{ .Values.redis.external.port }}
+redis://{{ index .Values "redis-ha" "external" "username" }}:{{ index .Values "redis-ha" "external" "password" }}@{{ index .Values "redis-ha" "external" "host" }}:{{ index .Values "redis-ha" "external" "port" }}
 {{- end -}}
 {{- end -}}

templates/deployment.yaml

@@ -21,6 +21,9 @@ spec:
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
         checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        {{- if .Values.seaweedfs.enabled }}
+        checksum/s3secret: {{ include (print $.Template.BasePath "/seaweedfs-s3-secret.yaml") . | sha256sum }}
+        {{- end }}
       {{- with .Values.ctfd.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
@@ -39,6 +42,20 @@ spec:
       securityContext:
         {{- toYaml .Values.ctfd.podSecurityContext | nindent 8 }}
       {{- end }}
+      initContainers:
+        - name: wait-for-sentinel-quorum
+          image: redis:7-alpine
+          command:
+            - 'sh'
+            - '-c'
+            - >
+              until redis-cli -h {{ .Release.Name }}-redis-ha -p 26379 sentinel ckquorum ctfd;
+              do echo "Waiting for Sentinel quorum...";
+              sleep 10;
+              done
+      {{- with .Values.ctfd.initContainers }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
@@ -52,6 +69,25 @@ spec:
               name: {{ include "ctfd.fullname" . }}-secret-key
           - configMapRef:
               name: {{ include "ctfd.fullname" . }}
+          {{- if .Values.seaweedfs.enabled }}
+          env:
+          - name: AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: seaweedfs-s3-secret
+                key: admin_access_key_id
+          - name: AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: seaweedfs-s3-secret
+                key: admin_secret_access_key
+          - name: AWS_S3_BUCKET
+            value: {{ (index .Values.seaweedfs.filer.s3.createBuckets 0).name }}
+          - name: AWS_S3_ENDPOINT_URL
+            value: http://seaweedfs-s3:8333
+          - name: AWS_S3_CUSTOM_DOMAIN
+            value: {{ .Values.seaweedfs.s3.ingress.host }}
+          {{- end }}
           ports:
             - name: http
               containerPort: {{ .Values.ctfd.service.port }}
@@ -66,10 +102,6 @@ spec:
           volumeMounts:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-      {{- with .Values.ctfd.initContainers }}
-      initContainers:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
       {{- with .Values.ctfd.volumes }}
       volumes:
         {{- toYaml . | nindent 8 }}

templates/seaweedfs-s3-secret.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.seaweedfs.enabled }}
+{{- $access_key_admin := randAlphaNum 16 -}}
+{{- $secret_key_admin := randAlphaNum 32 -}}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ .Values.seaweedfs.s3.existingConfigSecret }}
+stringData:
+  admin_access_key_id: {{ $access_key_admin }}
+  admin_secret_access_key: {{ $secret_key_admin }}
+  seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anonymous","actions":["Read"]}]}'
+{{- end }}

templates/secret.yaml

@@ -12,9 +12,3 @@ data:
   AWS_S3_BUCKET: {{ .Values.ctfd.uploadprovider.s3.bucket | b64enc }}
   AWS_S3_ENDPOINT_URL: {{ .Values.ctfd.uploadprovider.s3.endpoint_url | b64enc }}
 {{- end }}
-{{- if .Values.seaweedfs.enabled }}
-  AWS_ACCESS_KEY_ID: {{ "seaweedfs" | b64enc }}
-  AWS_SECRET_ACCESS_KEY: {{ "seaweedfs" | b64enc }}
-  AWS_S3_BUCKET: {{ "ctfd-bucket" | b64enc }}
-  AWS_S3_ENDPOINT_URL: {{ "http://seaweedfs-s3:8333" | b64enc }}
-{{- end }}