Migrate Redis PVC from gp2 to gp3 across all environments (#1019)

* Add production DB table dump workflow via K8s Job

Introduces a manually-triggered GitHub Actions workflow that spins up a
K8s Job in credreg-prod to pg_dump a specified table, uploads the result
to a dedicated private S3 bucket, and notifies Slack with a 1-hour
presigned download URL.

- New terraform module: db_dumps_s3 (private bucket, AES-256, 7-day expiry)
- IRSA application policy extended with PutObject on cer-db-dumps-prod
- github-oidc-widget gets s3:GetObject for presigned URL generation
- K8s Job template: postgres:16-alpine, uses existing app-secrets for DB creds
- GH Actions workflow: validates table name, creates job, waits, presigns, notifies Slack

* Fix db-table-dump job: mount main-app-config for POSTGRESQL_DATABASE and POSTGRESQL_USERNAME

* Fix db-table-dump: stream via psql COPY to S3, add optional where_clause input, bump deadline to 4h

* Fix Slack notification formatting: proper newlines and hyperlink for presigned URL

* Switch to pg_dump plain SQL format, remove where_clause

* Fix Slack message: rename CSV to SQL dump

* Make db-table-dump fire-and-forget: job handles Slack notifications autonomously

- Dedicated db-dump-service-account with 12h IRSA token expiration
- Job posts started/done/failed to Slack independently
- GH Actions exits after launching job and posting started notification
- Add s3:GetObject to IRSA policy for presigned URL generation from the job
- Add db_dump_service_account_prod to IRSA trust policy

* Simplify: reuse main-app-service-account, drop dedicated dump SA

* Fix variables.tf: restore missing closing brace

* Add dedicated db-dump-service-account with 12h IRSA token for long-running dumps

* Migrate Redis PVC storageClassName from gp2 to gp3 across all envs

Switches Redis StatefulSet volumeClaimTemplates from the deprecated
in-tree aws-ebs provisioner (gp2) to the EBS CSI driver (gp3) in
prod, staging, and sandbox. Reduces EBS force-detach delay on node
failure from ~6min to ~1-2min and eliminates use of the deprecated
kubernetes.io/aws-ebs provisioner.

Live migration was performed for each environment with EBS snapshots
taken as backup before cutover.

---------

Co-authored-by: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>

Author: arielr-lt

.github/workflows/db-table-dump.yaml

@@ -15,8 +15,6 @@ permissions:
 env:
   AWS_REGION: us-east-1
   EKS_CLUSTER: ce-registry-eks
-  S3_DUMP_BUCKET: cer-db-dumps-prod
-  PRESIGNED_URL_TTL: 3600
 
 jobs:
   dump:
@@ -53,60 +51,24 @@ jobs:
         run: |
           TABLE="${{ inputs.table_name }}"
           JOB_NAME=$(
-            sed "s/__TABLE_NAME__/${TABLE}/" \
+            sed -e "s/__TABLE_NAME__/${TABLE}/" \
+                -e "s|__SLACK_WEBHOOK_URL__|${{ secrets.SLACK_WEBHOOK_URL }}|" \
               terraform/environments/eks/k8s-manifests-prod/db-table-dump-job.yaml |
             kubectl -n credreg-prod create -f - -o name | sed 's|job.batch/||'
           )
           echo "job_name=${JOB_NAME}" >> "$GITHUB_OUTPUT"
           echo "Launched job: ${JOB_NAME}"
 
-      - name: Wait for job completion
-        run: |
-          kubectl -n credreg-prod wait \
-            --for=condition=complete \
-            "job/${{ steps.create_job.outputs.job_name }}" \
-            --timeout=4h
-
-      - name: Get presigned URL
-        id: presign
-        run: |
-          S3_KEY=$(
-            kubectl -n credreg-prod logs "job/${{ steps.create_job.outputs.job_name }}" |
-            grep "^S3_KEY=" | tail -1 | cut -d= -f2-
-          )
-          if [ -z "$S3_KEY" ]; then
-            echo "Could not parse S3_KEY from job logs" >&2
-            exit 1
-          fi
-          PRESIGNED_URL=$(
-            aws s3 presign "s3://${{ env.S3_DUMP_BUCKET }}/${S3_KEY}" \
-              --expires-in "${{ env.PRESIGNED_URL_TTL }}" \
-              --region "${{ env.AWS_REGION }}"
-          )
-          echo "presigned_url=${PRESIGNED_URL}" >> "$GITHUB_OUTPUT"
-
-      - name: Notify Slack
-        if: always()
+      - name: Notify Slack (started)
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
           RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
-          if [ -z "${SLACK_WEBHOOK_URL}" ]; then
-            echo "SLACK_WEBHOOK_URL not set; skipping"
-            exit 0
-          fi
-
-          STATUS="${{ job.status }}"
+          if [ -z "${SLACK_WEBHOOK_URL}" ]; then exit 0; fi
           TABLE="${{ inputs.table_name }}"
           ACTOR="${{ github.actor }}"
-          NL=$'\n'
-
-          if [ "$STATUS" = "success" ]; then
-            PRESIGNED_URL="${{ steps.presign.outputs.presigned_url }}"
-            MSG=":white_check_mark: *DB dump ready*${NL}• Table: \`${TABLE}\`${NL}• Triggered by: ${ACTOR}${NL}• <${PRESIGNED_URL}|Download SQL dump> _(expires in 1h)_${NL}• <${RUN_URL}|View run>"
-          else
-            MSG=":x: *DB dump failed*${NL}• Table: \`${TABLE}\`${NL}• Triggered by: ${ACTOR}${NL}• <${RUN_URL}|View run>"
-          fi
-
-          payload=$(jq -nc --arg text "$MSG" '{text:$text}')
+          JOB="${{ steps.create_job.outputs.job_name }}"
+          payload=$(jq -nc \
+            --arg text ":hourglass: *DB dump started* | table: \`${TABLE}\` | triggered by: ${ACTOR} | job: \`${JOB}\`" \
+            '{text:$text}')
           curl -sS -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" || true

terraform/environments/eks/k8s-manifests-prod/db-dump-service-account.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: db-dump-service-account
+  namespace: credreg-prod
+  annotations:
+    eks.amazonaws.com/role-arn: "arn:aws:iam::996810415034:role/ce-registry-eks-db-dump-irsa-role"
+    eks.amazonaws.com/token-expiration: "43200"

terraform/environments/eks/k8s-manifests-prod/db-table-dump-job.yaml

@@ -11,7 +11,7 @@ spec:
   ttlSecondsAfterFinished: 300
   template:
     spec:
-      serviceAccountName: main-app-service-account
+      serviceAccountName: db-dump-service-account
       restartPolicy: Never
       containers:
         - name: pg-dump
@@ -21,10 +21,22 @@ spec:
             - |
               set -euo pipefail
 
-              apk add --no-cache aws-cli >/dev/null 2>&1
+              apk add --no-cache aws-cli curl >/dev/null 2>&1
 
               TIMESTAMP=$(date +%Y%m%d_%H%M%S)
               S3_KEY="table-dumps/${TABLE_NAME}/${TIMESTAMP}.sql.gz"
+              NL=$'\n'
+
+              notify_slack() {
+                [ -z "${SLACK_WEBHOOK_URL:-}" ] && return 0
+                payload=$(printf '{"text":"%s"}' "$(echo "$1" | sed 's/"/\\"/g; s/$/\\n/' | tr -d '\n' | sed 's/\\n$//')")
+                curl -sS -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" || true
+              }
+
+              on_failure() {
+                notify_slack ":x: *DB dump failed* | table: \`${TABLE_NAME}\`"
+              }
+              trap on_failure ERR
 
               echo "Dumping table '${TABLE_NAME}' from ${POSTGRESQL_DATABASE}..."
               PGPASSWORD="${POSTGRESQL_PASSWORD}" pg_dump \
@@ -40,11 +52,22 @@ spec:
                 --region us-east-1 \
                 --content-encoding gzip \
                 --content-type "text/plain"
+<<<<<<< feature/redis-pvc-gp2-to-gp3-migration
+
+              PRESIGNED_URL=$(aws s3 presign "s3://${S3_DUMP_BUCKET}/${S3_KEY}" \
+                --expires-in 3600 \
+                --region us-east-1)
+
+              notify_slack ":white_check_mark: *DB dump ready*\n• Table: \`${TABLE_NAME}\`\n• <${PRESIGNED_URL}|Download SQL dump> _(expires in 1h)_"
+=======
+>>>>>>> master
 
               echo "S3_KEY=${S3_KEY}"
           env:
             - name: TABLE_NAME
               value: "__TABLE_NAME__"
+            - name: SLACK_WEBHOOK_URL
+              value: "__SLACK_WEBHOOK_URL__"
             - name: S3_DUMP_BUCKET
               value: "cer-db-dumps-prod"
           envFrom:

terraform/environments/eks/k8s-manifests-prod/redis-deployment.yaml

@@ -81,7 +81,7 @@ spec:
         name: redis-data
       spec:
         accessModes: ["ReadWriteOnce"]
-        storageClassName: "gp2" # AWS EBS gp3 (adjust if needed)
+        storageClassName: "gp3"
         resources:
           requests:
             storage: 10Gi

terraform/environments/eks/k8s-manifests-sandbox/redis-deployment.yaml

@@ -79,7 +79,7 @@ spec:
         name: redis-data
       spec:
         accessModes: ["ReadWriteOnce"]
-        storageClassName: "gp2" # AWS EBS gp3 (adjust if needed)
+        storageClassName: "gp3"
         resources:
           requests:
             storage: 10Gi

terraform/environments/eks/k8s-manifests-staging/redis-deployment.yaml

@@ -79,7 +79,7 @@ spec:
         name: redis-data
       spec:
         accessModes: ["ReadWriteOnce"]
-        storageClassName: "gp2" # AWS EBS gp3 (adjust if needed)
+        storageClassName: "gp3"
         resources:
           requests:
             storage: 10Gi

terraform/modules/eks/irsa-iam-policy-and-role.tf

@@ -131,6 +131,12 @@ resource "aws_iam_policy" "application_policy" {
           "arn:aws:s3:::cer-db-dumps-prod/*"
         ]
       },
+      {
+        "Sid" : "DbDumpsGetObject",
+        "Effect" : "Allow",
+        "Action" : ["s3:GetObject"],
+        "Resource" : ["arn:aws:s3:::cer-db-dumps-prod/*"]
+      },
       {
         "Sid" : "S3BucketReadMeta",
         "Effect" : "Allow",