K8SPG-552 Add more cert-manager-tls validations and more platforms for snapshot test (#1505)

* Add EKS snapshot test and more backup-enable-disable checks

* fix eks snapshot test

* add correct storage class in test

* add more checks to cert-manager-tls

* solve comments

* fix openshift storage class in snapshot test

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: jvpasinatto

e2e-tests/conf/cmctl.yml

@@ -0,0 +1,54 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cmctl
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cmctl
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates", "certificates/status", "certificaterequests"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cmctl
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cmctl
+subjects:
+  - kind: ServiceAccount
+    name: cmctl
+    namespace: ${NAMESPACE}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cmctl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: cmctl
+  template:
+    metadata:
+      labels:
+        name: cmctl
+    spec:
+      serviceAccountName: cmctl
+      containers:
+        - name: cmctl
+          image: curlimages/curl
+          imagePullPolicy: Always
+          command:
+          - /bin/sh
+          - -c
+          - |
+            curl -fsSL -o /tmp/cmctl https://github.com/cert-manager/cmctl/releases/download/v2.4.1/cmctl_linux_amd64 \
+            && chmod +x /tmp/cmctl \
+            && sleep 100500
+      restartPolicy: Always

e2e-tests/functions

@@ -322,6 +322,11 @@ deploy_client() {
 	kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/client.yaml"
 }
 
+deploy_cmctl() {
+	envsubst < "${TESTS_CONFIG_DIR}/cmctl.yml" | kubectl -n "${NAMESPACE}" apply -f -
+	kubectl -n "${NAMESPACE}" wait deployment cmctl --for=condition=available --timeout=60s
+}
+
 get_client_pod() {
 	kubectl -n ${NAMESPACE} get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
 }
@@ -1322,6 +1327,26 @@ detect_k8s_platform() {
 	echo "${platform}"
 }
 
+ensure_ebs_gp3_storage_class() {
+	local platform=${1:-$(detect_k8s_platform)}
+
+	if [[ "$platform" != "eks" && "$platform" != "openshift" ]]; then
+		return 0
+	fi
+
+	cat <<EOF | kubectl apply -f -
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ebs-csi-gp3
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+parameters:
+  type: gp3
+EOF
+}
+
 enable_hugepages() {
 	local platform=$1
 	local hugepage_count=${2:-1024}

e2e-tests/tests/backup-enable-disable/06-check-backups-with-datasource.yaml

@@ -16,3 +16,8 @@ commands:
         echo "expected pgBackRest backup job to not exist"
         exit 1
       fi
+
+      if kubectl -n "${NAMESPACE}" get pvc some-name-datasource-repo1 >/dev/null 2>&1; then
+        echo "expected pgBackRest repo PVC to not exist"
+        exit 1
+      fi

e2e-tests/tests/backup-enable-disable/08-check-backups.yaml

@@ -16,3 +16,8 @@ commands:
         echo "backups are not disabled properly"
         exit 1
       fi
+
+      if kubectl -n "${NAMESPACE}" get pvc some-name-repo1 >/dev/null 2>&1; then
+        echo "expected pgBackRest repo PVC to not exist"
+        exit 1
+      fi

e2e-tests/tests/backup-enable-disable/09-assert.yaml

@@ -0,0 +1,47 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 300
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: some-name-repo-host
+  labels:
+    postgres-operator.crunchydata.com/cluster: some-name
+    postgres-operator.crunchydata.com/data: pgbackrest
+    postgres-operator.crunchydata.com/pgbackrest: ''
+    postgres-operator.crunchydata.com/pgbackrest-dedicated: ''
+status:
+  replicas: 1
+  readyReplicas: 1
+  updatedReplicas: 1
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: some-name-repo1
+  labels:
+    postgres-operator.crunchydata.com/cluster: some-name
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+    postgres-operator.crunchydata.com/pgbackrest-volume: ''
+---
+kind: Job
+apiVersion: batch/v1
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: some-name
+    postgres-operator.crunchydata.com/pgbackrest: ''
+    postgres-operator.crunchydata.com/pgbackrest-backup: replica-create
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+status:
+  succeeded: 1
+---
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: some-name
+status:
+  pgbackrest:
+    repos:
+      - name: repo1
+        stanzaCreated: true

e2e-tests/tests/backup-enable-disable/09-enable-backups-again.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      get_cr "some-name" ${RANDOM} \
+        | yq 'del(.metadata.annotations."pgv2.percona.com/authorizeBackupRemoval")' \
+        | yq 'del(.spec.backups.enabled)' \
+        | kubectl -n "${NAMESPACE}" apply -f -

e2e-tests/tests/backup-enable-disable/10-assert.yaml

@@ -0,0 +1,26 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 300
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: some-name-repo-host
+  labels:
+    postgres-operator.crunchydata.com/cluster: some-name
+    postgres-operator.crunchydata.com/data: pgbackrest
+    postgres-operator.crunchydata.com/pgbackrest: ''
+    postgres-operator.crunchydata.com/pgbackrest-dedicated: ''
+status:
+  replicas: 1
+  readyReplicas: 1
+  updatedReplicas: 1
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: some-name-repo1
+  labels:
+    postgres-operator.crunchydata.com/cluster: some-name
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+    postgres-operator.crunchydata.com/pgbackrest-volume: ''

e2e-tests/tests/backup-enable-disable/10-disable-backups-without-annotation.yaml

@@ -0,0 +1,18 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      get_cr "some-name" ${RANDOM} \
+        | yq 'del(.metadata.annotations."pgv2.percona.com/authorizeBackupRemoval")' \
+        | yq '.spec.backups.enabled=false' \
+        | kubectl -n "${NAMESPACE}" apply -f -
+
+      retry 60 5 bash -c '
+        kubectl -n "${NAMESPACE}" get postgrescluster some-name -o json \
+          | jq -e '"'"'.status.conditions[] | select(.type == "Progressing" and .reason == "Paused" and .status == "False" and (.message | contains("authorize backup removal")))'"'"' >/dev/null
+      '

e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml

@@ -11,4 +11,5 @@ commands:
 
       deploy_cert_manager
       deploy_operator
-      deploy_client
+      deploy_client
+      deploy_cmctl

e2e-tests/tests/cert-manager-tls/13-verify-tls-after-recovery.yaml

@@ -0,0 +1,42 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 60
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      pg_certificate_data=$(run_comand_on_pod "openssl s_client -connect cert-manager-tls-primary:5432 -starttls postgres <<< '' 2>/dev/null | openssl x509 -noout -issuer")
+      if [[ "$pg_certificate_data" != *"issuer=CN=cert-manager-tls-ca"* ]]; then
+          echo "Unexpected PostgreSQL certificate issuer after recovery"
+          echo "Got: $pg_certificate_data"
+          exit 1
+      fi
+
+      ssl_info=$(run_psql_local "SHOW ssl;" "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)")
+      if [[ "$ssl_info" != *"on"* ]]; then
+          echo "SSL is not enabled on PostgreSQL after recovery"
+          exit 1
+      fi
+
+      pgb_certificate_data=$(run_comand_on_pod "openssl s_client -connect cert-manager-tls-pgbouncer:5432 -starttls postgres <<< '' 2>/dev/null | openssl x509 -noout -issuer")
+      if [[ "$pgb_certificate_data" != *"issuer=CN=cert-manager-tls-ca"* ]]; then
+          echo "Unexpected PgBouncer certificate issuer after recovery"
+          echo "Got: $pgb_certificate_data"
+          exit 1
+      fi
+
+      instance=$(kubectl -n "$NAMESPACE" get pod \
+        -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/role=primary \
+        -o jsonpath='{.items[0].metadata.name}')
+
+      pgbr_certificate_data=$(run_comand_on_pod "openssl s_client -connect ${instance}.cert-manager-tls-pods:8432 <<< '' 2>/dev/null | openssl x509 -noout -issuer")
+      if [[ "$pgbr_certificate_data" != *"issuer=CN=cert-manager-tls-ca"* ]]; then
+          echo "Unexpected pgBackRest certificate issuer after recovery"
+          echo "Got: $pgbr_certificate_data"
+          exit 1
+      fi
+
+      kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info