diff --git a/cmd/postgres-operator/main.go b/cmd/postgres-operator/main.go index e4401cefc2..6dcbe28c7c 100644 --- a/cmd/postgres-operator/main.go +++ b/cmd/postgres-operator/main.go @@ -175,6 +175,7 @@ func addControllersToManager(ctx context.Context, mgr manager.Manager) error { if cm.Controller() == nil { return errors.New("missing controller in manager") } + r.Controller = cm.Controller() if err := mgr.GetFieldIndexer().IndexField( context.Background(), diff --git a/e2e-tests/functions b/e2e-tests/functions index 1356059887..2f7c8996d5 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -1142,6 +1142,11 @@ deploy_cert_manager() { until kubectl get validatingwebhookconfiguration cert-manager-webhook -o jsonpath='{.webhooks[0].clientConfig.caBundle}' | grep -q '[A-Za-z0-9+/=]'; do sleep 5 done + + echo "Waiting for cert-manager webhook service to have endpoints..." + until kubectl -n cert-manager get endpoints cert-manager-webhook -o jsonpath='{.subsets[*].addresses}' | grep -q '.'; do + sleep 5 + done } destroy_cert_manager() { diff --git a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml index beef88442f..0de950d193 100644 --- a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml +++ b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 commands: - script: |- set -o errexit @@ -9,7 +8,7 @@ commands: source ../../functions init_temp_dir # do this only in the first TestStep - deploy_cert_manager + destroy_cert_manager deploy_operator deploy_client - deploy_cmctl + timeout: 120 diff --git a/e2e-tests/tests/cert-manager-tls/01-assert.yaml b/e2e-tests/tests/cert-manager-tls/01-assert.yaml index ba38593ec0..99d26e99b8 100644 --- a/e2e-tests/tests/cert-manager-tls/01-assert.yaml +++ b/e2e-tests/tests/cert-manager-tls/01-assert.yaml @@ -58,34 +58,6 @@ metadata: status: succeeded: 1 --- -apiVersion: upstream.pgv2.percona.com/v1beta1 -kind: PostgresCluster -metadata: - name: cert-manager-tls - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGCluster - name: cert-manager-tls - controller: true - blockOwnerDeletion: true - finalizers: - - postgres-operator.crunchydata.com/finalizer -status: - instances: - - name: instance1 - readyReplicas: 3 - replicas: 3 - updatedReplicas: 3 - observedGeneration: 1 - pgbackrest: - repos: - - name: repo1 - stanzaCreated: true - proxy: - pgBouncer: - readyReplicas: 3 - replicas: 3 ---- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: @@ -101,4 +73,4 @@ status: size: 3 ready: 3 size: 3 - state: ready \ No newline at end of file + state: ready diff --git a/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml b/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml index eb0ab4d3c2..bd9d303ecc 100644 --- a/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml +++ b/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 commands: - script: |- set -o errexit @@ -9,4 +8,5 @@ commands: source ../../functions get_cr "cert-manager-tls" \ - | kubectl -n "${NAMESPACE}" apply -f - \ No newline at end of file + | kubectl -n "${NAMESPACE}" apply -f - + timeout: 10 diff --git a/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml b/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml new file mode 100644 index 0000000000..77c97e277e --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml @@ -0,0 +1,47 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + root_ca_annotation=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + + if [[ -n "$root_ca_annotation" ]]; then + echo "Root CA secret has cert-manager annotation but cert-manager is not installed!" + exit 1 + fi + + cert_count=$(kubectl -n "$NAMESPACE" get certificate 2>/dev/null | grep -c cert-manager-tls || true) + if [[ "$cert_count" -gt 0 ]]; then + echo "Found cert-manager Certificate resources but cert-manager should not be installed" + exit 1 + fi + + instance_sts=$(kubectl -n "$NAMESPACE" get sts \ + -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/instance-set=instance1 \ + -o jsonpath='{.items[*].metadata.name}') + + for sts_name in $instance_sts; do + secret_name="${sts_name}-certs" + + cm_annotation=$(kubectl -n "$NAMESPACE" get secret "$secret_name" \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + if [[ -n "$cm_annotation" ]]; then + echo "Instance secret $secret_name has cert-manager annotation unexpectedly" + exit 1 + fi + + for key in dns.crt dns.key patroni.ca-roots patroni.crt-combined; do + escaped_key="${key//./\\.}" + val=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath="{.data.${escaped_key}}") + if [[ -z "$val" ]]; then + echo "Instance secret $secret_name is missing key: $key" + exit 1 + fi + done + done + timeout: 60 diff --git a/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml b/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml index d1d67003c6..d9a2847beb 100644 --- a/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml +++ b/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 120 commands: - script: |- set -o errexit @@ -8,118 +7,25 @@ commands: source ../../functions - verify_secret_data() { - local secret_name="$1" - shift - for key in "$@"; do - escaped_key="${key//./\\.}" - val=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath="{.data.${escaped_key}}") - if [[ -z "$val" ]]; then - echo "Secret $secret_name is missing key: $key" - return 1 - fi - done - } + ssl_info=$(run_psql_local "SHOW ssl;" "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") - retry 12 5 verify_secret_data cert-manager-tls-cluster-ca-cert tls.crt tls.key ca.crt - - ca_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-ca-cert -o jsonpath='{.type}') - if [[ "$ca_secret_type" != "kubernetes.io/tls" ]]; then - echo "CA secret type is incorrect: $ca_secret_type (expected kubernetes.io/tls)" - exit 1 - fi - - ca_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-ca-cert -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$ca_issuer_name" != "cert-manager-tls-ca-issuer" ]]; then - echo "CA secret issuer annotation is incorrect: $ca_issuer_name" - exit 1 - fi - - retry 12 5 verify_secret_data cert-manager-tls-cluster-cert tls.crt tls.key ca.crt - - cluster_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert -o jsonpath='{.type}') - if [[ "$cluster_secret_type" != "kubernetes.io/tls" ]]; then - echo "Cluster TLS secret type is incorrect: $cluster_secret_type (expected kubernetes.io/tls)" - exit 1 - fi - - cluster_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$cluster_issuer_name" != "cert-manager-tls-tls-issuer" ]]; then - echo "Cluster TLS secret issuer annotation is incorrect: $cluster_issuer_name" - exit 1 - fi - - retry 12 5 verify_secret_data cert-manager-tls-pgbouncer-frontend-tls tls.crt tls.key ca.crt - - pgb_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.type}') - if [[ "$pgb_secret_type" != "kubernetes.io/tls" ]]; then - echo "PgBouncer TLS secret type is incorrect: $pgb_secret_type (expected kubernetes.io/tls)" - exit 1 - fi - - pgb_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$pgb_issuer_name" != "cert-manager-tls-tls-issuer" ]]; then - echo "PgBouncer TLS secret issuer annotation is incorrect: $pgb_issuer_name" + if [[ "$ssl_info" != *"on"* ]]; then + echo "SSL is not enabled on PostgreSQL with internal PKI" exit 1 fi - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-client-tls tls.crt tls.key + repl_ssl_count=$(run_psql_local \ + "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_replication r ON s.pid = r.pid WHERE s.ssl = true;" \ + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@cert-manager-tls-primary") + repl_ssl_count=$(echo "$repl_ssl_count" | tr -d '[:space:]') - pgbr_client_issuer=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-client-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$pgbr_client_issuer" != "cert-manager-tls-tls-issuer" ]]; then - echo "pgBackRest client TLS secret issuer annotation is incorrect: $pgbr_client_issuer" + if [[ "$repl_ssl_count" -lt 1 ]]; then + echo "No SSL replication connections found with internal PKI, got: $repl_ssl_count" exit 1 fi - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-repo-tls tls.crt tls.key - - pgbr_repo_issuer=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-repo-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$pgbr_repo_issuer" != "cert-manager-tls-tls-issuer" ]]; then - echo "pgBackRest repo TLS secret issuer annotation is incorrect: $pgbr_repo_issuer" - exit 1 - fi - - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest pgbackrest.ca-roots pgbackrest-client.crt pgbackrest-client.key pgbackrest-repo-host.crt pgbackrest-repo-host.key - - pgbr_client_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest -o jsonpath='{.data.pgbackrest-client\.crt}') - pgbr_client_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-client-tls -o jsonpath='{.data.tls\.crt}') - if [[ "$pgbr_client_cert_in_secret" != "$pgbr_client_cert_from_cm" ]]; then - echo "pgBackRest main secret client cert does not match cert-manager-issued cert" - exit 1 - fi - - pgbr_repo_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest -o jsonpath='{.data.pgbackrest-repo-host\.crt}') - pgbr_repo_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-repo-tls -o jsonpath='{.data.tls\.crt}') - if [[ "$pgbr_repo_cert_in_secret" != "$pgbr_repo_cert_from_cm" ]]; then - echo "pgBackRest main secret repo cert does not match cert-manager-issued cert" - exit 1 - fi - - retry 12 5 verify_secret_data cert-manager-tls-pgbouncer pgbouncer-frontend.crt pgbouncer-frontend.key pgbouncer-frontend.ca-roots - - pgb_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer -o jsonpath='{.data.pgbouncer-frontend\.crt}') - pgb_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.data.tls\.crt}') - if [[ "$pgb_cert_in_secret" != "$pgb_cert_from_cm" ]]; then - echo "pgBouncer main secret frontend cert does not match cert-manager-issued cert" - exit 1 - fi - - instance_sts=$(kubectl -n "$NAMESPACE" get sts -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/instance-set=instance1 -o jsonpath='{.items[*].metadata.name}') - for sts_name in $instance_sts; do - secret_name="${sts_name}-certs" - - secret_type=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath='{.type}') - if [[ "$secret_type" != "kubernetes.io/tls" ]]; then - echo "Instance secret $secret_name type is incorrect: $secret_type (expected kubernetes.io/tls)" - exit 1 - fi - - issuer_name=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$issuer_name" != "cert-manager-tls-tls-issuer" ]]; then - echo "Instance secret $secret_name issuer annotation is incorrect: $issuer_name" - exit 1 - fi + pg_cert_serial=$(run_comand_on_pod "openssl s_client -connect cert-manager-tls-primary:5432 -starttls postgres <<< '' 2>/dev/null | openssl x509 -noout -serial" | tr -d '[:space:]') - verify_secret_data "$secret_name" tls.crt tls.key dns.crt dns.key patroni.ca-roots patroni.crt-combined pgbackrest-server.crt pgbackrest-server.key - echo "Instance secret $secret_name is valid" - done + kubectl create configmap -n "${NAMESPACE}" internal-pki-cert-serial \ + --from-literal=pg-serial="$pg_cert_serial" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/04-write-data.yaml b/e2e-tests/tests/cert-manager-tls/04-write-data.yaml index 8871dd1855..f44c9fbaac 100644 --- a/e2e-tests/tests/cert-manager-tls/04-write-data.yaml +++ b/e2e-tests/tests/cert-manager-tls/04-write-data.yaml @@ -13,4 +13,5 @@ commands: run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ - "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" \ No newline at end of file + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml b/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml new file mode 100644 index 0000000000..716800c0cc --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + deploy_cert_manager + deploy_cmctl + + kubectl -n "$NAMESPACE" delete pod -l postgres-operator.crunchydata.com/role=pgbouncer,postgres-operator.crunchydata.com/cluster=cert-manager-tls + + wait_cluster_consistency cert-manager-tls + timeout: 120 diff --git a/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml b/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml new file mode 100644 index 0000000000..a02eccafb3 --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml @@ -0,0 +1,46 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + root_ca_annotation=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + + if [[ -n "$root_ca_annotation" ]]; then + echo "FAIL: Root CA secret was taken over by cert-manager!" + exit 1 + fi + + cert_count=$(kubectl -n "$NAMESPACE" get certificate 2>/dev/null | grep -c cert-manager-tls || true) + if [[ "$cert_count" -gt 0 ]]; then + echo "FAIL: cert-manager Certificate resources were created for the existing cluster" + exit 1 + fi + + instance_sts=$(kubectl -n "$NAMESPACE" get sts \ + -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/instance-set=instance1 \ + -o jsonpath='{.items[*].metadata.name}') + + for sts_name in $instance_sts; do + secret_name="${sts_name}-certs" + + cm_annotation=$(kubectl -n "$NAMESPACE" get secret "$secret_name" \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + if [[ -n "$cm_annotation" ]]; then + echo "FAIL: Instance secret $secret_name was taken over by cert-manager" + exit 1 + fi + done + + pg_cert_serial=$(run_comand_on_pod "openssl s_client -connect cert-manager-tls-primary:5432 -starttls postgres <<< '' 2>/dev/null | openssl x509 -noout -serial" | tr -d '[:space:]') + pg_cert_serial_before=$(kubectl -n "$NAMESPACE" get configmap internal-pki-cert-serial -o jsonpath='{.data.pg-serial}') + + if [[ "$pg_cert_serial" != "$pg_cert_serial_before" ]]; then + echo "FAIL: PostgreSQL certificate changed after cert-manager installation!" + exit 1 + fi + timeout: 60 diff --git a/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml b/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml new file mode 100644 index 0000000000..8ba9f2779f --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml @@ -0,0 +1,46 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + ssl_info=$(run_psql_local "SHOW ssl;" "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") + + if [[ "$ssl_info" != *"on"* ]]; then + echo "SSL is not enabled on PostgreSQL after cert-manager installation" + exit 1 + fi + + repl_ssl_count=$(run_psql_local \ + "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_replication r ON s.pid = r.pid WHERE s.ssl = true;" \ + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@cert-manager-tls-primary") + repl_ssl_count=$(echo "$repl_ssl_count" | tr -d '[:space:]') + + if [[ "$repl_ssl_count" -lt 1 ]]; then + echo "No SSL replication connections found after cert-manager install, got: $repl_ssl_count" + exit 1 + fi + + pgb_ssl=$(run_psql_local "SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid();" "cert-manager-tls:$(get_psql_user_pass cert-manager-tls-pguser-cert-manager-tls)@cert-manager-tls-pgbouncer/postgres") + + if [[ "$pgb_ssl" != *"t"* ]]; then + echo "PgBouncer-to-PostgreSQL connection is not using SSL after cert-manager install" + exit 1 + fi + + instance=$(kubectl -n "$NAMESPACE" get pod \ + -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/role=primary \ + -o jsonpath='{.items[0].metadata.name}') + + kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info + + data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") + + if [[ "$data" != *"100500"* ]]; then + echo "FAIL: Data is not intact after cert-manager installation" + exit 1 + fi + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml b/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml new file mode 100644 index 0000000000..555dcf591f --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kubectl -n "$NAMESPACE" delete perconapgcluster cert-manager-tls --wait=true || true + kubectl -n "$NAMESPACE" delete postgrescluster cert-manager-tls --wait=true || true + + kubectl -n "$NAMESPACE" delete pvc -l postgres-operator.crunchydata.com/cluster=cert-manager-tls || true + kubectl -n "$NAMESPACE" delete secret -l postgres-operator.crunchydata.com/cluster=cert-manager-tls || true + kubectl -n "$NAMESPACE" delete configmap internal-pki-cert-serial || true + + cluster_deleted() { + ! kubectl -n "$NAMESPACE" get postgrescluster cert-manager-tls 2>/dev/null + } + retry 12 5 cluster_deleted + timeout: 120 diff --git a/e2e-tests/tests/cert-manager-tls/09-assert.yaml b/e2e-tests/tests/cert-manager-tls/09-assert.yaml new file mode 100644 index 0000000000..3fa4968d67 --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/09-assert.yaml @@ -0,0 +1,104 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: cert-manager-tls + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + ownerReferences: + - apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster + name: cert-manager-tls + controller: true + blockOwnerDeletion: true +status: + observedGeneration: 1 + replicas: 1 + readyReplicas: 1 + currentReplicas: 1 + updatedReplicas: 1 + collisionCount: 0 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: cert-manager-tls-pgbouncer + labels: + postgres-operator.crunchydata.com/cluster: cert-manager-tls + postgres-operator.crunchydata.com/role: pgbouncer + ownerReferences: + - apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster + name: cert-manager-tls + controller: true + blockOwnerDeletion: true +status: + observedGeneration: 1 + replicas: 3 + updatedReplicas: 3 + readyReplicas: 3 +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: cert-manager-tls + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 +--- +apiVersion: upstream.pgv2.percona.com/v1beta1 +kind: PostgresCluster +metadata: + name: cert-manager-tls + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + name: cert-manager-tls + controller: true + blockOwnerDeletion: true + finalizers: + - postgres-operator.crunchydata.com/finalizer +status: + instances: + - name: instance1 + readyReplicas: 3 + replicas: 3 + updatedReplicas: 3 + observedGeneration: 1 + pgbackrest: + repos: + - name: repo1 + stanzaCreated: true + proxy: + pgBouncer: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: cert-manager-tls +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml b/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml new file mode 100644 index 0000000000..bd9d303ecc --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr "cert-manager-tls" \ + | kubectl -n "${NAMESPACE}" apply -f - + timeout: 10 diff --git a/e2e-tests/tests/cert-manager-tls/02-verify-cert-manager-resources.yaml b/e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml similarity index 99% rename from e2e-tests/tests/cert-manager-tls/02-verify-cert-manager-resources.yaml rename to e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml index f8911c9fe7..53699ada57 100644 --- a/e2e-tests/tests/cert-manager-tls/02-verify-cert-manager-resources.yaml +++ b/e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 60 commands: - script: |- set -o errexit @@ -60,3 +59,4 @@ commands: fi echo "Instance certificate $cert_name is ready" done + timeout: 60 diff --git a/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml b/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml new file mode 100644 index 0000000000..68762bb765 --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml @@ -0,0 +1,125 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + verify_secret_data() { + local secret_name="$1" + shift + for key in "$@"; do + escaped_key="${key//./\\.}" + val=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath="{.data.${escaped_key}}") + if [[ -z "$val" ]]; then + echo "Secret $secret_name is missing key: $key" + return 1 + fi + done + } + + retry 12 5 verify_secret_data cert-manager-tls-cluster-ca-cert tls.crt tls.key ca.crt + + ca_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-ca-cert -o jsonpath='{.type}') + if [[ "$ca_secret_type" != "kubernetes.io/tls" ]]; then + echo "CA secret type is incorrect: $ca_secret_type (expected kubernetes.io/tls)" + exit 1 + fi + + ca_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-ca-cert -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$ca_issuer_name" != "cert-manager-tls-ca-issuer" ]]; then + echo "CA secret issuer annotation is incorrect: $ca_issuer_name" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-cluster-cert tls.crt tls.key ca.crt + + cluster_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert -o jsonpath='{.type}') + if [[ "$cluster_secret_type" != "kubernetes.io/tls" ]]; then + echo "Cluster TLS secret type is incorrect: $cluster_secret_type (expected kubernetes.io/tls)" + exit 1 + fi + + cluster_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$cluster_issuer_name" != "cert-manager-tls-tls-issuer" ]]; then + echo "Cluster TLS secret issuer annotation is incorrect: $cluster_issuer_name" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbouncer-frontend-tls tls.crt tls.key ca.crt + + pgb_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.type}') + if [[ "$pgb_secret_type" != "kubernetes.io/tls" ]]; then + echo "PgBouncer TLS secret type is incorrect: $pgb_secret_type (expected kubernetes.io/tls)" + exit 1 + fi + + pgb_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$pgb_issuer_name" != "cert-manager-tls-tls-issuer" ]]; then + echo "PgBouncer TLS secret issuer annotation is incorrect: $pgb_issuer_name" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-client-tls tls.crt tls.key + + pgbr_client_issuer=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-client-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$pgbr_client_issuer" != "cert-manager-tls-tls-issuer" ]]; then + echo "pgBackRest client TLS secret issuer annotation is incorrect: $pgbr_client_issuer" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-repo-tls tls.crt tls.key + + pgbr_repo_issuer=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-repo-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$pgbr_repo_issuer" != "cert-manager-tls-tls-issuer" ]]; then + echo "pgBackRest repo TLS secret issuer annotation is incorrect: $pgbr_repo_issuer" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbackrest pgbackrest.ca-roots pgbackrest-client.crt pgbackrest-client.key pgbackrest-repo-host.crt pgbackrest-repo-host.key + + pgbr_client_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest -o jsonpath='{.data.pgbackrest-client\.crt}') + pgbr_client_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-client-tls -o jsonpath='{.data.tls\.crt}') + if [[ "$pgbr_client_cert_in_secret" != "$pgbr_client_cert_from_cm" ]]; then + echo "pgBackRest main secret client cert does not match cert-manager-issued cert" + exit 1 + fi + + pgbr_repo_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest -o jsonpath='{.data.pgbackrest-repo-host\.crt}') + pgbr_repo_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-repo-tls -o jsonpath='{.data.tls\.crt}') + if [[ "$pgbr_repo_cert_in_secret" != "$pgbr_repo_cert_from_cm" ]]; then + echo "pgBackRest main secret repo cert does not match cert-manager-issued cert" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbouncer pgbouncer-frontend.crt pgbouncer-frontend.key pgbouncer-frontend.ca-roots + + pgb_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer -o jsonpath='{.data.pgbouncer-frontend\.crt}') + pgb_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.data.tls\.crt}') + if [[ "$pgb_cert_in_secret" != "$pgb_cert_from_cm" ]]; then + echo "pgBouncer main secret frontend cert does not match cert-manager-issued cert" + exit 1 + fi + + instance_sts=$(kubectl -n "$NAMESPACE" get sts -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/instance-set=instance1 -o jsonpath='{.items[*].metadata.name}') + for sts_name in $instance_sts; do + secret_name="${sts_name}-certs" + + secret_type=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath='{.type}') + if [[ "$secret_type" != "kubernetes.io/tls" ]]; then + echo "Instance secret $secret_name type is incorrect: $secret_type (expected kubernetes.io/tls)" + exit 1 + fi + + issuer_name=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$issuer_name" != "cert-manager-tls-tls-issuer" ]]; then + echo "Instance secret $secret_name issuer annotation is incorrect: $issuer_name" + exit 1 + fi + + verify_secret_data "$secret_name" tls.crt tls.key dns.crt dns.key patroni.ca-roots patroni.crt-combined pgbackrest-server.crt pgbackrest-server.key + echo "Instance secret $secret_name is valid" + done + timeout: 120 diff --git a/e2e-tests/tests/cert-manager-tls/12-write-data.yaml b/e2e-tests/tests/cert-manager-tls/12-write-data.yaml new file mode 100644 index 0000000000..81e1c7ba18 --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/12-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_psql_local \ + 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" + + run_psql_local \ + '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" diff --git a/e2e-tests/tests/cert-manager-tls/05-assert.yaml b/e2e-tests/tests/cert-manager-tls/13-assert.yaml similarity index 70% rename from e2e-tests/tests/cert-manager-tls/05-assert.yaml rename to e2e-tests/tests/cert-manager-tls/13-assert.yaml index 80d86f1bf0..7ed7e0b433 100644 --- a/e2e-tests/tests/cert-manager-tls/05-assert.yaml +++ b/e2e-tests/tests/cert-manager-tls/13-assert.yaml @@ -5,6 +5,6 @@ timeout: 30 kind: ConfigMap apiVersion: v1 metadata: - name: 05-read-from-primary + name: 13-read-from-primary data: - data: ' 100500' \ No newline at end of file + data: ' 100500' diff --git a/e2e-tests/tests/cert-manager-tls/05-read-from-primary.yaml b/e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml similarity index 73% rename from e2e-tests/tests/cert-manager-tls/05-read-from-primary.yaml rename to e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml index 2b8166ba5d..9ccfa5c308 100644 --- a/e2e-tests/tests/cert-manager-tls/05-read-from-primary.yaml +++ b/e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -10,4 +9,5 @@ commands: data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") - kubectl create configmap -n "${NAMESPACE}" 05-read-from-primary --from-literal=data="${data}" \ No newline at end of file + kubectl create configmap -n "${NAMESPACE}" 13-read-from-primary --from-literal=data="${data}" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/06-assert.yaml b/e2e-tests/tests/cert-manager-tls/14-assert.yaml similarity index 67% rename from e2e-tests/tests/cert-manager-tls/06-assert.yaml rename to e2e-tests/tests/cert-manager-tls/14-assert.yaml index 477b3886cf..28a8250ef1 100644 --- a/e2e-tests/tests/cert-manager-tls/06-assert.yaml +++ b/e2e-tests/tests/cert-manager-tls/14-assert.yaml @@ -5,6 +5,6 @@ timeout: 30 kind: ConfigMap apiVersion: v1 metadata: - name: 06-verify-tls-connection + name: 14-verify-tls-connection data: - status: verified \ No newline at end of file + status: verified diff --git a/e2e-tests/tests/cert-manager-tls/06-verify-tls-connection.yaml b/e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml similarity index 90% rename from e2e-tests/tests/cert-manager-tls/06-verify-tls-connection.yaml rename to e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml index 3babb02057..16158c4ca6 100644 --- a/e2e-tests/tests/cert-manager-tls/06-verify-tls-connection.yaml +++ b/e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -32,4 +31,5 @@ commands: exit 1 fi - kubectl create configmap -n "${NAMESPACE}" 06-verify-tls-connection --from-literal=status="verified" \ No newline at end of file + kubectl create configmap -n "${NAMESPACE}" 14-verify-tls-connection --from-literal=status="verified" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/07-verify-tls-pgbouncer.yaml b/e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml similarity index 98% rename from e2e-tests/tests/cert-manager-tls/07-verify-tls-pgbouncer.yaml rename to e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml index b12df05cbd..7a2fa480b0 100644 --- a/e2e-tests/tests/cert-manager-tls/07-verify-tls-pgbouncer.yaml +++ b/e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -29,3 +28,4 @@ commands: echo "PgBouncer-to-PostgreSQL connection is not using SSL" exit 1 fi + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/08-verify-tls-replication.yaml b/e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml similarity index 97% rename from e2e-tests/tests/cert-manager-tls/08-verify-tls-replication.yaml rename to e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml index 7b0eb6fa6f..25d293ec7d 100644 --- a/e2e-tests/tests/cert-manager-tls/08-verify-tls-replication.yaml +++ b/e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -17,3 +16,4 @@ commands: echo "No SSL replication connections found, got: $repl_ssl_count" exit 1 fi + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/09-verify-tls-pgbackrest.yaml b/e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml similarity index 98% rename from e2e-tests/tests/cert-manager-tls/09-verify-tls-pgbackrest.yaml rename to e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml index c8afa2304d..43db32b66a 100644 --- a/e2e-tests/tests/cert-manager-tls/09-verify-tls-pgbackrest.yaml +++ b/e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -34,3 +33,4 @@ commands: # Verify pgBackRest works over TLS, not just cert file existence kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/10-cert-deletion-recovery.yaml b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml similarity index 91% rename from e2e-tests/tests/cert-manager-tls/10-cert-deletion-recovery.yaml rename to e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml index 88bd8602b7..ab2276ac37 100644 --- a/e2e-tests/tests/cert-manager-tls/10-cert-deletion-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 300 commands: - script: |- set -o errexit @@ -28,13 +27,13 @@ commands: [[ "$cert_ready" == "True" ]] } - retry 30 10 verify_cert_recreated cert-manager-tls-pgbouncer-cert + retry 60 10 verify_cert_recreated cert-manager-tls-pgbouncer-cert echo "pgBouncer Certificate recreated and ready" - retry 30 10 verify_cert_recreated cert-manager-tls-pgbackrest-repo-cert + retry 60 10 verify_cert_recreated cert-manager-tls-pgbackrest-repo-cert echo "pgBackRest repo Certificate recreated and ready" - retry 30 10 verify_cert_recreated cert-manager-tls-pgbackrest-client-cert + retry 60 10 verify_cert_recreated cert-manager-tls-pgbackrest-client-cert echo "pgBackRest client Certificate recreated and ready" verify_secret_data() { @@ -50,9 +49,9 @@ commands: done } - retry 12 5 verify_secret_data cert-manager-tls-pgbouncer-frontend-tls tls.crt tls.key ca.crt - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-repo-tls tls.crt tls.key - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-client-tls tls.crt tls.key + retry 30 5 verify_secret_data cert-manager-tls-pgbouncer-frontend-tls tls.crt tls.key ca.crt + retry 30 5 verify_secret_data cert-manager-tls-pgbackrest-repo-tls tls.crt tls.key + retry 30 5 verify_secret_data cert-manager-tls-pgbackrest-client-tls tls.crt tls.key pgb_cert_after=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.data.tls\.crt}') if [[ "$pgb_cert_before" == "$pgb_cert_after" ]]; then @@ -101,3 +100,4 @@ commands: done wait_cluster_consistency cert-manager-tls +timeout: 900 diff --git a/e2e-tests/tests/cert-manager-tls/11-write-data-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml similarity index 97% rename from e2e-tests/tests/cert-manager-tls/11-write-data-after-recovery.yaml rename to e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml index d81df23a94..130fe102f7 100644 --- a/e2e-tests/tests/cert-manager-tls/11-write-data-after-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -15,3 +14,4 @@ commands: run_psql_local \ '\c newapp \\\ INSERT INTO newApp (id) VALUES (200500)' \ "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/12-verify-data-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml similarity index 96% rename from e2e-tests/tests/cert-manager-tls/12-verify-data-after-recovery.yaml rename to e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml index bfebc746ba..8edd9ac1b7 100644 --- a/e2e-tests/tests/cert-manager-tls/12-verify-data-after-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -14,3 +13,4 @@ commands: echo "Missing data in primary" exit 1 fi + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/13-verify-tls-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml similarity index 99% rename from e2e-tests/tests/cert-manager-tls/13-verify-tls-after-recovery.yaml rename to e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml index 2ce31fabb7..f4b56621fe 100644 --- a/e2e-tests/tests/cert-manager-tls/13-verify-tls-after-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 60 commands: - script: |- set -o errexit @@ -40,3 +39,4 @@ commands: fi kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info + timeout: 60 diff --git a/e2e-tests/tests/cert-manager-tls/14-verify-tls-required.yaml b/e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml similarity index 98% rename from e2e-tests/tests/cert-manager-tls/14-verify-tls-required.yaml rename to e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml index 34583accc3..8302c3d325 100644 --- a/e2e-tests/tests/cert-manager-tls/14-verify-tls-required.yaml +++ b/e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -21,3 +20,4 @@ commands: exit 1 fi echo "Correctly rejected non-TLS connection to pgbouncer" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/15-verify-cert-rotation.yaml b/e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml similarity index 99% rename from e2e-tests/tests/cert-manager-tls/15-verify-cert-rotation.yaml rename to e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml index 0206dd06a3..a1d48c07ee 100644 --- a/e2e-tests/tests/cert-manager-tls/15-verify-cert-rotation.yaml +++ b/e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 600 commands: - script: |- set -o errexit @@ -109,3 +108,4 @@ commands: fi kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info + timeout: 600 diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go index 282ef97945..e0518ce57b 100644 --- a/internal/controller/postgrescluster/controller.go +++ b/internal/controller/postgrescluster/controller.go @@ -7,6 +7,7 @@ package postgrescluster import ( "context" "fmt" + "sync/atomic" "time" cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -26,12 +27,16 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/percona/percona-postgresql-operator/v2/internal/config" "github.com/percona/percona-postgresql-operator/v2/internal/controller/runtime" @@ -60,17 +65,20 @@ const ( // Reconciler holds resources for the PostgresCluster reconciler type Reconciler struct { - Client client.Client - Scheme *k8sruntime.Scheme - DiscoveryClient *discovery.DiscoveryClient - IsOpenShift bool - Owner client.FieldOwner - PodExec runtime.PodExecutor - Recorder record.EventRecorder - Registration registration.Registration - Tracer trace.Tracer - CertManagerCtrlFunc certmanager.NewControllerFunc - RestConfig *rest.Config + Client client.Client + Scheme *k8sruntime.Scheme + DiscoveryClient *discovery.DiscoveryClient + IsOpenShift bool + Owner client.FieldOwner + PodExec runtime.PodExecutor + Recorder record.EventRecorder + Registration registration.Registration + Tracer trace.Tracer + CertManagerCtrlFunc certmanager.NewControllerFunc + RestConfig *rest.Config + Controller controller.Controller + Cache cache.Cache + certManagerWatchesRegistered atomic.Bool } // +kubebuilder:rbac:groups="",resources="events",verbs={create,patch} @@ -284,6 +292,16 @@ func (r *Reconciler) Reconcile( rootCA, err = r.reconcileRootCertificate(ctx, cluster) } + if err == nil && rootCA != nil { + certManagerManaged, certErr := r.isRootCACertManagerManaged(ctx, cluster) + if certErr != nil { + log.V(1).Info("failed to check if root CA is cert-manager managed, will retry on next reconcile", + "error", certErr) + } else if certManagerManaged { + r.registerCertManagerWatches(ctx) + } + } + if err == nil { // Since any existing data directories must be moved prior to bootstrapping the // cluster, further reconciliation will not occur until the directory move Jobs @@ -543,6 +561,8 @@ func (r *Reconciler) SetupWithManager(mgr manager.Manager) error { } } + r.Cache = mgr.GetCache() + // K8SPG-712: Allow overriding default configurations configMapPredicate := builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { @@ -596,3 +616,72 @@ func (r *Reconciler) SetupWithManager(mgr manager.Manager) error { return bldr.Complete(r) } + +// registerCertManagerWatches dynamically registers watches for cert-manager +// Certificate, Issuer, and cert-manager-issued Secret resources for +// case where cert-manager is installed after the operator starts, so the +// watches were not registered in SetupWithManager. +func (r *Reconciler) registerCertManagerWatches(ctx context.Context) { + if r.Controller == nil || r.Cache == nil { + return + } + + if r.certManagerWatchesRegistered.Load() { + return + } + + log := logging.FromContext(ctx) + + certHandler := handler.TypedEnqueueRequestForOwner[*cmv1.Certificate]( + r.Scheme, r.Client.RESTMapper(), + &v1beta1.PostgresCluster{}, + handler.OnlyControllerOwner(), + ) + if err := r.Controller.Watch(source.Kind( + r.Cache, &cmv1.Certificate{}, certHandler, + )); err != nil { + log.Error(err, "failed to register dynamic watch for Certificates") + return + } + + issuerHandler := handler.TypedEnqueueRequestForOwner[*cmv1.Issuer]( + r.Scheme, r.Client.RESTMapper(), + &v1beta1.PostgresCluster{}, + handler.OnlyControllerOwner(), + ) + if err := r.Controller.Watch(source.Kind( + r.Cache, &cmv1.Issuer{}, issuerHandler, + )); err != nil { + log.Error(err, "failed to register dynamic watch for Issuers") + return + } + secretHandler := handler.TypedEnqueueRequestsFromMapFunc( + func(ctx context.Context, secret *corev1.Secret) []reconcile.Request { + cluster := secret.GetLabels()[naming.LabelCluster] + if len(cluster) > 0 { + return []reconcile.Request{ + {NamespacedName: client.ObjectKey{ + Namespace: secret.GetNamespace(), + Name: cluster, + }}, + } + } + return nil + }, + ) + certManagerSecretPredicate := predicate.NewTypedPredicateFuncs(func(secret *corev1.Secret) bool { + _, hasCluster := secret.GetLabels()[naming.LabelCluster] + _, hasCertAnnotation := secret.GetAnnotations()["cert-manager.io/certificate-name"] + return hasCluster && hasCertAnnotation + }) + if err := r.Controller.Watch(source.Kind( + r.Cache, &corev1.Secret{}, secretHandler, certManagerSecretPredicate, + )); err != nil { + log.Error(err, "failed to register dynamic watch for cert-manager Secrets") + return + } + + r.certManagerWatchesRegistered.Store(true) + + log.Info("dynamically registered cert-manager watches") +} diff --git a/internal/controller/postgrescluster/instance.go b/internal/controller/postgrescluster/instance.go index e99c155497..dd2e6b8649 100644 --- a/internal/controller/postgrescluster/instance.go +++ b/internal/controller/postgrescluster/instance.go @@ -1487,12 +1487,12 @@ func (r *Reconciler) reconcileInstanceCertificates( rootCertificateAuth *pki.RootCertificateAuthority, ) (*corev1.Secret, error) { if cluster.Spec.CustomTLSSecret == nil { - certManagerInstalled, err := r.isCertManagerInstalled(ctx, cluster.Namespace) + certManagerManaged, err := r.isRootCACertManagerManaged(ctx, cluster) if err != nil { - return nil, errors.Wrap(err, "failed to check if cert-manager is installed") + return nil, errors.Wrap(err, "failed to check if cert-manager manages root CA") } - if certManagerInstalled { + if certManagerManaged { return r.reconcileCertManagerInstanceCertificates(ctx, cluster, spec, instance, rootCertificateAuth) } } diff --git a/internal/controller/postgrescluster/pgbackrest.go b/internal/controller/postgrescluster/pgbackrest.go index ec7e2875c3..bcbbdd8d2b 100644 --- a/internal/controller/postgrescluster/pgbackrest.go +++ b/internal/controller/postgrescluster/pgbackrest.go @@ -2227,16 +2227,16 @@ func (r *Reconciler) reconcilePGBackRestSecret(ctx context.Context, // } if err == nil && repoHost != nil { - certManagerInstalled := false + certManagerManaged := false if cluster.Spec.CustomTLSSecret == nil { var certErr error - certManagerInstalled, certErr = r.isCertManagerInstalled(ctx, cluster.Namespace) + certManagerManaged, certErr = r.isRootCACertManagerManaged(ctx, cluster) if certErr != nil { - return errors.Wrap(certErr, "failed to check if cert-manager is installed") + return errors.Wrap(certErr, "failed to check if cert-manager manages root CA") } } - if certManagerInstalled { + if certManagerManaged { err = r.reconcileCertManagerPGBackRestSecret(ctx, cluster, repoHost, rootCA, existing, intent) } else { err = pgbackrest.Secret(ctx, cluster, repoHost, rootCA, existing, intent) diff --git a/internal/controller/postgrescluster/pgbouncer.go b/internal/controller/postgrescluster/pgbouncer.go index 6c86bdc303..6d2fecc53a 100644 --- a/internal/controller/postgrescluster/pgbouncer.go +++ b/internal/controller/postgrescluster/pgbouncer.go @@ -223,12 +223,12 @@ func (r *Reconciler) reconcilePGBouncerSecret( var frontendCertManagerSecret *corev1.Secret if cluster.Spec.Proxy.PGBouncer.CustomTLSSecret == nil { - certManagerInstalled, certErr := r.isCertManagerInstalled(ctx, cluster.Namespace) + certManagerManaged, certErr := r.isRootCACertManagerManaged(ctx, cluster) if certErr != nil { - return nil, errors.Wrap(certErr, "failed to check if cert-manager is installed") + return nil, errors.Wrap(certErr, "failed to check if cert-manager manages root CA") } - if certManagerInstalled { + if certManagerManaged { c := r.CertManagerCtrlFunc(r.Client, r.Scheme, false) dnsNames, dnsErr := naming.ServiceDNSNames(ctx, service, cluster.Spec.ClusterServiceDNSSuffix) diff --git a/internal/controller/postgrescluster/pki.go b/internal/controller/postgrescluster/pki.go index 4314ad32ef..8b4cea4840 100644 --- a/internal/controller/postgrescluster/pki.go +++ b/internal/controller/postgrescluster/pki.go @@ -221,12 +221,12 @@ func (r *Reconciler) reconcileClusterCertificate( return cluster.Spec.CustomTLSSecret, nil } - certManagerInstalled, err := r.isCertManagerInstalled(ctx, cluster.Namespace) + certManagerManaged, err := r.isRootCACertManagerManaged(ctx, cluster) if err != nil { - return nil, errors.Wrap(err, "failed to check if cert-manager is installed") + return nil, errors.Wrap(err, "failed to check if cert-manager manages root CA") } - if certManagerInstalled { + if certManagerManaged { return r.reconcileCertManagerClusterCertificate(ctx, root, cluster, primaryService, replicaService) } @@ -351,6 +351,28 @@ func (r *Reconciler) reconcileCertManagerClusterCertificate( }), nil } +func (r *Reconciler) isRootCACertManagerManaged(ctx context.Context, cluster *v1beta1.PostgresCluster) (bool, error) { + if cluster.Spec.CustomRootCATLSSecret != nil { + return false, nil + } + + installed, err := r.isCertManagerInstalled(ctx, cluster.Namespace) + if err != nil || !installed { + return false, err + } + + rootSecret := &corev1.Secret{ObjectMeta: naming.PostgresRootCASecret(cluster)} + err = r.Client.Get(ctx, client.ObjectKeyFromObject(rootSecret), rootSecret) + if err != nil { + if k8serrors.IsNotFound(err) { + return true, nil + } + return false, errors.WithStack(err) + } + + return rootSecret.Annotations["cert-manager.io/certificate-name"] != "", nil +} + func (r *Reconciler) isCertManagerInstalled(ctx context.Context, ns string) (bool, error) { if r.RestConfig == nil { return false, nil diff --git a/internal/controller/postgrescluster/pki_test.go b/internal/controller/postgrescluster/pki_test.go index 5e23f11f76..dc055bd0cb 100644 --- a/internal/controller/postgrescluster/pki_test.go +++ b/internal/controller/postgrescluster/pki_test.go @@ -17,7 +17,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/percona/percona-postgresql-operator/v2/internal/naming" @@ -389,6 +391,160 @@ func TestReconcileCerts(t *testing.T) { }) } +type mockCertManagerController struct{} + +func (m *mockCertManagerController) Check(context.Context, *rest.Config, string) error { return nil } +func (m *mockCertManagerController) ApplyIssuer(context.Context, *v1beta1.PostgresCluster) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyCAIssuer(context.Context, *v1beta1.PostgresCluster) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyCACertificate(context.Context, *v1beta1.PostgresCluster) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyClusterCertificate(context.Context, *v1beta1.PostgresCluster, []string) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyInstanceCertificate(context.Context, *v1beta1.PostgresCluster, string, []string) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyPGBouncerCertificate(context.Context, *v1beta1.PostgresCluster, []string) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyPGBackRestClientCertificate(context.Context, *v1beta1.PostgresCluster) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyPGBackRestRepoCertificate(context.Context, *v1beta1.PostgresCluster, []string) error { + panic("unexpected call") +} + +func mockCertManagerCtrlFunc(_ client.Client, _ *runtime.Scheme, _ bool) certmanager.Controller { + return &mockCertManagerController{} +} + +func TestUpgradeCertManagerDoesNotTakeOverInternalPKI(t *testing.T) { + if strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") { + t.Skip("USE_EXISTING_CLUSTER: Test fails due to garbage collection") + } + + _, tClient := setupKubernetes(t) + require.ParallelCapacity(t, 1) + ctx := t.Context() + namespace := require.Namespace(t, tClient).Name + + reconcilerWithoutCertManager := &Reconciler{ + Client: tClient, + Owner: ControllerName, + CertManagerCtrlFunc: certmanager.NewController, + } + + cluster := testCluster() + cluster.Name = "upgrade-test" + cluster.Namespace = namespace + assert.NilError(t, tClient.Create(ctx, cluster)) + + root, err := reconcilerWithoutCertManager.reconcileRootCertificate(ctx, cluster) + assert.NilError(t, err) + + primaryService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, Name: "upgrade-test-primary", + }} + replicaService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, Name: "upgrade-test-replicas", + }} + + _, err = reconcilerWithoutCertManager.reconcileClusterCertificate(ctx, root, cluster, primaryService, replicaService) + assert.NilError(t, err) + + rootSecret := &corev1.Secret{} + assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ + Name: naming.PostgresRootCASecret(cluster).Name, + Namespace: namespace, + }, rootSecret)) + assert.Equal(t, rootSecret.Annotations["cert-manager.io/certificate-name"], "") + + clusterCertSecret := &corev1.Secret{} + assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf(naming.ClusterCertSecret, cluster.Name), + Namespace: namespace, + }, clusterCertSecret)) + originalCertData := clusterCertSecret.Data["tls.crt"] + + reconcilerWithCertManager := &Reconciler{ + Client: tClient, + Owner: ControllerName, + CertManagerCtrlFunc: mockCertManagerCtrlFunc, + RestConfig: &rest.Config{}, + } + + t.Run("isRootCACertManagerManaged returns false for internal PKI root", func(t *testing.T) { + managed, err := reconcilerWithCertManager.isRootCACertManagerManaged(ctx, cluster) + assert.NilError(t, err) + assert.Assert(t, !managed) + }) + + t.Run("reconcileClusterCertificate uses internal PKI after upgrade", func(t *testing.T) { + _, err := reconcilerWithCertManager.reconcileClusterCertificate(ctx, root, cluster, primaryService, replicaService) + assert.NilError(t, err) + + updatedCertSecret := &corev1.Secret{} + assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf(naming.ClusterCertSecret, cluster.Name), + Namespace: namespace, + }, updatedCertSecret)) + + assert.DeepEqual(t, originalCertData, updatedCertSecret.Data["tls.crt"]) + }) + + t.Run("isRootCACertManagerManaged returns true for cert-manager root", func(t *testing.T) { + rootSecret.Annotations = map[string]string{ + "cert-manager.io/certificate-name": "test-ca-cert", + } + assert.NilError(t, tClient.Update(ctx, rootSecret)) + + managed, err := reconcilerWithCertManager.isRootCACertManagerManaged(ctx, cluster) + assert.NilError(t, err) + assert.Assert(t, managed) + }) + + t.Run("isRootCACertManagerManaged returns true when no root CA exists", func(t *testing.T) { + freshCluster := testCluster() + freshCluster.Name = "fresh-cluster" + freshCluster.Namespace = namespace + assert.NilError(t, tClient.Create(ctx, freshCluster)) + + managed, err := reconcilerWithCertManager.isRootCACertManagerManaged(ctx, freshCluster) + assert.NilError(t, err) + assert.Assert(t, managed) + }) + + t.Run("isRootCACertManagerManaged returns false when CustomRootCATLSSecret is set", func(t *testing.T) { + clusterWithCustomRoot := testCluster() + clusterWithCustomRoot.Name = cluster.Name + clusterWithCustomRoot.Namespace = namespace + clusterWithCustomRoot.Spec.CustomRootCATLSSecret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "my-custom-root-ca"}, + } + + managed, err := reconcilerWithCertManager.isRootCACertManagerManaged(ctx, clusterWithCustomRoot) + assert.NilError(t, err) + assert.Assert(t, !managed) + }) + + t.Run("isRootCACertManagerManaged returns false when cert-manager not installed", func(t *testing.T) { + rNoCertManager := &Reconciler{ + Client: tClient, + Owner: ControllerName, + CertManagerCtrlFunc: certmanager.NewController, + } + + managed, err := rNoCertManager.isRootCACertManagerManaged(ctx, cluster) + assert.NilError(t, err) + assert.Assert(t, !managed) + }) +} + // getCertFromSecret returns a parsed certificate from the named secret func getCertFromSecret( ctx context.Context, tClient client.Client, name, namespace, dataKey string, diff --git a/internal/testing/require/kubernetes.go b/internal/testing/require/kubernetes.go index cab581ec76..f06a7318f3 100644 --- a/internal/testing/require/kubernetes.go +++ b/internal/testing/require/kubernetes.go @@ -118,8 +118,10 @@ func kubernetes3(t TestingT) (*envtest.Environment, client.Client) { ); assert.Check(t, err == nil && len(pkgs) > 0 && pkgs[0].Module != nil, "got %v\n%#v", err, pkgs, ) { - snapshotter, err = filepath.Rel(root, pkgs[0].Module.Dir) - assert.NilError(t, err) + if pkgs[0].Module.Dir != "" { + snapshotter, err = filepath.Rel(root, pkgs[0].Module.Dir) + assert.NilError(t, err) + } } kubernetes.Lock()