From efb08455849d377aa4252ba216b22c19e65db2bb Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Mon, 4 May 2026 14:41:15 +0300 Subject: [PATCH 01/15] K8SPG-1017 leaf cert function needs to check whether the existing secret is already managed by internal PKI before switching to cert-manager --- .../controller/postgrescluster/instance.go | 6 ++--- .../controller/postgrescluster/pgbackrest.go | 8 +++---- .../controller/postgrescluster/pgbouncer.go | 6 ++--- internal/controller/postgrescluster/pki.go | 24 ++++++++++++++++--- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/internal/controller/postgrescluster/instance.go b/internal/controller/postgrescluster/instance.go index 24c601d13..057577044 100644 --- a/internal/controller/postgrescluster/instance.go +++ b/internal/controller/postgrescluster/instance.go @@ -1487,12 +1487,12 @@ func (r *Reconciler) reconcileInstanceCertificates( rootCertificateAuth *pki.RootCertificateAuthority, ) (*corev1.Secret, error) { if cluster.Spec.CustomTLSSecret == nil { - certManagerInstalled, err := r.isCertManagerInstalled(ctx, cluster.Namespace) + certManagerManaged, err := r.isRootCACertManagerManaged(ctx, cluster) if err != nil { - return nil, errors.Wrap(err, "failed to check if cert-manager is installed") + return nil, errors.Wrap(err, "failed to check if cert-manager manages root CA") } - if certManagerInstalled { + if certManagerManaged { return r.reconcileCertManagerInstanceCertificates(ctx, cluster, spec, instance, rootCertificateAuth) } } diff --git a/internal/controller/postgrescluster/pgbackrest.go b/internal/controller/postgrescluster/pgbackrest.go index 52e49515f..8a709d4df 100644 --- a/internal/controller/postgrescluster/pgbackrest.go +++ b/internal/controller/postgrescluster/pgbackrest.go @@ -2227,16 +2227,16 @@ func (r *Reconciler) reconcilePGBackRestSecret(ctx context.Context, // } if err == nil && repoHost != nil { - certManagerInstalled := false + certManagerManaged := false if cluster.Spec.CustomTLSSecret == nil { var certErr error - certManagerInstalled, certErr = r.isCertManagerInstalled(ctx, cluster.Namespace) + certManagerManaged, certErr = r.isRootCACertManagerManaged(ctx, cluster) if certErr != nil { - return errors.Wrap(certErr, "failed to check if cert-manager is installed") + return errors.Wrap(certErr, "failed to check if cert-manager manages root CA") } } - if certManagerInstalled { + if certManagerManaged { err = r.reconcileCertManagerPGBackRestSecret(ctx, cluster, repoHost, rootCA, existing, intent) } else { err = pgbackrest.Secret(ctx, cluster, repoHost, rootCA, existing, intent) diff --git a/internal/controller/postgrescluster/pgbouncer.go b/internal/controller/postgrescluster/pgbouncer.go index 3419ed9c1..be7697f08 100644 --- a/internal/controller/postgrescluster/pgbouncer.go +++ b/internal/controller/postgrescluster/pgbouncer.go @@ -223,12 +223,12 @@ func (r *Reconciler) reconcilePGBouncerSecret( var frontendCertManagerSecret *corev1.Secret if cluster.Spec.Proxy.PGBouncer.CustomTLSSecret == nil { - certManagerInstalled, certErr := r.isCertManagerInstalled(ctx, cluster.Namespace) + certManagerManaged, certErr := r.isRootCACertManagerManaged(ctx, cluster) if certErr != nil { - return nil, errors.Wrap(certErr, "failed to check if cert-manager is installed") + return nil, errors.Wrap(certErr, "failed to check if cert-manager manages root CA") } - if certManagerInstalled { + if certManagerManaged { c := r.CertManagerCtrlFunc(r.Client, r.Scheme, false) dnsNames, dnsErr := naming.ServiceDNSNames(ctx, service, cluster.Spec.ClusterServiceDNSSuffix) diff --git a/internal/controller/postgrescluster/pki.go b/internal/controller/postgrescluster/pki.go index ba742cf81..7fb2984d1 100644 --- a/internal/controller/postgrescluster/pki.go +++ b/internal/controller/postgrescluster/pki.go @@ -231,12 +231,12 @@ func (r *Reconciler) reconcileClusterCertificate( return cluster.Spec.CustomTLSSecret, nil } - certManagerInstalled, err := r.isCertManagerInstalled(ctx, cluster.Namespace) + certManagerManaged, err := r.isRootCACertManagerManaged(ctx, cluster) if err != nil { - return nil, errors.Wrap(err, "failed to check if cert-manager is installed") + return nil, errors.Wrap(err, "failed to check if cert-manager manages root CA") } - if certManagerInstalled { + if certManagerManaged { return r.reconcileCertManagerClusterCertificate(ctx, root, cluster, primaryService, replicaService) } @@ -361,6 +361,24 @@ func (r *Reconciler) reconcileCertManagerClusterCertificate( }), nil } +func (r *Reconciler) isRootCACertManagerManaged(ctx context.Context, cluster *v1beta1.PostgresCluster) (bool, error) { + installed, err := r.isCertManagerInstalled(ctx, cluster.Namespace) + if err != nil || !installed { + return false, err + } + + rootSecret := &corev1.Secret{ObjectMeta: naming.PostgresRootCASecret(cluster)} + err = r.Client.Get(ctx, client.ObjectKeyFromObject(rootSecret), rootSecret) + if err != nil { + if k8serrors.IsNotFound(err) { + return true, nil + } + return false, errors.WithStack(err) + } + + return rootSecret.Annotations["cert-manager.io/certificate-name"] != "", nil +} + func (r *Reconciler) isCertManagerInstalled(ctx context.Context, ns string) (bool, error) { if r.RestConfig == nil { return false, nil From c0c13d2cefab6dad85574fd27a8f1d14c1567e52 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Mon, 4 May 2026 16:39:17 +0300 Subject: [PATCH 02/15] add unit test --- .../controller/postgrescluster/pki_test.go | 143 ++++++++++++++++++ internal/testing/require/kubernetes.go | 6 +- 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/internal/controller/postgrescluster/pki_test.go b/internal/controller/postgrescluster/pki_test.go index 191a12c01..6949a99b8 100644 --- a/internal/controller/postgrescluster/pki_test.go +++ b/internal/controller/postgrescluster/pki_test.go @@ -17,7 +17,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/percona/percona-postgresql-operator/v2/internal/naming" @@ -380,6 +382,147 @@ func TestReconcileCerts(t *testing.T) { }) } +type mockCertManagerController struct{} + +func (m *mockCertManagerController) Check(context.Context, *rest.Config, string) error { return nil } +func (m *mockCertManagerController) ApplyIssuer(context.Context, *v1beta1.PostgresCluster) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyCAIssuer(context.Context, *v1beta1.PostgresCluster) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyCACertificate(context.Context, *v1beta1.PostgresCluster) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyClusterCertificate(context.Context, *v1beta1.PostgresCluster, []string) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyInstanceCertificate(context.Context, *v1beta1.PostgresCluster, string, []string) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyPGBouncerCertificate(context.Context, *v1beta1.PostgresCluster, []string) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyPGBackRestClientCertificate(context.Context, *v1beta1.PostgresCluster) error { + panic("unexpected call") +} +func (m *mockCertManagerController) ApplyPGBackRestRepoCertificate(context.Context, *v1beta1.PostgresCluster, []string) error { + panic("unexpected call") +} + +func mockCertManagerCtrlFunc(_ client.Client, _ *runtime.Scheme, _ bool) certmanager.Controller { + return &mockCertManagerController{} +} + +func TestUpgradeCertManagerDoesNotTakeOverInternalPKI(t *testing.T) { + if strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") { + t.Skip("USE_EXISTING_CLUSTER: Test fails due to garbage collection") + } + + _, tClient := setupKubernetes(t) + require.ParallelCapacity(t, 1) + ctx := t.Context() + namespace := require.Namespace(t, tClient).Name + + reconcilerWithoutCertManager := &Reconciler{ + Client: tClient, + Owner: ControllerName, + CertManagerCtrlFunc: certmanager.NewController, + } + + cluster := testCluster() + cluster.Name = "upgrade-test" + cluster.Namespace = namespace + assert.NilError(t, tClient.Create(ctx, cluster)) + + root, err := reconcilerWithoutCertManager.reconcileRootCertificate(ctx, cluster) + assert.NilError(t, err) + + primaryService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, Name: "upgrade-test-primary", + }} + replicaService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, Name: "upgrade-test-replicas", + }} + + _, err = reconcilerWithoutCertManager.reconcileClusterCertificate(ctx, root, cluster, primaryService, replicaService) + assert.NilError(t, err) + + rootSecret := &corev1.Secret{} + assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ + Name: naming.PostgresRootCASecret(cluster).Name, + Namespace: namespace, + }, rootSecret)) + assert.Equal(t, rootSecret.Annotations["cert-manager.io/certificate-name"], "") + + clusterCertSecret := &corev1.Secret{} + assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf(naming.ClusterCertSecret, cluster.Name), + Namespace: namespace, + }, clusterCertSecret)) + originalCertData := clusterCertSecret.Data["tls.crt"] + + reconcilerWithCertManager := &Reconciler{ + Client: tClient, + Owner: ControllerName, + CertManagerCtrlFunc: mockCertManagerCtrlFunc, + RestConfig: &rest.Config{}, + } + + t.Run("isRootCACertManagerManaged returns false for internal PKI root", func(t *testing.T) { + managed, err := reconcilerWithCertManager.isRootCACertManagerManaged(ctx, cluster) + assert.NilError(t, err) + assert.Assert(t, !managed) + }) + + t.Run("reconcileClusterCertificate uses internal PKI after upgrade", func(t *testing.T) { + _, err := reconcilerWithCertManager.reconcileClusterCertificate(ctx, root, cluster, primaryService, replicaService) + assert.NilError(t, err) + + updatedCertSecret := &corev1.Secret{} + assert.NilError(t, tClient.Get(ctx, types.NamespacedName{ + Name: fmt.Sprintf(naming.ClusterCertSecret, cluster.Name), + Namespace: namespace, + }, updatedCertSecret)) + + assert.DeepEqual(t, originalCertData, updatedCertSecret.Data["tls.crt"]) + }) + + t.Run("isRootCACertManagerManaged returns true for cert-manager root", func(t *testing.T) { + rootSecret.Annotations = map[string]string{ + "cert-manager.io/certificate-name": "test-ca-cert", + } + assert.NilError(t, tClient.Update(ctx, rootSecret)) + + managed, err := reconcilerWithCertManager.isRootCACertManagerManaged(ctx, cluster) + assert.NilError(t, err) + assert.Assert(t, managed) + }) + + t.Run("isRootCACertManagerManaged returns true when no root CA exists", func(t *testing.T) { + freshCluster := testCluster() + freshCluster.Name = "fresh-cluster" + freshCluster.Namespace = namespace + assert.NilError(t, tClient.Create(ctx, freshCluster)) + + managed, err := reconcilerWithCertManager.isRootCACertManagerManaged(ctx, freshCluster) + assert.NilError(t, err) + assert.Assert(t, managed) + }) + + t.Run("isRootCACertManagerManaged returns false when cert-manager not installed", func(t *testing.T) { + rNoCertManager := &Reconciler{ + Client: tClient, + Owner: ControllerName, + CertManagerCtrlFunc: certmanager.NewController, + } + + managed, err := rNoCertManager.isRootCACertManagerManaged(ctx, cluster) + assert.NilError(t, err) + assert.Assert(t, !managed) + }) +} + // getCertFromSecret returns a parsed certificate from the named secret func getCertFromSecret( ctx context.Context, tClient client.Client, name, namespace, dataKey string, diff --git a/internal/testing/require/kubernetes.go b/internal/testing/require/kubernetes.go index cab581ec7..f06a7318f 100644 --- a/internal/testing/require/kubernetes.go +++ b/internal/testing/require/kubernetes.go @@ -118,8 +118,10 @@ func kubernetes3(t TestingT) (*envtest.Environment, client.Client) { ); assert.Check(t, err == nil && len(pkgs) > 0 && pkgs[0].Module != nil, "got %v\n%#v", err, pkgs, ) { - snapshotter, err = filepath.Rel(root, pkgs[0].Module.Dir) - assert.NilError(t, err) + if pkgs[0].Module.Dir != "" { + snapshotter, err = filepath.Rel(root, pkgs[0].Module.Dir) + assert.NilError(t, err) + } } kubernetes.Lock() From 9909d88dcc97a7d5f4d534f0bab4c68e1e96bb89 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 5 May 2026 13:50:35 +0300 Subject: [PATCH 03/15] improve e2e test to include the scenario where cluster starts with pki certs and cert manager is installed afterwards --- .../cert-manager-tls/00-deploy-operator.yaml | 2 - .../tests/cert-manager-tls/01-assert.yaml | 30 +---- .../cert-manager-tls/01-create-cluster.yaml | 2 +- .../02-verify-internal-pki.yaml | 47 +++++++ .../03-verify-tls-secrets.yaml | 120 ++--------------- .../tests/cert-manager-tls/04-write-data.yaml | 3 +- .../05-deploy-cert-manager.yaml | 16 +++ .../06-verify-internal-pki-preserved.yaml | 46 +++++++ .../07-verify-tls-after-certmanager.yaml | 46 +++++++ .../cert-manager-tls/08-delete-cluster.yaml | 21 +++ .../tests/cert-manager-tls/09-assert.yaml | 104 +++++++++++++++ .../cert-manager-tls/09-create-cluster.yaml | 12 ++ ... => 10-verify-cert-manager-resources.yaml} | 0 .../11-verify-tls-secrets.yaml | 125 ++++++++++++++++++ .../tests/cert-manager-tls/12-write-data.yaml | 16 +++ .../{05-assert.yaml => 13-assert.yaml} | 4 +- ...primary.yaml => 13-read-from-primary.yaml} | 2 +- .../{06-assert.yaml => 14-assert.yaml} | 4 +- ...ion.yaml => 14-verify-tls-connection.yaml} | 2 +- ...ncer.yaml => 15-verify-tls-pgbouncer.yaml} | 0 ...on.yaml => 16-verify-tls-replication.yaml} | 0 ...est.yaml => 17-verify-tls-pgbackrest.yaml} | 0 ...ry.yaml => 18-cert-deletion-recovery.yaml} | 0 ...yaml => 19-write-data-after-recovery.yaml} | 0 ...aml => 20-verify-data-after-recovery.yaml} | 0 ...yaml => 21-verify-tls-after-recovery.yaml} | 0 ...uired.yaml => 22-verify-tls-required.yaml} | 0 ...tion.yaml => 23-verify-cert-rotation.yaml} | 0 28 files changed, 456 insertions(+), 146 deletions(-) create mode 100644 e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml create mode 100644 e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml create mode 100644 e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml create mode 100644 e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml create mode 100644 e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml create mode 100644 e2e-tests/tests/cert-manager-tls/09-assert.yaml create mode 100644 e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml rename e2e-tests/tests/cert-manager-tls/{02-verify-cert-manager-resources.yaml => 10-verify-cert-manager-resources.yaml} (100%) create mode 100644 e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml create mode 100644 e2e-tests/tests/cert-manager-tls/12-write-data.yaml rename e2e-tests/tests/cert-manager-tls/{05-assert.yaml => 13-assert.yaml} (70%) rename e2e-tests/tests/cert-manager-tls/{05-read-from-primary.yaml => 13-read-from-primary.yaml} (77%) rename e2e-tests/tests/cert-manager-tls/{06-assert.yaml => 14-assert.yaml} (67%) rename e2e-tests/tests/cert-manager-tls/{06-verify-tls-connection.yaml => 14-verify-tls-connection.yaml} (92%) rename e2e-tests/tests/cert-manager-tls/{07-verify-tls-pgbouncer.yaml => 15-verify-tls-pgbouncer.yaml} (100%) rename e2e-tests/tests/cert-manager-tls/{08-verify-tls-replication.yaml => 16-verify-tls-replication.yaml} (100%) rename e2e-tests/tests/cert-manager-tls/{09-verify-tls-pgbackrest.yaml => 17-verify-tls-pgbackrest.yaml} (100%) rename e2e-tests/tests/cert-manager-tls/{10-cert-deletion-recovery.yaml => 18-cert-deletion-recovery.yaml} (100%) rename e2e-tests/tests/cert-manager-tls/{11-write-data-after-recovery.yaml => 19-write-data-after-recovery.yaml} (100%) rename e2e-tests/tests/cert-manager-tls/{12-verify-data-after-recovery.yaml => 20-verify-data-after-recovery.yaml} (100%) rename e2e-tests/tests/cert-manager-tls/{13-verify-tls-after-recovery.yaml => 21-verify-tls-after-recovery.yaml} (100%) rename e2e-tests/tests/cert-manager-tls/{14-verify-tls-required.yaml => 22-verify-tls-required.yaml} (100%) rename e2e-tests/tests/cert-manager-tls/{15-verify-cert-rotation.yaml => 23-verify-cert-rotation.yaml} (100%) diff --git a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml index beef88442..1aaca58be 100644 --- a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml +++ b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml @@ -9,7 +9,5 @@ commands: source ../../functions init_temp_dir # do this only in the first TestStep - deploy_cert_manager deploy_operator deploy_client - deploy_cmctl diff --git a/e2e-tests/tests/cert-manager-tls/01-assert.yaml b/e2e-tests/tests/cert-manager-tls/01-assert.yaml index dd9990a9e..b5c0443c3 100644 --- a/e2e-tests/tests/cert-manager-tls/01-assert.yaml +++ b/e2e-tests/tests/cert-manager-tls/01-assert.yaml @@ -58,34 +58,6 @@ metadata: status: succeeded: 1 --- -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: cert-manager-tls - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGCluster - name: cert-manager-tls - controller: true - blockOwnerDeletion: true - finalizers: - - postgres-operator.crunchydata.com/finalizer -status: - instances: - - name: instance1 - readyReplicas: 3 - replicas: 3 - updatedReplicas: 3 - observedGeneration: 1 - pgbackrest: - repos: - - name: repo1 - stanzaCreated: true - proxy: - pgBouncer: - readyReplicas: 3 - replicas: 3 ---- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: @@ -101,4 +73,4 @@ status: size: 3 ready: 3 size: 3 - state: ready \ No newline at end of file + state: ready diff --git a/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml b/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml index eb0ab4d3c..18343ad97 100644 --- a/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml +++ b/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml @@ -9,4 +9,4 @@ commands: source ../../functions get_cr "cert-manager-tls" \ - | kubectl -n "${NAMESPACE}" apply -f - \ No newline at end of file + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml b/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml new file mode 100644 index 000000000..0fa8e7cef --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml @@ -0,0 +1,47 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + root_ca_annotation=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + + if [[ -n "$root_ca_annotation" ]]; then + echo "Root CA secret has cert-manager annotation but cert-manager is not installed!" + exit 1 + fi + + cert_count=$(kubectl -n "$NAMESPACE" get certificate 2>/dev/null | grep -c cert-manager-tls || true) + if [[ "$cert_count" -gt 0 ]]; then + echo "Found cert-manager Certificate resources but cert-manager should not be installed" + exit 1 + fi + + instance_sts=$(kubectl -n "$NAMESPACE" get sts \ + -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/instance-set=instance1 \ + -o jsonpath='{.items[*].metadata.name}') + + for sts_name in $instance_sts; do + secret_name="${sts_name}-certs" + + cm_annotation=$(kubectl -n "$NAMESPACE" get secret "$secret_name" \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + if [[ -n "$cm_annotation" ]]; then + echo "Instance secret $secret_name has cert-manager annotation unexpectedly" + exit 1 + fi + + for key in dns.crt dns.key patroni.ca-roots patroni.crt-combined; do + escaped_key="${key//./\\.}" + val=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath="{.data.${escaped_key}}") + if [[ -z "$val" ]]; then + echo "Instance secret $secret_name is missing key: $key" + exit 1 + fi + done + done diff --git a/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml b/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml index d1d67003c..55fd6db3f 100644 --- a/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml +++ b/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 120 +timeout: 30 commands: - script: |- set -o errexit @@ -8,118 +8,24 @@ commands: source ../../functions - verify_secret_data() { - local secret_name="$1" - shift - for key in "$@"; do - escaped_key="${key//./\\.}" - val=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath="{.data.${escaped_key}}") - if [[ -z "$val" ]]; then - echo "Secret $secret_name is missing key: $key" - return 1 - fi - done - } + ssl_info=$(run_psql_local "SHOW ssl;" "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") - retry 12 5 verify_secret_data cert-manager-tls-cluster-ca-cert tls.crt tls.key ca.crt - - ca_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-ca-cert -o jsonpath='{.type}') - if [[ "$ca_secret_type" != "kubernetes.io/tls" ]]; then - echo "CA secret type is incorrect: $ca_secret_type (expected kubernetes.io/tls)" - exit 1 - fi - - ca_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-ca-cert -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$ca_issuer_name" != "cert-manager-tls-ca-issuer" ]]; then - echo "CA secret issuer annotation is incorrect: $ca_issuer_name" - exit 1 - fi - - retry 12 5 verify_secret_data cert-manager-tls-cluster-cert tls.crt tls.key ca.crt - - cluster_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert -o jsonpath='{.type}') - if [[ "$cluster_secret_type" != "kubernetes.io/tls" ]]; then - echo "Cluster TLS secret type is incorrect: $cluster_secret_type (expected kubernetes.io/tls)" - exit 1 - fi - - cluster_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$cluster_issuer_name" != "cert-manager-tls-tls-issuer" ]]; then - echo "Cluster TLS secret issuer annotation is incorrect: $cluster_issuer_name" - exit 1 - fi - - retry 12 5 verify_secret_data cert-manager-tls-pgbouncer-frontend-tls tls.crt tls.key ca.crt - - pgb_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.type}') - if [[ "$pgb_secret_type" != "kubernetes.io/tls" ]]; then - echo "PgBouncer TLS secret type is incorrect: $pgb_secret_type (expected kubernetes.io/tls)" - exit 1 - fi - - pgb_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$pgb_issuer_name" != "cert-manager-tls-tls-issuer" ]]; then - echo "PgBouncer TLS secret issuer annotation is incorrect: $pgb_issuer_name" + if [[ "$ssl_info" != *"on"* ]]; then + echo "SSL is not enabled on PostgreSQL with internal PKI" exit 1 fi - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-client-tls tls.crt tls.key + repl_ssl_count=$(run_psql_local \ + "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_replication r ON s.pid = r.pid WHERE s.ssl = true;" \ + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@cert-manager-tls-primary") + repl_ssl_count=$(echo "$repl_ssl_count" | tr -d '[:space:]') - pgbr_client_issuer=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-client-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$pgbr_client_issuer" != "cert-manager-tls-tls-issuer" ]]; then - echo "pgBackRest client TLS secret issuer annotation is incorrect: $pgbr_client_issuer" + if [[ "$repl_ssl_count" -lt 1 ]]; then + echo "No SSL replication connections found with internal PKI, got: $repl_ssl_count" exit 1 fi - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-repo-tls tls.crt tls.key - - pgbr_repo_issuer=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-repo-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$pgbr_repo_issuer" != "cert-manager-tls-tls-issuer" ]]; then - echo "pgBackRest repo TLS secret issuer annotation is incorrect: $pgbr_repo_issuer" - exit 1 - fi - - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest pgbackrest.ca-roots pgbackrest-client.crt pgbackrest-client.key pgbackrest-repo-host.crt pgbackrest-repo-host.key - - pgbr_client_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest -o jsonpath='{.data.pgbackrest-client\.crt}') - pgbr_client_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-client-tls -o jsonpath='{.data.tls\.crt}') - if [[ "$pgbr_client_cert_in_secret" != "$pgbr_client_cert_from_cm" ]]; then - echo "pgBackRest main secret client cert does not match cert-manager-issued cert" - exit 1 - fi - - pgbr_repo_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest -o jsonpath='{.data.pgbackrest-repo-host\.crt}') - pgbr_repo_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-repo-tls -o jsonpath='{.data.tls\.crt}') - if [[ "$pgbr_repo_cert_in_secret" != "$pgbr_repo_cert_from_cm" ]]; then - echo "pgBackRest main secret repo cert does not match cert-manager-issued cert" - exit 1 - fi - - retry 12 5 verify_secret_data cert-manager-tls-pgbouncer pgbouncer-frontend.crt pgbouncer-frontend.key pgbouncer-frontend.ca-roots - - pgb_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer -o jsonpath='{.data.pgbouncer-frontend\.crt}') - pgb_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.data.tls\.crt}') - if [[ "$pgb_cert_in_secret" != "$pgb_cert_from_cm" ]]; then - echo "pgBouncer main secret frontend cert does not match cert-manager-issued cert" - exit 1 - fi - - instance_sts=$(kubectl -n "$NAMESPACE" get sts -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/instance-set=instance1 -o jsonpath='{.items[*].metadata.name}') - for sts_name in $instance_sts; do - secret_name="${sts_name}-certs" - - secret_type=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath='{.type}') - if [[ "$secret_type" != "kubernetes.io/tls" ]]; then - echo "Instance secret $secret_name type is incorrect: $secret_type (expected kubernetes.io/tls)" - exit 1 - fi - - issuer_name=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') - if [[ "$issuer_name" != "cert-manager-tls-tls-issuer" ]]; then - echo "Instance secret $secret_name issuer annotation is incorrect: $issuer_name" - exit 1 - fi + pg_cert_serial=$(run_comand_on_pod "openssl s_client -connect cert-manager-tls-primary:5432 -starttls postgres <<< '' 2>/dev/null | openssl x509 -noout -serial" | tr -d '[:space:]') - verify_secret_data "$secret_name" tls.crt tls.key dns.crt dns.key patroni.ca-roots patroni.crt-combined pgbackrest-server.crt pgbackrest-server.key - echo "Instance secret $secret_name is valid" - done + kubectl create configmap -n "${NAMESPACE}" internal-pki-cert-serial \ + --from-literal=pg-serial="$pg_cert_serial" diff --git a/e2e-tests/tests/cert-manager-tls/04-write-data.yaml b/e2e-tests/tests/cert-manager-tls/04-write-data.yaml index 8871dd185..375cfd968 100644 --- a/e2e-tests/tests/cert-manager-tls/04-write-data.yaml +++ b/e2e-tests/tests/cert-manager-tls/04-write-data.yaml @@ -1,5 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep +timeout: 30 commands: - script: |- set -o errexit @@ -13,4 +14,4 @@ commands: run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ - "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" \ No newline at end of file + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" diff --git a/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml b/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml new file mode 100644 index 000000000..7761a6bd9 --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + deploy_cert_manager + deploy_cmctl + + kubectl -n "$NAMESPACE" delete pod -l postgres-operator.crunchydata.com/role=pgbouncer,postgres-operator.crunchydata.com/cluster=cert-manager-tls + + wait_cluster_consistency cert-manager-tls diff --git a/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml b/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml new file mode 100644 index 000000000..d32b70dba --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml @@ -0,0 +1,46 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + root_ca_annotation=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + + if [[ -n "$root_ca_annotation" ]]; then + echo "FAIL: Root CA secret was taken over by cert-manager!" + exit 1 + fi + + cert_count=$(kubectl -n "$NAMESPACE" get certificate 2>/dev/null | grep -c cert-manager-tls || true) + if [[ "$cert_count" -gt 0 ]]; then + echo "FAIL: cert-manager Certificate resources were created for the existing cluster" + exit 1 + fi + + instance_sts=$(kubectl -n "$NAMESPACE" get sts \ + -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/instance-set=instance1 \ + -o jsonpath='{.items[*].metadata.name}') + + for sts_name in $instance_sts; do + secret_name="${sts_name}-certs" + + cm_annotation=$(kubectl -n "$NAMESPACE" get secret "$secret_name" \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + if [[ -n "$cm_annotation" ]]; then + echo "FAIL: Instance secret $secret_name was taken over by cert-manager" + exit 1 + fi + done + + pg_cert_serial=$(run_comand_on_pod "openssl s_client -connect cert-manager-tls-primary:5432 -starttls postgres <<< '' 2>/dev/null | openssl x509 -noout -serial" | tr -d '[:space:]') + pg_cert_serial_before=$(kubectl -n "$NAMESPACE" get configmap internal-pki-cert-serial -o jsonpath='{.data.pg-serial}') + + if [[ "$pg_cert_serial" != "$pg_cert_serial_before" ]]; then + echo "FAIL: PostgreSQL certificate changed after cert-manager installation!" + exit 1 + fi diff --git a/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml b/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml new file mode 100644 index 000000000..fb07dfd0c --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml @@ -0,0 +1,46 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + ssl_info=$(run_psql_local "SHOW ssl;" "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") + + if [[ "$ssl_info" != *"on"* ]]; then + echo "SSL is not enabled on PostgreSQL after cert-manager installation" + exit 1 + fi + + repl_ssl_count=$(run_psql_local \ + "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_replication r ON s.pid = r.pid WHERE s.ssl = true;" \ + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@cert-manager-tls-primary") + repl_ssl_count=$(echo "$repl_ssl_count" | tr -d '[:space:]') + + if [[ "$repl_ssl_count" -lt 1 ]]; then + echo "No SSL replication connections found after cert-manager install, got: $repl_ssl_count" + exit 1 + fi + + pgb_ssl=$(run_psql_local "SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid();" "cert-manager-tls:$(get_psql_user_pass cert-manager-tls-pguser-cert-manager-tls)@cert-manager-tls-pgbouncer/postgres") + + if [[ "$pgb_ssl" != *"t"* ]]; then + echo "PgBouncer-to-PostgreSQL connection is not using SSL after cert-manager install" + exit 1 + fi + + instance=$(kubectl -n "$NAMESPACE" get pod \ + -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/role=primary \ + -o jsonpath='{.items[0].metadata.name}') + + kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info + + data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") + + if [[ "$data" != *"100500"* ]]; then + echo "FAIL: Data is not intact after cert-manager installation" + exit 1 + fi diff --git a/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml b/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml new file mode 100644 index 000000000..8f2c5a223 --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kubectl -n "$NAMESPACE" delete perconapgcluster cert-manager-tls --wait=true || true + kubectl -n "$NAMESPACE" delete postgrescluster cert-manager-tls --wait=true || true + + kubectl -n "$NAMESPACE" delete pvc -l postgres-operator.crunchydata.com/cluster=cert-manager-tls || true + kubectl -n "$NAMESPACE" delete secret -l postgres-operator.crunchydata.com/cluster=cert-manager-tls || true + kubectl -n "$NAMESPACE" delete configmap internal-pki-cert-serial || true + + cluster_deleted() { + ! kubectl -n "$NAMESPACE" get postgrescluster cert-manager-tls 2>/dev/null + } + retry 12 5 cluster_deleted diff --git a/e2e-tests/tests/cert-manager-tls/09-assert.yaml b/e2e-tests/tests/cert-manager-tls/09-assert.yaml new file mode 100644 index 000000000..e6d50016e --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/09-assert.yaml @@ -0,0 +1,104 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: cert-manager-tls + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + ownerReferences: + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: cert-manager-tls + controller: true + blockOwnerDeletion: true +status: + observedGeneration: 1 + replicas: 1 + readyReplicas: 1 + currentReplicas: 1 + updatedReplicas: 1 + collisionCount: 0 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: cert-manager-tls-pgbouncer + labels: + postgres-operator.crunchydata.com/cluster: cert-manager-tls + postgres-operator.crunchydata.com/role: pgbouncer + ownerReferences: + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: cert-manager-tls + controller: true + blockOwnerDeletion: true +status: + observedGeneration: 1 + replicas: 3 + updatedReplicas: 3 + readyReplicas: 3 +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: cert-manager-tls + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 +--- +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: cert-manager-tls + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + name: cert-manager-tls + controller: true + blockOwnerDeletion: true + finalizers: + - postgres-operator.crunchydata.com/finalizer +status: + instances: + - name: instance1 + readyReplicas: 3 + replicas: 3 + updatedReplicas: 3 + observedGeneration: 1 + pgbackrest: + repos: + - name: repo1 + stanzaCreated: true + proxy: + pgBouncer: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: cert-manager-tls +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml b/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml new file mode 100644 index 000000000..18343ad97 --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr "cert-manager-tls" \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/cert-manager-tls/02-verify-cert-manager-resources.yaml b/e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/02-verify-cert-manager-resources.yaml rename to e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml diff --git a/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml b/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml new file mode 100644 index 000000000..d1d67003c --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml @@ -0,0 +1,125 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + verify_secret_data() { + local secret_name="$1" + shift + for key in "$@"; do + escaped_key="${key//./\\.}" + val=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath="{.data.${escaped_key}}") + if [[ -z "$val" ]]; then + echo "Secret $secret_name is missing key: $key" + return 1 + fi + done + } + + retry 12 5 verify_secret_data cert-manager-tls-cluster-ca-cert tls.crt tls.key ca.crt + + ca_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-ca-cert -o jsonpath='{.type}') + if [[ "$ca_secret_type" != "kubernetes.io/tls" ]]; then + echo "CA secret type is incorrect: $ca_secret_type (expected kubernetes.io/tls)" + exit 1 + fi + + ca_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-ca-cert -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$ca_issuer_name" != "cert-manager-tls-ca-issuer" ]]; then + echo "CA secret issuer annotation is incorrect: $ca_issuer_name" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-cluster-cert tls.crt tls.key ca.crt + + cluster_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert -o jsonpath='{.type}') + if [[ "$cluster_secret_type" != "kubernetes.io/tls" ]]; then + echo "Cluster TLS secret type is incorrect: $cluster_secret_type (expected kubernetes.io/tls)" + exit 1 + fi + + cluster_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-cluster-cert -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$cluster_issuer_name" != "cert-manager-tls-tls-issuer" ]]; then + echo "Cluster TLS secret issuer annotation is incorrect: $cluster_issuer_name" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbouncer-frontend-tls tls.crt tls.key ca.crt + + pgb_secret_type=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.type}') + if [[ "$pgb_secret_type" != "kubernetes.io/tls" ]]; then + echo "PgBouncer TLS secret type is incorrect: $pgb_secret_type (expected kubernetes.io/tls)" + exit 1 + fi + + pgb_issuer_name=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$pgb_issuer_name" != "cert-manager-tls-tls-issuer" ]]; then + echo "PgBouncer TLS secret issuer annotation is incorrect: $pgb_issuer_name" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-client-tls tls.crt tls.key + + pgbr_client_issuer=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-client-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$pgbr_client_issuer" != "cert-manager-tls-tls-issuer" ]]; then + echo "pgBackRest client TLS secret issuer annotation is incorrect: $pgbr_client_issuer" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-repo-tls tls.crt tls.key + + pgbr_repo_issuer=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-repo-tls -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$pgbr_repo_issuer" != "cert-manager-tls-tls-issuer" ]]; then + echo "pgBackRest repo TLS secret issuer annotation is incorrect: $pgbr_repo_issuer" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbackrest pgbackrest.ca-roots pgbackrest-client.crt pgbackrest-client.key pgbackrest-repo-host.crt pgbackrest-repo-host.key + + pgbr_client_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest -o jsonpath='{.data.pgbackrest-client\.crt}') + pgbr_client_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-client-tls -o jsonpath='{.data.tls\.crt}') + if [[ "$pgbr_client_cert_in_secret" != "$pgbr_client_cert_from_cm" ]]; then + echo "pgBackRest main secret client cert does not match cert-manager-issued cert" + exit 1 + fi + + pgbr_repo_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest -o jsonpath='{.data.pgbackrest-repo-host\.crt}') + pgbr_repo_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbackrest-repo-tls -o jsonpath='{.data.tls\.crt}') + if [[ "$pgbr_repo_cert_in_secret" != "$pgbr_repo_cert_from_cm" ]]; then + echo "pgBackRest main secret repo cert does not match cert-manager-issued cert" + exit 1 + fi + + retry 12 5 verify_secret_data cert-manager-tls-pgbouncer pgbouncer-frontend.crt pgbouncer-frontend.key pgbouncer-frontend.ca-roots + + pgb_cert_in_secret=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer -o jsonpath='{.data.pgbouncer-frontend\.crt}') + pgb_cert_from_cm=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.data.tls\.crt}') + if [[ "$pgb_cert_in_secret" != "$pgb_cert_from_cm" ]]; then + echo "pgBouncer main secret frontend cert does not match cert-manager-issued cert" + exit 1 + fi + + instance_sts=$(kubectl -n "$NAMESPACE" get sts -l postgres-operator.crunchydata.com/cluster=cert-manager-tls,postgres-operator.crunchydata.com/instance-set=instance1 -o jsonpath='{.items[*].metadata.name}') + for sts_name in $instance_sts; do + secret_name="${sts_name}-certs" + + secret_type=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath='{.type}') + if [[ "$secret_type" != "kubernetes.io/tls" ]]; then + echo "Instance secret $secret_name type is incorrect: $secret_type (expected kubernetes.io/tls)" + exit 1 + fi + + issuer_name=$(kubectl -n "$NAMESPACE" get secret "$secret_name" -o jsonpath='{.metadata.annotations.cert-manager\.io/issuer-name}') + if [[ "$issuer_name" != "cert-manager-tls-tls-issuer" ]]; then + echo "Instance secret $secret_name issuer annotation is incorrect: $issuer_name" + exit 1 + fi + + verify_secret_data "$secret_name" tls.crt tls.key dns.crt dns.key patroni.ca-roots patroni.crt-combined pgbackrest-server.crt pgbackrest-server.key + echo "Instance secret $secret_name is valid" + done diff --git a/e2e-tests/tests/cert-manager-tls/12-write-data.yaml b/e2e-tests/tests/cert-manager-tls/12-write-data.yaml new file mode 100644 index 000000000..81e1c7ba1 --- /dev/null +++ b/e2e-tests/tests/cert-manager-tls/12-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_psql_local \ + 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" + + run_psql_local \ + '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ + "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" diff --git a/e2e-tests/tests/cert-manager-tls/05-assert.yaml b/e2e-tests/tests/cert-manager-tls/13-assert.yaml similarity index 70% rename from e2e-tests/tests/cert-manager-tls/05-assert.yaml rename to e2e-tests/tests/cert-manager-tls/13-assert.yaml index 80d86f1bf..7ed7e0b43 100644 --- a/e2e-tests/tests/cert-manager-tls/05-assert.yaml +++ b/e2e-tests/tests/cert-manager-tls/13-assert.yaml @@ -5,6 +5,6 @@ timeout: 30 kind: ConfigMap apiVersion: v1 metadata: - name: 05-read-from-primary + name: 13-read-from-primary data: - data: ' 100500' \ No newline at end of file + data: ' 100500' diff --git a/e2e-tests/tests/cert-manager-tls/05-read-from-primary.yaml b/e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml similarity index 77% rename from e2e-tests/tests/cert-manager-tls/05-read-from-primary.yaml rename to e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml index 2b8166ba5..bb5b3847c 100644 --- a/e2e-tests/tests/cert-manager-tls/05-read-from-primary.yaml +++ b/e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml @@ -10,4 +10,4 @@ commands: data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") - kubectl create configmap -n "${NAMESPACE}" 05-read-from-primary --from-literal=data="${data}" \ No newline at end of file + kubectl create configmap -n "${NAMESPACE}" 13-read-from-primary --from-literal=data="${data}" diff --git a/e2e-tests/tests/cert-manager-tls/06-assert.yaml b/e2e-tests/tests/cert-manager-tls/14-assert.yaml similarity index 67% rename from e2e-tests/tests/cert-manager-tls/06-assert.yaml rename to e2e-tests/tests/cert-manager-tls/14-assert.yaml index 477b3886c..28a8250ef 100644 --- a/e2e-tests/tests/cert-manager-tls/06-assert.yaml +++ b/e2e-tests/tests/cert-manager-tls/14-assert.yaml @@ -5,6 +5,6 @@ timeout: 30 kind: ConfigMap apiVersion: v1 metadata: - name: 06-verify-tls-connection + name: 14-verify-tls-connection data: - status: verified \ No newline at end of file + status: verified diff --git a/e2e-tests/tests/cert-manager-tls/06-verify-tls-connection.yaml b/e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml similarity index 92% rename from e2e-tests/tests/cert-manager-tls/06-verify-tls-connection.yaml rename to e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml index 3babb0205..641d86d77 100644 --- a/e2e-tests/tests/cert-manager-tls/06-verify-tls-connection.yaml +++ b/e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml @@ -32,4 +32,4 @@ commands: exit 1 fi - kubectl create configmap -n "${NAMESPACE}" 06-verify-tls-connection --from-literal=status="verified" \ No newline at end of file + kubectl create configmap -n "${NAMESPACE}" 14-verify-tls-connection --from-literal=status="verified" diff --git a/e2e-tests/tests/cert-manager-tls/07-verify-tls-pgbouncer.yaml b/e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/07-verify-tls-pgbouncer.yaml rename to e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml diff --git a/e2e-tests/tests/cert-manager-tls/08-verify-tls-replication.yaml b/e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/08-verify-tls-replication.yaml rename to e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml diff --git a/e2e-tests/tests/cert-manager-tls/09-verify-tls-pgbackrest.yaml b/e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/09-verify-tls-pgbackrest.yaml rename to e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml diff --git a/e2e-tests/tests/cert-manager-tls/10-cert-deletion-recovery.yaml b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/10-cert-deletion-recovery.yaml rename to e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml diff --git a/e2e-tests/tests/cert-manager-tls/11-write-data-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/11-write-data-after-recovery.yaml rename to e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml diff --git a/e2e-tests/tests/cert-manager-tls/12-verify-data-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/12-verify-data-after-recovery.yaml rename to e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml diff --git a/e2e-tests/tests/cert-manager-tls/13-verify-tls-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/13-verify-tls-after-recovery.yaml rename to e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml diff --git a/e2e-tests/tests/cert-manager-tls/14-verify-tls-required.yaml b/e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/14-verify-tls-required.yaml rename to e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml diff --git a/e2e-tests/tests/cert-manager-tls/15-verify-cert-rotation.yaml b/e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml similarity index 100% rename from e2e-tests/tests/cert-manager-tls/15-verify-cert-rotation.yaml rename to e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml From eba07006fd5647e5871570ab4a729178702b0d1d Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 5 May 2026 13:59:19 +0300 Subject: [PATCH 04/15] fix timeout configuration --- e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml | 2 +- e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml | 2 +- e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml | 2 +- e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml | 2 +- e2e-tests/tests/cert-manager-tls/04-write-data.yaml | 2 +- e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml | 2 +- .../cert-manager-tls/06-verify-internal-pki-preserved.yaml | 2 +- .../tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml | 2 +- e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml | 2 +- e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml | 2 +- .../cert-manager-tls/10-verify-cert-manager-resources.yaml | 2 +- e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml | 2 +- e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml | 2 +- e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml | 2 +- e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml | 2 +- e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml | 2 +- e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml | 2 +- e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml | 2 +- .../tests/cert-manager-tls/19-write-data-after-recovery.yaml | 2 +- .../tests/cert-manager-tls/20-verify-data-after-recovery.yaml | 2 +- .../tests/cert-manager-tls/21-verify-tls-after-recovery.yaml | 2 +- e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml | 2 +- e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml | 2 +- 23 files changed, 23 insertions(+), 23 deletions(-) diff --git a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml index 1aaca58be..77cab8c26 100644 --- a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml +++ b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 commands: - script: |- set -o errexit @@ -11,3 +10,4 @@ commands: deploy_operator deploy_client + timeout: 10 diff --git a/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml b/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml index 18343ad97..bd9d303ec 100644 --- a/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml +++ b/e2e-tests/tests/cert-manager-tls/01-create-cluster.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 commands: - script: |- set -o errexit @@ -10,3 +9,4 @@ commands: get_cr "cert-manager-tls" \ | kubectl -n "${NAMESPACE}" apply -f - + timeout: 10 diff --git a/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml b/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml index 0fa8e7cef..77c97e277 100644 --- a/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml +++ b/e2e-tests/tests/cert-manager-tls/02-verify-internal-pki.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 60 commands: - script: |- set -o errexit @@ -45,3 +44,4 @@ commands: fi done done + timeout: 60 diff --git a/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml b/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml index 55fd6db3f..d9a2847be 100644 --- a/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml +++ b/e2e-tests/tests/cert-manager-tls/03-verify-tls-secrets.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -29,3 +28,4 @@ commands: kubectl create configmap -n "${NAMESPACE}" internal-pki-cert-serial \ --from-literal=pg-serial="$pg_cert_serial" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/04-write-data.yaml b/e2e-tests/tests/cert-manager-tls/04-write-data.yaml index 375cfd968..f44c9fbaa 100644 --- a/e2e-tests/tests/cert-manager-tls/04-write-data.yaml +++ b/e2e-tests/tests/cert-manager-tls/04-write-data.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -15,3 +14,4 @@ commands: run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml b/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml index 7761a6bd9..716800c0c 100644 --- a/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml +++ b/e2e-tests/tests/cert-manager-tls/05-deploy-cert-manager.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 120 commands: - script: |- set -o errexit @@ -14,3 +13,4 @@ commands: kubectl -n "$NAMESPACE" delete pod -l postgres-operator.crunchydata.com/role=pgbouncer,postgres-operator.crunchydata.com/cluster=cert-manager-tls wait_cluster_consistency cert-manager-tls + timeout: 120 diff --git a/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml b/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml index d32b70dba..a02eccafb 100644 --- a/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml +++ b/e2e-tests/tests/cert-manager-tls/06-verify-internal-pki-preserved.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 60 commands: - script: |- set -o errexit @@ -44,3 +43,4 @@ commands: echo "FAIL: PostgreSQL certificate changed after cert-manager installation!" exit 1 fi + timeout: 60 diff --git a/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml b/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml index fb07dfd0c..8ba9f2779 100644 --- a/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml +++ b/e2e-tests/tests/cert-manager-tls/07-verify-tls-after-certmanager.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -44,3 +43,4 @@ commands: echo "FAIL: Data is not intact after cert-manager installation" exit 1 fi + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml b/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml index 8f2c5a223..555dcf591 100644 --- a/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml +++ b/e2e-tests/tests/cert-manager-tls/08-delete-cluster.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 120 commands: - script: |- set -o errexit @@ -19,3 +18,4 @@ commands: ! kubectl -n "$NAMESPACE" get postgrescluster cert-manager-tls 2>/dev/null } retry 12 5 cluster_deleted + timeout: 120 diff --git a/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml b/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml index 18343ad97..bd9d303ec 100644 --- a/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml +++ b/e2e-tests/tests/cert-manager-tls/09-create-cluster.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 commands: - script: |- set -o errexit @@ -10,3 +9,4 @@ commands: get_cr "cert-manager-tls" \ | kubectl -n "${NAMESPACE}" apply -f - + timeout: 10 diff --git a/e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml b/e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml index f8911c9fe..53699ada5 100644 --- a/e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml +++ b/e2e-tests/tests/cert-manager-tls/10-verify-cert-manager-resources.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 60 commands: - script: |- set -o errexit @@ -60,3 +59,4 @@ commands: fi echo "Instance certificate $cert_name is ready" done + timeout: 60 diff --git a/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml b/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml index d1d67003c..68762bb76 100644 --- a/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml +++ b/e2e-tests/tests/cert-manager-tls/11-verify-tls-secrets.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 120 commands: - script: |- set -o errexit @@ -123,3 +122,4 @@ commands: verify_secret_data "$secret_name" tls.crt tls.key dns.crt dns.key patroni.ca-roots patroni.crt-combined pgbackrest-server.crt pgbackrest-server.key echo "Instance secret $secret_name is valid" done + timeout: 120 diff --git a/e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml b/e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml index bb5b3847c..9ccfa5c30 100644 --- a/e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml +++ b/e2e-tests/tests/cert-manager-tls/13-read-from-primary.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -11,3 +10,4 @@ commands: data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)") kubectl create configmap -n "${NAMESPACE}" 13-read-from-primary --from-literal=data="${data}" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml b/e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml index 641d86d77..16158c4ca 100644 --- a/e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml +++ b/e2e-tests/tests/cert-manager-tls/14-verify-tls-connection.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -33,3 +32,4 @@ commands: fi kubectl create configmap -n "${NAMESPACE}" 14-verify-tls-connection --from-literal=status="verified" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml b/e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml index b12df05cb..7a2fa480b 100644 --- a/e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml +++ b/e2e-tests/tests/cert-manager-tls/15-verify-tls-pgbouncer.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -29,3 +28,4 @@ commands: echo "PgBouncer-to-PostgreSQL connection is not using SSL" exit 1 fi + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml b/e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml index 7b0eb6fa6..25d293ec7 100644 --- a/e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml +++ b/e2e-tests/tests/cert-manager-tls/16-verify-tls-replication.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -17,3 +16,4 @@ commands: echo "No SSL replication connections found, got: $repl_ssl_count" exit 1 fi + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml b/e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml index c8afa2304..43db32b66 100644 --- a/e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml +++ b/e2e-tests/tests/cert-manager-tls/17-verify-tls-pgbackrest.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -34,3 +33,4 @@ commands: # Verify pgBackRest works over TLS, not just cert file existence kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml index 88bd8602b..e21d889e4 100644 --- a/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 300 commands: - script: |- set -o errexit @@ -101,3 +100,4 @@ commands: done wait_cluster_consistency cert-manager-tls + timeout: 300 diff --git a/e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml index d81df23a9..130fe102f 100644 --- a/e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/19-write-data-after-recovery.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -15,3 +14,4 @@ commands: run_psql_local \ '\c newapp \\\ INSERT INTO newApp (id) VALUES (200500)' \ "postgres:$(get_psql_user_pass cert-manager-tls-pguser-postgres)@$(get_psql_user_host cert-manager-tls-pguser-postgres)" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml index bfebc746b..8edd9ac1b 100644 --- a/e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/20-verify-data-after-recovery.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -14,3 +13,4 @@ commands: echo "Missing data in primary" exit 1 fi + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml b/e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml index 2ce31fabb..f4b56621f 100644 --- a/e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/21-verify-tls-after-recovery.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 60 commands: - script: |- set -o errexit @@ -40,3 +39,4 @@ commands: fi kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info + timeout: 60 diff --git a/e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml b/e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml index 34583accc..8302c3d32 100644 --- a/e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml +++ b/e2e-tests/tests/cert-manager-tls/22-verify-tls-required.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 commands: - script: |- set -o errexit @@ -21,3 +20,4 @@ commands: exit 1 fi echo "Correctly rejected non-TLS connection to pgbouncer" + timeout: 30 diff --git a/e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml b/e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml index 0206dd06a..a1d48c07e 100644 --- a/e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml +++ b/e2e-tests/tests/cert-manager-tls/23-verify-cert-rotation.yaml @@ -1,6 +1,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 600 commands: - script: |- set -o errexit @@ -109,3 +108,4 @@ commands: fi kubectl -n "$NAMESPACE" exec "$instance" -c pgbackrest -- pgbackrest info + timeout: 600 From 2f231353bfd89fb2ef822943119ce74dc81326a4 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 5 May 2026 14:14:33 +0300 Subject: [PATCH 05/15] fixes on timeouts now that they are working --- e2e-tests/functions | 5 +++++ e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e-tests/functions b/e2e-tests/functions index 135605988..2f7c8996d 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -1142,6 +1142,11 @@ deploy_cert_manager() { until kubectl get validatingwebhookconfiguration cert-manager-webhook -o jsonpath='{.webhooks[0].clientConfig.caBundle}' | grep -q '[A-Za-z0-9+/=]'; do sleep 5 done + + echo "Waiting for cert-manager webhook service to have endpoints..." + until kubectl -n cert-manager get endpoints cert-manager-webhook -o jsonpath='{.subsets[*].addresses}' | grep -q '.'; do + sleep 5 + done } destroy_cert_manager() { diff --git a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml index 77cab8c26..5ec55d178 100644 --- a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml +++ b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml @@ -10,4 +10,4 @@ commands: deploy_operator deploy_client - timeout: 10 + timeout: 120 From a211bbc94c4aaac02f9b2cb9e662456c77149fd3 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 5 May 2026 14:38:09 +0300 Subject: [PATCH 06/15] fix timeout --- e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml index e21d889e4..51e63da0f 100644 --- a/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml @@ -100,4 +100,4 @@ commands: done wait_cluster_consistency cert-manager-tls - timeout: 300 + timeout: 600 From 29e346ee9d68d59d8c941886862eaacb071e1aaa Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Tue, 5 May 2026 16:38:30 +0300 Subject: [PATCH 07/15] fixes on timeouts --- .../18-cert-deletion-recovery.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml index 51e63da0f..ab2276ac3 100644 --- a/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml +++ b/e2e-tests/tests/cert-manager-tls/18-cert-deletion-recovery.yaml @@ -27,13 +27,13 @@ commands: [[ "$cert_ready" == "True" ]] } - retry 30 10 verify_cert_recreated cert-manager-tls-pgbouncer-cert + retry 60 10 verify_cert_recreated cert-manager-tls-pgbouncer-cert echo "pgBouncer Certificate recreated and ready" - retry 30 10 verify_cert_recreated cert-manager-tls-pgbackrest-repo-cert + retry 60 10 verify_cert_recreated cert-manager-tls-pgbackrest-repo-cert echo "pgBackRest repo Certificate recreated and ready" - retry 30 10 verify_cert_recreated cert-manager-tls-pgbackrest-client-cert + retry 60 10 verify_cert_recreated cert-manager-tls-pgbackrest-client-cert echo "pgBackRest client Certificate recreated and ready" verify_secret_data() { @@ -49,9 +49,9 @@ commands: done } - retry 12 5 verify_secret_data cert-manager-tls-pgbouncer-frontend-tls tls.crt tls.key ca.crt - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-repo-tls tls.crt tls.key - retry 12 5 verify_secret_data cert-manager-tls-pgbackrest-client-tls tls.crt tls.key + retry 30 5 verify_secret_data cert-manager-tls-pgbouncer-frontend-tls tls.crt tls.key ca.crt + retry 30 5 verify_secret_data cert-manager-tls-pgbackrest-repo-tls tls.crt tls.key + retry 30 5 verify_secret_data cert-manager-tls-pgbackrest-client-tls tls.crt tls.key pgb_cert_after=$(kubectl -n "$NAMESPACE" get secret cert-manager-tls-pgbouncer-frontend-tls -o jsonpath='{.data.tls\.crt}') if [[ "$pgb_cert_before" == "$pgb_cert_after" ]]; then @@ -100,4 +100,4 @@ commands: done wait_cluster_consistency cert-manager-tls - timeout: 600 +timeout: 900 From 2cdeb532bcc4bf747cf7cae78e56d3608c267f05 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Wed, 6 May 2026 17:42:05 +0300 Subject: [PATCH 08/15] fixes for when cert manager is installed on a running operator --- cmd/postgres-operator/main.go | 1 + .../controller/postgrescluster/controller.go | 120 ++++++++++++++++-- 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/cmd/postgres-operator/main.go b/cmd/postgres-operator/main.go index bc28e6e46..28fb9b628 100644 --- a/cmd/postgres-operator/main.go +++ b/cmd/postgres-operator/main.go @@ -175,6 +175,7 @@ func addControllersToManager(ctx context.Context, mgr manager.Manager) error { if cm.Controller() == nil { return errors.New("missing controller in manager") } + r.Controller = cm.Controller() if err := mgr.GetFieldIndexer().IndexField( context.Background(), diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go index 285e70595..49c102fd9 100644 --- a/internal/controller/postgrescluster/controller.go +++ b/internal/controller/postgrescluster/controller.go @@ -7,6 +7,7 @@ package postgrescluster import ( "context" "fmt" + "sync" "time" cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -26,12 +27,16 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/percona/percona-postgresql-operator/v2/internal/config" "github.com/percona/percona-postgresql-operator/v2/internal/controller/runtime" @@ -60,17 +65,20 @@ const ( // Reconciler holds resources for the PostgresCluster reconciler type Reconciler struct { - Client client.Client - Scheme *k8sruntime.Scheme - DiscoveryClient *discovery.DiscoveryClient - IsOpenShift bool - Owner client.FieldOwner - PodExec runtime.PodExecutor - Recorder record.EventRecorder - Registration registration.Registration - Tracer trace.Tracer - CertManagerCtrlFunc certmanager.NewControllerFunc - RestConfig *rest.Config + Client client.Client + Scheme *k8sruntime.Scheme + DiscoveryClient *discovery.DiscoveryClient + IsOpenShift bool + Owner client.FieldOwner + PodExec runtime.PodExecutor + Recorder record.EventRecorder + Registration registration.Registration + Tracer trace.Tracer + CertManagerCtrlFunc certmanager.NewControllerFunc + RestConfig *rest.Config + Controller controller.Controller + Cache cache.Cache + certManagerWatchesOnce sync.Once } // +kubebuilder:rbac:groups="",resources="events",verbs={create,patch} @@ -284,6 +292,12 @@ func (r *Reconciler) Reconcile( rootCA, err = r.reconcileRootCertificate(ctx, cluster) } + if err == nil && rootCA != nil { + if certManagerManaged, _ := r.isRootCACertManagerManaged(ctx, cluster); certManagerManaged { + r.registerCertManagerWatches(ctx) + } + } + if err == nil { // Since any existing data directories must be moved prior to bootstrapping the // cluster, further reconciliation will not occur until the directory move Jobs @@ -543,6 +557,8 @@ func (r *Reconciler) SetupWithManager(mgr manager.Manager) error { } } + r.Cache = mgr.GetCache() + // K8SPG-712: Allow overriding default configurations configMapPredicate := builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { @@ -596,3 +612,85 @@ func (r *Reconciler) SetupWithManager(mgr manager.Manager) error { return bldr.Complete(r) } + +// registerCertManagerWatches dynamically registers watches for cert-manager +// Certificate, Issuer, and cert-manager-issued Secret resources for +// case where cert-manager is installed after the operator starts, so the +// watches were not registered in SetupWithManager. +func (r *Reconciler) registerCertManagerWatches(ctx context.Context) { + if r.Controller == nil || r.Cache == nil { + return + } + + r.certManagerWatchesOnce.Do(func() { + log := logging.FromContext(ctx) + + certHandler := handler.TypedEnqueueRequestForOwner[*cmv1.Certificate]( + r.Scheme, r.Client.RESTMapper(), + &v1beta1.PostgresCluster{}, + handler.OnlyControllerOwner(), + ) + if err := r.Controller.Watch(source.Kind( + r.Cache, &cmv1.Certificate{}, certHandler, + )); err != nil { + log.Error(err, "failed to register dynamic watch for Certificates") + return + } + + issuerHandler := handler.TypedEnqueueRequestForOwner[*cmv1.Issuer]( + r.Scheme, r.Client.RESTMapper(), + &v1beta1.PostgresCluster{}, + handler.OnlyControllerOwner(), + ) + if err := r.Controller.Watch(source.Kind( + r.Cache, &cmv1.Issuer{}, issuerHandler, + )); err != nil { + log.Error(err, "failed to register dynamic watch for Issuers") + return + } + secretHandler := handler.TypedEnqueueRequestsFromMapFunc( + func(ctx context.Context, secret *corev1.Secret) []reconcile.Request { + cluster := secret.GetLabels()[naming.LabelCluster] + if len(cluster) > 0 { + return []reconcile.Request{ + {NamespacedName: client.ObjectKey{ + Namespace: secret.GetNamespace(), + Name: cluster, + }}, + } + } + return nil + }, + ) + certManagerSecretPredicate := predicate.TypedFuncs[*corev1.Secret]{ + CreateFunc: func(e event.TypedCreateEvent[*corev1.Secret]) bool { + _, hasCluster := e.Object.GetLabels()[naming.LabelCluster] + _, hasCertAnnotation := e.Object.GetAnnotations()["cert-manager.io/certificate-name"] + return hasCluster && hasCertAnnotation + }, + UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Secret]) bool { + _, hasCluster := e.ObjectNew.GetLabels()[naming.LabelCluster] + _, hasCertAnnotation := e.ObjectNew.GetAnnotations()["cert-manager.io/certificate-name"] + return hasCluster && hasCertAnnotation + }, + DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Secret]) bool { + _, hasCluster := e.Object.GetLabels()[naming.LabelCluster] + _, hasCertAnnotation := e.Object.GetAnnotations()["cert-manager.io/certificate-name"] + return hasCluster && hasCertAnnotation + }, + GenericFunc: func(e event.TypedGenericEvent[*corev1.Secret]) bool { + _, hasCluster := e.Object.GetLabels()[naming.LabelCluster] + _, hasCertAnnotation := e.Object.GetAnnotations()["cert-manager.io/certificate-name"] + return hasCluster && hasCertAnnotation + }, + } + if err := r.Controller.Watch(source.Kind( + r.Cache, &corev1.Secret{}, secretHandler, certManagerSecretPredicate, + )); err != nil { + log.Error(err, "failed to register dynamic watch for cert-manager Secrets") + return + } + + log.Info("dynamically registered cert-manager watches") + }) +} From b89d4fa58ce8e0b31063f4d050bf78eb01469a41 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 7 May 2026 12:05:04 +0300 Subject: [PATCH 09/15] CLOUD-727 fix flaky test TestGetLatestCompleteBackupJob due to list order is non-deterministic --- .../postgrescluster/snapshots_test.go | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/controller/postgrescluster/snapshots_test.go b/internal/controller/postgrescluster/snapshots_test.go index 2da7b65ab..6c90096cc 100644 --- a/internal/controller/postgrescluster/snapshots_test.go +++ b/internal/controller/postgrescluster/snapshots_test.go @@ -46,7 +46,7 @@ func TestReconcileVolumeSnapshots(t *testing.T) { DiscoveryClient: discoveryClient, Recorder: recorder, } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) // Enable snapshots feature gate gate := feature.NewGate() @@ -370,7 +370,7 @@ func TestReconcileDedicatedSnapshotVolume(t *testing.T) { t.Run("SnapshotsDisabledDeletePvc", func(t *testing.T) { // Create cluster without snapshots spec - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name cluster.ObjectMeta.UID = "the-uid-123" @@ -427,7 +427,7 @@ func TestReconcileDedicatedSnapshotVolume(t *testing.T) { t.Run("SnapshotsEnabledCreatePvcNoBackupNoRestore", func(t *testing.T) { // Create cluster with snapshots enabled - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name cluster.ObjectMeta.UID = "the-uid-123" @@ -459,7 +459,7 @@ func TestReconcileDedicatedSnapshotVolume(t *testing.T) { t.Run("SnapshotsEnabledBackupExistsCreateRestore", func(t *testing.T) { // Create cluster with snapshots enabled - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name cluster.ObjectMeta.UID = "the-uid-123" @@ -523,7 +523,7 @@ func TestReconcileDedicatedSnapshotVolume(t *testing.T) { t.Run("SnapshotsEnabledSuccessfulRestoreExists", func(t *testing.T) { // Create cluster with snapshots enabled - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name cluster.ObjectMeta.UID = "the-uid-123" @@ -622,7 +622,7 @@ func TestReconcileDedicatedSnapshotVolume(t *testing.T) { t.Run("SnapshotsEnabledFailedRestoreExists", func(t *testing.T) { // Create cluster with snapshots enabled - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name cluster.ObjectMeta.UID = "the-uid-123" @@ -724,7 +724,7 @@ func TestCreateDedicatedSnapshotVolume(t *testing.T) { Owner: client.FieldOwner(t.Name()), } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name cluster.ObjectMeta.UID = "the-uid-123" @@ -752,7 +752,7 @@ func TestDedicatedSnapshotVolumeRestore(t *testing.T) { Owner: client.FieldOwner(t.Name()), } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name cluster.ObjectMeta.UID = "the-uid-123" @@ -793,7 +793,7 @@ func TestGenerateSnapshotOfDedicatedSnapshotVolume(t *testing.T) { Client: cc, Owner: client.FieldOwner(t.Name()), } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name @@ -824,7 +824,7 @@ func TestGenerateVolumeSnapshot(t *testing.T) { Client: cc, Owner: client.FieldOwner(t.Name()), } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name @@ -853,7 +853,7 @@ func TestGetDedicatedSnapshotVolumeRestoreJob(t *testing.T) { Client: cc, Owner: client.FieldOwner(t.Name()), } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name @@ -908,7 +908,7 @@ func TestGetLatestCompleteBackupJob(t *testing.T) { Client: cc, Owner: client.FieldOwner(t.Name()), } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name @@ -976,9 +976,9 @@ func TestGetLatestCompleteBackupJob(t *testing.T) { }) t.Run("TwoCompleteBackupJobs", func(t *testing.T) { - currentTime := metav1.Now() - earlierTime := metav1.NewTime(currentTime.AddDate(-1, 0, 0)) - assert.Check(t, earlierTime.Before(¤tTime)) + laterTime := metav1.NewTime(time.Now().AddDate(-1, 0, 0)) + earlierTime := metav1.NewTime(laterTime.AddDate(-1, 0, 0)) + assert.Check(t, earlierTime.Before(&laterTime)) job1 := testBackupJob(cluster, "backup-job-two-complete-1") job1.Namespace = ns.Name @@ -996,9 +996,9 @@ func TestGetLatestCompleteBackupJob(t *testing.T) { job1.Status = batchv1.JobStatus{ Succeeded: 1, - CompletionTime: ¤tTime, + CompletionTime: &laterTime, // K8SPG-714: ENVTEST_K8S_VERSION=1.32 - StartTime: ptr.To(metav1.NewTime(currentTime.Add(-time.Minute))), + StartTime: ptr.To(metav1.NewTime(laterTime.Add(-time.Minute))), Conditions: []batchv1.JobCondition{ { Type: batchv1.JobSuccessCriteriaMet, @@ -1162,7 +1162,7 @@ func TestGetSnapshotsForCluster(t *testing.T) { Client: cc, Owner: client.FieldOwner(t.Name()), } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name @@ -1391,7 +1391,7 @@ func TestDeleteSnapshots(t *testing.T) { Owner: client.FieldOwner(t.Name()), DiscoveryClient: discoveryClient, } - ns := setupNamespace(t, cc) + ns := require.Namespace(t, cc) cluster := testCluster() cluster.Namespace = ns.Name From 792a74790a95d7b1adcd09d80c135deebc7a0efc Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 7 May 2026 13:36:19 +0300 Subject: [PATCH 10/15] destory cert manager as a first step of the test in case it exists in the namespace --- e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml index 5ec55d178..0de950d19 100644 --- a/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml +++ b/e2e-tests/tests/cert-manager-tls/00-deploy-operator.yaml @@ -8,6 +8,7 @@ commands: source ../../functions init_temp_dir # do this only in the first TestStep + destroy_cert_manager deploy_operator deploy_client timeout: 120 From a6c2fa4f92e01c97451c6c95fbee02988e3b6ee9 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 7 May 2026 13:44:52 +0300 Subject: [PATCH 11/15] fixes around crd renaming --- e2e-tests/tests/cert-manager-tls/09-assert.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/tests/cert-manager-tls/09-assert.yaml b/e2e-tests/tests/cert-manager-tls/09-assert.yaml index e6d50016e..53ea7777b 100644 --- a/e2e-tests/tests/cert-manager-tls/09-assert.yaml +++ b/e2e-tests/tests/cert-manager-tls/09-assert.yaml @@ -10,7 +10,7 @@ metadata: postgres-operator.crunchydata.com/data: postgres postgres-operator.crunchydata.com/instance-set: instance1 ownerReferences: - - apiVersion: postgres-operator.crunchydata.com/v1beta1 + - apiVersion: upstream.pgv2.percona.com/v1beta1 kind: PostgresCluster name: cert-manager-tls controller: true @@ -31,7 +31,7 @@ metadata: postgres-operator.crunchydata.com/cluster: cert-manager-tls postgres-operator.crunchydata.com/role: pgbouncer ownerReferences: - - apiVersion: postgres-operator.crunchydata.com/v1beta1 + - apiVersion: upstream.pgv2.percona.com/v1beta1 kind: PostgresCluster name: cert-manager-tls controller: true From 9b59493d3abedc57f9a5770ef0d1dd6469c88913 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Thu, 7 May 2026 14:41:48 +0300 Subject: [PATCH 12/15] a fix more --- e2e-tests/tests/cert-manager-tls/09-assert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/tests/cert-manager-tls/09-assert.yaml b/e2e-tests/tests/cert-manager-tls/09-assert.yaml index 53ea7777b..3fa4968d6 100644 --- a/e2e-tests/tests/cert-manager-tls/09-assert.yaml +++ b/e2e-tests/tests/cert-manager-tls/09-assert.yaml @@ -58,7 +58,7 @@ metadata: status: succeeded: 1 --- -apiVersion: postgres-operator.crunchydata.com/v1beta1 +apiVersion: upstream.pgv2.percona.com/v1beta1 kind: PostgresCluster metadata: name: cert-manager-tls From e9875aa96fb11253f823acc370ed03409892b51b Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Mon, 11 May 2026 13:22:21 +0300 Subject: [PATCH 13/15] cr: combine cert manager predicates --- .../controller/postgrescluster/controller.go | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go index 2246b93f7..530a275ae 100644 --- a/internal/controller/postgrescluster/controller.go +++ b/internal/controller/postgrescluster/controller.go @@ -662,28 +662,11 @@ func (r *Reconciler) registerCertManagerWatches(ctx context.Context) { return nil }, ) - certManagerSecretPredicate := predicate.TypedFuncs[*corev1.Secret]{ - CreateFunc: func(e event.TypedCreateEvent[*corev1.Secret]) bool { - _, hasCluster := e.Object.GetLabels()[naming.LabelCluster] - _, hasCertAnnotation := e.Object.GetAnnotations()["cert-manager.io/certificate-name"] - return hasCluster && hasCertAnnotation - }, - UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Secret]) bool { - _, hasCluster := e.ObjectNew.GetLabels()[naming.LabelCluster] - _, hasCertAnnotation := e.ObjectNew.GetAnnotations()["cert-manager.io/certificate-name"] - return hasCluster && hasCertAnnotation - }, - DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Secret]) bool { - _, hasCluster := e.Object.GetLabels()[naming.LabelCluster] - _, hasCertAnnotation := e.Object.GetAnnotations()["cert-manager.io/certificate-name"] - return hasCluster && hasCertAnnotation - }, - GenericFunc: func(e event.TypedGenericEvent[*corev1.Secret]) bool { - _, hasCluster := e.Object.GetLabels()[naming.LabelCluster] - _, hasCertAnnotation := e.Object.GetAnnotations()["cert-manager.io/certificate-name"] - return hasCluster && hasCertAnnotation - }, - } + certManagerSecretPredicate := predicate.NewTypedPredicateFuncs(func(secret *corev1.Secret) bool { + _, hasCluster := secret.GetLabels()[naming.LabelCluster] + _, hasCertAnnotation := secret.GetAnnotations()["cert-manager.io/certificate-name"] + return hasCluster && hasCertAnnotation + }) if err := r.Controller.Watch(source.Kind( r.Cache, &corev1.Secret{}, secretHandler, certManagerSecretPredicate, )); err != nil { From 37fd1eced962af7ea63405744ebd31e239a8eba8 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Mon, 11 May 2026 13:38:42 +0300 Subject: [PATCH 14/15] handle custom secret value when checking for cert manager and remove sync one with atomic boolean flag --- .../controller/postgrescluster/controller.go | 134 +++++++++--------- internal/controller/postgrescluster/pki.go | 4 + .../controller/postgrescluster/pki_test.go | 13 ++ 3 files changed, 86 insertions(+), 65 deletions(-) diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go index 530a275ae..246701529 100644 --- a/internal/controller/postgrescluster/controller.go +++ b/internal/controller/postgrescluster/controller.go @@ -7,7 +7,7 @@ package postgrescluster import ( "context" "fmt" - "sync" + "sync/atomic" "time" cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -65,20 +65,20 @@ const ( // Reconciler holds resources for the PostgresCluster reconciler type Reconciler struct { - Client client.Client - Scheme *k8sruntime.Scheme - DiscoveryClient *discovery.DiscoveryClient - IsOpenShift bool - Owner client.FieldOwner - PodExec runtime.PodExecutor - Recorder record.EventRecorder - Registration registration.Registration - Tracer trace.Tracer - CertManagerCtrlFunc certmanager.NewControllerFunc - RestConfig *rest.Config - Controller controller.Controller - Cache cache.Cache - certManagerWatchesOnce sync.Once + Client client.Client + Scheme *k8sruntime.Scheme + DiscoveryClient *discovery.DiscoveryClient + IsOpenShift bool + Owner client.FieldOwner + PodExec runtime.PodExecutor + Recorder record.EventRecorder + Registration registration.Registration + Tracer trace.Tracer + CertManagerCtrlFunc certmanager.NewControllerFunc + RestConfig *rest.Config + Controller controller.Controller + Cache cache.Cache + certManagerWatchesRegistered atomic.Bool } // +kubebuilder:rbac:groups="",resources="events",verbs={create,patch} @@ -622,58 +622,62 @@ func (r *Reconciler) registerCertManagerWatches(ctx context.Context) { return } - r.certManagerWatchesOnce.Do(func() { - log := logging.FromContext(ctx) + if r.certManagerWatchesRegistered.Load() { + return + } - certHandler := handler.TypedEnqueueRequestForOwner[*cmv1.Certificate]( - r.Scheme, r.Client.RESTMapper(), - &v1beta1.PostgresCluster{}, - handler.OnlyControllerOwner(), - ) - if err := r.Controller.Watch(source.Kind( - r.Cache, &cmv1.Certificate{}, certHandler, - )); err != nil { - log.Error(err, "failed to register dynamic watch for Certificates") - return - } + log := logging.FromContext(ctx) - issuerHandler := handler.TypedEnqueueRequestForOwner[*cmv1.Issuer]( - r.Scheme, r.Client.RESTMapper(), - &v1beta1.PostgresCluster{}, - handler.OnlyControllerOwner(), - ) - if err := r.Controller.Watch(source.Kind( - r.Cache, &cmv1.Issuer{}, issuerHandler, - )); err != nil { - log.Error(err, "failed to register dynamic watch for Issuers") - return - } - secretHandler := handler.TypedEnqueueRequestsFromMapFunc( - func(ctx context.Context, secret *corev1.Secret) []reconcile.Request { - cluster := secret.GetLabels()[naming.LabelCluster] - if len(cluster) > 0 { - return []reconcile.Request{ - {NamespacedName: client.ObjectKey{ - Namespace: secret.GetNamespace(), - Name: cluster, - }}, - } - } - return nil - }, - ) - certManagerSecretPredicate := predicate.NewTypedPredicateFuncs(func(secret *corev1.Secret) bool { - _, hasCluster := secret.GetLabels()[naming.LabelCluster] - _, hasCertAnnotation := secret.GetAnnotations()["cert-manager.io/certificate-name"] - return hasCluster && hasCertAnnotation - }) - if err := r.Controller.Watch(source.Kind( - r.Cache, &corev1.Secret{}, secretHandler, certManagerSecretPredicate, - )); err != nil { - log.Error(err, "failed to register dynamic watch for cert-manager Secrets") - return - } + certHandler := handler.TypedEnqueueRequestForOwner[*cmv1.Certificate]( + r.Scheme, r.Client.RESTMapper(), + &v1beta1.PostgresCluster{}, + handler.OnlyControllerOwner(), + ) + if err := r.Controller.Watch(source.Kind( + r.Cache, &cmv1.Certificate{}, certHandler, + )); err != nil { + log.Error(err, "failed to register dynamic watch for Certificates") + return + } - log.Info("dynamically registered cert-manager watches") + issuerHandler := handler.TypedEnqueueRequestForOwner[*cmv1.Issuer]( + r.Scheme, r.Client.RESTMapper(), + &v1beta1.PostgresCluster{}, + handler.OnlyControllerOwner(), + ) + if err := r.Controller.Watch(source.Kind( + r.Cache, &cmv1.Issuer{}, issuerHandler, + )); err != nil { + log.Error(err, "failed to register dynamic watch for Issuers") + return + } + secretHandler := handler.TypedEnqueueRequestsFromMapFunc( + func(ctx context.Context, secret *corev1.Secret) []reconcile.Request { + cluster := secret.GetLabels()[naming.LabelCluster] + if len(cluster) > 0 { + return []reconcile.Request{ + {NamespacedName: client.ObjectKey{ + Namespace: secret.GetNamespace(), + Name: cluster, + }}, + } + } + return nil + }, + ) + certManagerSecretPredicate := predicate.NewTypedPredicateFuncs(func(secret *corev1.Secret) bool { + _, hasCluster := secret.GetLabels()[naming.LabelCluster] + _, hasCertAnnotation := secret.GetAnnotations()["cert-manager.io/certificate-name"] + return hasCluster && hasCertAnnotation }) + if err := r.Controller.Watch(source.Kind( + r.Cache, &corev1.Secret{}, secretHandler, certManagerSecretPredicate, + )); err != nil { + log.Error(err, "failed to register dynamic watch for cert-manager Secrets") + return + } + + r.certManagerWatchesRegistered.Store(true) + + log.Info("dynamically registered cert-manager watches") } diff --git a/internal/controller/postgrescluster/pki.go b/internal/controller/postgrescluster/pki.go index 9db13a448..8b4cea484 100644 --- a/internal/controller/postgrescluster/pki.go +++ b/internal/controller/postgrescluster/pki.go @@ -352,6 +352,10 @@ func (r *Reconciler) reconcileCertManagerClusterCertificate( } func (r *Reconciler) isRootCACertManagerManaged(ctx context.Context, cluster *v1beta1.PostgresCluster) (bool, error) { + if cluster.Spec.CustomRootCATLSSecret != nil { + return false, nil + } + installed, err := r.isCertManagerInstalled(ctx, cluster.Namespace) if err != nil || !installed { return false, err diff --git a/internal/controller/postgrescluster/pki_test.go b/internal/controller/postgrescluster/pki_test.go index 2d3070e9c..dc055bd0c 100644 --- a/internal/controller/postgrescluster/pki_test.go +++ b/internal/controller/postgrescluster/pki_test.go @@ -519,6 +519,19 @@ func TestUpgradeCertManagerDoesNotTakeOverInternalPKI(t *testing.T) { assert.Assert(t, managed) }) + t.Run("isRootCACertManagerManaged returns false when CustomRootCATLSSecret is set", func(t *testing.T) { + clusterWithCustomRoot := testCluster() + clusterWithCustomRoot.Name = cluster.Name + clusterWithCustomRoot.Namespace = namespace + clusterWithCustomRoot.Spec.CustomRootCATLSSecret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "my-custom-root-ca"}, + } + + managed, err := reconcilerWithCertManager.isRootCACertManagerManaged(ctx, clusterWithCustomRoot) + assert.NilError(t, err) + assert.Assert(t, !managed) + }) + t.Run("isRootCACertManagerManaged returns false when cert-manager not installed", func(t *testing.T) { rNoCertManager := &Reconciler{ Client: tClient, From 90534f78bcabe4b8d9830582899ea57631ee43f3 Mon Sep 17 00:00:00 2001 From: George Kechagias Date: Mon, 11 May 2026 13:53:33 +0300 Subject: [PATCH 15/15] handle isRootCACertManagerManaged error --- internal/controller/postgrescluster/controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go index 246701529..e0518ce57 100644 --- a/internal/controller/postgrescluster/controller.go +++ b/internal/controller/postgrescluster/controller.go @@ -293,7 +293,11 @@ func (r *Reconciler) Reconcile( } if err == nil && rootCA != nil { - if certManagerManaged, _ := r.isRootCACertManagerManaged(ctx, cluster); certManagerManaged { + certManagerManaged, certErr := r.isRootCACertManagerManaged(ctx, cluster) + if certErr != nil { + log.V(1).Info("failed to check if root CA is cert-manager managed, will retry on next reconcile", + "error", certErr) + } else if certManagerManaged { r.registerCertManagerWatches(ctx) } }