fix: Certificate handling in case SecretClass has multiple CAs

sbernauer · sbernauer · commit 40862c0f3b4b · 2025-09-11T08:56:07.000+02:00
diff --git a/_WORK/Dockerfile b/_WORK/Dockerfile
@@ -0,0 +1,42 @@
+# FROM stackable/image/stackable-devel AS truststore-merger
+FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:383329bf9c4f968e87e85d30ba3a5cb988a3bbde28b8e4932dcd3a025fd9c98c AS truststore-merger
+
+
+RUN microdnf update -y
+RUN microdnf install -y gcc findutils git
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+ENV CARGO_CYCLONEDX_CRATE_VERSION=0.5.7
+ENV CARGO_AUDITABLE_CRATE_VERSION=0.6.6
+RUN . "$HOME/.cargo/env" && cargo --locked --quiet install cargo-cyclonedx@"$CARGO_CYCLONEDX_CRATE_VERSION" cargo-auditable@"$CARGO_AUDITABLE_CRATE_VERSION" && rustup toolchain install
+
+RUN microdnf install -y openssl-devel pkg-config
+
+
+
+
+
+
+
+
+
+
+
+
+
+# TRASH ABOVE!!!
+
+# docker build . -t oci.stackable.tech/sdp/trino:476-stackable0.0.0-dev-with-merger
+
+RUN echo "invalidate cache 2 :)"
+
+RUN <<EOF
+git clone --depth 1 --branch "feat/truststore-merger" https://github.com/stackabletech/secret-operator
+cd ./secret-operator
+. "$HOME/.cargo/env"
+cargo auditable --quiet build --release --package truststore-merger && cargo cyclonedx --all --spec-version 1.5 --describe binaries
+EOF
+
+FROM oci.stackable.tech/sdp/trino:476-stackable0.0.0-dev AS final
+
+COPY --from=truststore-merger --chown=stackable:0 /secret-operator/target/release/truststore-merger /stackable/truststore-merger
diff --git a/_WORK/README.md b/_WORK/README.md
@@ -0,0 +1,27 @@
+
+### Create problem
+
+```bash
+stackablectl op in commons listener secret trino
+kind load docker-image oci.stackable.tech/sdp/trino:476-stackable0.0.0-dev-with-merger
+kubectl apply -f _WORK/trino.yaml
+```
+The Trino Pod will not came up because of `backend failed to get secret data: failed to pick a CA: no CA in Secret.v1./secret-provisioner-short-tls-ca.stackable-operators will live until at least 2025-09-04 8:03:42.030007063 +00:00:00`.
+That's totally expected!
+
+Let's wait until the CA certificate Secrets have the desired amount of rotated certificates.
+
+Afterwards increase the ca cert lifetime to start the Pod:
+
+```bash
+kubectl patch secretclass short-tls --type=merge --patch '{"spec": {"backend": {"autoTls": {"ca": {"caCertificateLifetime": "365d"}}}}}'
+```
+
+Congrats, your secret-op now did a certificate rotation and your Pod should start up (after some time for retries)!
+
+### Debug commands
+
+```bash
+keytool -list -storepass "" -keystore /certs/pkcs12-1/truststore.p12
+openssl pkcs12 -password pass: -in /certs/pkcs12-1/truststore.p12 
+```
diff --git a/_WORK/trino.yaml b/_WORK/trino.yaml
@@ -0,0 +1,39 @@
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCluster
+metadata:
+  name: trino
+spec:
+  image:
+    productVersion: "476"
+    custom: oci.stackable.tech/sdp/trino:476-stackable0.0.0-dev-with-merger
+    pullPolicy: Never
+  clusterConfig:
+    tls:
+      serverSecretClass: short-tls
+    catalogLabelSelector:
+      matchLabels:
+        trino: simple-trino
+  coordinators:
+    roleConfig:
+      listenerClass: external-unstable
+    roleGroups:
+      default:
+        replicas: 1
+  workers:
+    roleGroups:
+      default:
+        replicas: 1
+---
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: SecretClass
+metadata:
+  name: short-tls
+spec:
+  backend: 
+    autoTls: 
+      ca:
+        secret:
+          name: secret-provisioner-short-tls-ca
+          namespace: stackable-operators
+        autoGenerate: true
+        caCertificateLifetime: 1m
diff --git a/rust/operator-binary/src/authentication/oidc/mod.rs b/rust/operator-binary/src/authentication/oidc/mod.rs
@@ -184,7 +184,7 @@ impl TrinoOidcAuthentication {
                 oauth2_authentication_config.add_commands(
                     TrinoRole::Coordinator,
                     crate::crd::Container::Prepare,
-                    command::add_cert_to_truststore(&path, STACKABLE_CLIENT_TLS_DIR, "oidc-idp"),
+                    command::add_cert_to_truststore(&path, STACKABLE_CLIENT_TLS_DIR),
                 );
             }
         }
diff --git a/rust/operator-binary/src/catalog/commons.rs b/rust/operator-binary/src/catalog/commons.rs
@@ -77,7 +77,7 @@ impl ExtendCatalogConfig for s3::v1alpha1::InlineConnectionOrReference {
     async fn extend_catalog_config(
         &self,
         catalog_config: &mut CatalogConfig,
-        catalog_name: &str,
+        _catalog_name: &str,
         catalog_namespace: Option<String>,
         client: &Client,
         trino_version: u16,
@@ -151,11 +151,7 @@ impl ExtendCatalogConfig for s3::v1alpha1::InlineConnectionOrReference {
                 }) => {
                     if let Some(ca_cert) = s3.tls.tls_ca_cert_mount_path() {
                         catalog_config.init_container_extra_start_commands.extend(
-                            command::add_cert_to_truststore(
-                                &ca_cert,
-                                STACKABLE_CLIENT_TLS_DIR,
-                                &format!("{catalog_name}-ca-cert"),
-                            ),
+                            command::add_cert_to_truststore(&ca_cert, STACKABLE_CLIENT_TLS_DIR),
                         );
                     }
                 }
diff --git a/rust/operator-binary/src/command.rs b/rust/operator-binary/src/command.rs
@@ -14,8 +14,7 @@ use crate::{
         CONFIG_DIR_NAME, Container, LOG_PROPERTIES, RW_CONFIG_DIR_NAME, STACKABLE_CLIENT_TLS_DIR,
         STACKABLE_INTERNAL_TLS_DIR, STACKABLE_MOUNT_INTERNAL_TLS_DIR,
         STACKABLE_MOUNT_SERVER_TLS_DIR, STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD,
-        SYSTEM_TRUST_STORE, SYSTEM_TRUST_STORE_PASSWORD, TrinoRole,
-        fault_tolerant_execution::ResolvedFaultTolerantExecutionConfig, v1alpha1,
+        TrinoRole, fault_tolerant_execution::ResolvedFaultTolerantExecutionConfig, v1alpha1,
     },
 };
 
@@ -43,38 +42,23 @@ pub fn container_prepare_args(
         ));
     }
 
+    // Create truststore that will be used when talking to external tools like S3
+    // It will be populated from the system truststore so that connections against public services like AWS S3 are still possible
+    args.push(format!("truststore-merger --pem /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem --out {STACKABLE_CLIENT_TLS_DIR}/truststore.p12 --out-password {STACKABLE_TLS_STORE_PASSWORD}"));
+
     if trino.tls_enabled() {
-        args.extend(import_truststore(
-            STACKABLE_MOUNT_SERVER_TLS_DIR,
-            STACKABLE_SERVER_TLS_DIR,
-        ));
-        args.extend(import_keystore(
-            STACKABLE_MOUNT_SERVER_TLS_DIR,
-            STACKABLE_SERVER_TLS_DIR,
-        ));
+        args.push(format!("cp {STACKABLE_MOUNT_SERVER_TLS_DIR}/truststore.p12 {STACKABLE_SERVER_TLS_DIR}/truststore.p12"));
+        args.push(format!("cp {STACKABLE_MOUNT_SERVER_TLS_DIR}/keystore.p12 {STACKABLE_SERVER_TLS_DIR}/keystore.p12"));
     }
 
     if trino.get_internal_tls().is_some() {
-        args.extend(import_truststore(
-            STACKABLE_MOUNT_INTERNAL_TLS_DIR,
-            STACKABLE_INTERNAL_TLS_DIR,
-        ));
-        args.extend(import_keystore(
-            STACKABLE_MOUNT_INTERNAL_TLS_DIR,
-            STACKABLE_INTERNAL_TLS_DIR,
-        ));
+        args.push(format!("cp {STACKABLE_MOUNT_INTERNAL_TLS_DIR}/truststore.p12 {STACKABLE_INTERNAL_TLS_DIR}/truststore.p12"));
+        args.push(format!("cp {STACKABLE_MOUNT_INTERNAL_TLS_DIR}/keystore.p12 {STACKABLE_INTERNAL_TLS_DIR}/keystore.p12"));
         if trino.tls_enabled() {
-            args.extend(import_truststore(
-                STACKABLE_MOUNT_SERVER_TLS_DIR,
-                STACKABLE_INTERNAL_TLS_DIR,
-            ))
+            args.push(format!("truststore-merger --pkcs12 {STACKABLE_MOUNT_SERVER_TLS_DIR}/truststore.p12:{STACKABLE_TLS_STORE_PASSWORD} --pkcs12 {STACKABLE_INTERNAL_TLS_DIR}/truststore.p12:{STACKABLE_TLS_STORE_PASSWORD} --out {STACKABLE_INTERNAL_TLS_DIR}/truststore.p12 --out-password {STACKABLE_TLS_STORE_PASSWORD}"));
         }
     }
 
-    // Create truststore that will be used when talking to external tools like S3
-    // It will be populated from the system truststore so that connections against public services like AWS S3 are still possible
-    args.extend(import_system_truststore(STACKABLE_CLIENT_TLS_DIR));
-
     // Add the commands that are needed to set up the catalogs
     catalogs.iter().for_each(|catalog| {
         args.extend_from_slice(&catalog.init_container_extra_start_commands);
@@ -148,77 +132,11 @@ wait_for_termination $!
     args
 }
 
-/// Adds a CA file from `cert_file` into a truststore named `truststore.p12` in `destination_directory`
-/// under the alias `alias_name`.
-pub fn add_cert_to_truststore(
-    cert_file: &str,
-    destination_directory: &str,
-    alias_name: &str,
-) -> Vec<String> {
-    vec![
-        format!(
-            "echo Adding cert from {cert_file} to truststore {destination_directory}/truststore.p12"
-        ),
-        format!(
-            "keytool -importcert -file {cert_file} -keystore {destination_directory}/truststore.p12 -storetype pkcs12 -noprompt -alias {alias_name} -storepass {STACKABLE_TLS_STORE_PASSWORD}"
-        ),
-    ]
-}
-
-/// Generates the shell script to import a secret operator provided keystore without password
-/// into a new keystore with password in a writeable empty dir
-///
-/// # Arguments
-/// - `source_directory`: The directory of the source keystore. Should usually be a secret operator volume mount.
-/// - `destination_directory`: The directory of the destination keystore. Should usually be an empty dir.
-fn import_keystore(source_directory: &str, destination_directory: &str) -> Vec<String> {
-    vec![
-        // The source directory is a secret-op mount and we do not want to write / add anything in there
-        // Therefore we import all the contents to a keystore in "writeable" empty dirs.
-        // Keytool is only barking if a password is not set for the destination keystore (which we set)
-        // and do provide an empty password for the source keystore coming from the secret-operator.
-        // Using no password will result in a warning.
-        format!(
-            "echo Importing {source_directory}/keystore.p12 to {destination_directory}/keystore.p12"
-        ),
-        format!(
-            "keytool -importkeystore -srckeystore {source_directory}/keystore.p12 -srcstoretype PKCS12 -srcstorepass \"\" -destkeystore {destination_directory}/keystore.p12 -deststoretype PKCS12 -deststorepass {STACKABLE_TLS_STORE_PASSWORD} -noprompt"
-        ),
-    ]
-}
-
-/// Generates the shell script to import a secret operator provided truststore without password
-/// into a new truststore with password in a writeable empty dir
-///
-/// # Arguments
-/// - `source_directory`: The directory of the source truststore. Should usually be a secret operator volume mount.
-/// - `destination_directory`: The directory of the destination truststore. Should usually be an empty dir.
-fn import_truststore(source_directory: &str, destination_directory: &str) -> Vec<String> {
-    vec![
-        // The source directory is a secret-op mount and we do not want to write / add anything in there
-        // Therefore we import all the contents to a truststore in "writeable" empty dirs.
-        // Keytool is only barking if a password is not set for the destination truststore (which we set)
-        // and do provide an empty password for the source truststore coming from the secret-operator.
-        // Using no password will result in a warning.
-        // All secret-op generated truststores have one entry with alias "1". We generate a UUID for
-        // the destination truststore to avoid conflicts when importing multiple secret-op generated
-        // truststores. We do not use the UUID rust crate since this will continuously change the STS... and
-        // leads to never-ending reconciles.
-        format!(
-            "echo Importing {source_directory}/truststore.p12 to {destination_directory}/truststore.p12"
-        ),
-        format!(
-            "keytool -importkeystore -srckeystore {source_directory}/truststore.p12 -srcstoretype PKCS12 -srcstorepass \"\" -srcalias 1 -destkeystore {destination_directory}/truststore.p12 -deststoretype PKCS12 -deststorepass {STACKABLE_TLS_STORE_PASSWORD} -destalias $(cat /proc/sys/kernel/random/uuid) -noprompt"
-        ),
-    ]
-}
-
-/// Import the system truststore to a truststore named `truststore.p12` in `destination_directory`.
-fn import_system_truststore(destination_directory: &str) -> Vec<String> {
-    vec![
-        format!("echo Importing {SYSTEM_TRUST_STORE} to {destination_directory}/truststore.p12"),
-        format!(
-            "keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE} -srcstoretype jks -srcstorepass {SYSTEM_TRUST_STORE_PASSWORD} -destkeystore {destination_directory}/truststore.p12 -deststoretype pkcs12 -deststorepass {STACKABLE_TLS_STORE_PASSWORD} -noprompt"
-        ),
-    ]
+/// Adds a PEM file to configured PKCS12 truststore (using the [`STACKABLE_TLS_STORE_PASSWORD`]
+/// password)
+pub fn add_cert_to_truststore(cert_file: &str, destination_directory: &str) -> Vec<String> {
+    let truststore = format!("{destination_directory}/truststore.p12");
+    vec![format!(
+        "truststore-merger --pkcs12 {truststore}:{STACKABLE_TLS_STORE_PASSWORD} --pem {cert_file} --out {truststore} --out-password {STACKABLE_TLS_STORE_PASSWORD}"
+    )]
 }
diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs
@@ -86,7 +86,7 @@ use crate::{
         MAX_TRINO_LOG_FILES_SIZE, METRICS_PORT, METRICS_PORT_NAME, NODE_PROPERTIES,
         RW_CONFIG_DIR_NAME, STACKABLE_CLIENT_TLS_DIR, STACKABLE_INTERNAL_TLS_DIR,
         STACKABLE_MOUNT_INTERNAL_TLS_DIR, STACKABLE_MOUNT_SERVER_TLS_DIR, STACKABLE_SERVER_TLS_DIR,
-        TrinoRole,
+        STACKABLE_TLS_STORE_PASSWORD, TrinoRole,
         authentication::resolve_authentication_classes,
         catalog,
         discovery::{TrinoDiscovery, TrinoDiscoveryProtocol, TrinoPodRef},
@@ -1576,6 +1576,7 @@ fn create_tls_volume(
     secret_volume_source_builder
         .with_pod_scope()
         .with_format(SecretFormat::TlsPkcs12)
+        .with_tls_pkcs12_password(STACKABLE_TLS_STORE_PASSWORD)
         .with_auto_tls_cert_lifetime(*requested_secret_lifetime);
 
     if let Some(listener_scope) = &listener_scope {
diff --git a/rust/operator-binary/src/crd/fault_tolerant_execution.rs b/rust/operator-binary/src/crd/fault_tolerant_execution.rs
@@ -545,11 +545,7 @@ impl ResolvedFaultTolerantExecutionConfig {
                 }) => {
                     if let Some(ca_cert) = s3_connection.tls.tls_ca_cert_mount_path() {
                         self.init_container_extra_start_commands.extend(
-                            command::add_cert_to_truststore(
-                                &ca_cert,
-                                STACKABLE_CLIENT_TLS_DIR,
-                                "exchange-s3-ca-cert",
-                            ),
+                            command::add_cert_to_truststore(&ca_cert, STACKABLE_CLIENT_TLS_DIR),
                         );
                     }
                 }
diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs
@@ -99,10 +99,8 @@ pub const STACKABLE_INTERNAL_TLS_DIR: &str = "/stackable/internal_tls";
 pub const STACKABLE_LOG_DIR: &str = "/stackable/log";
 pub const STACKABLE_MOUNT_SERVER_TLS_DIR: &str = "/stackable/mount_server_tls";
 pub const STACKABLE_MOUNT_INTERNAL_TLS_DIR: &str = "/stackable/mount_internal_tls";
-pub const SYSTEM_TRUST_STORE: &str = "/etc/pki/java/cacerts";
 // store pws
 pub const STACKABLE_TLS_STORE_PASSWORD: &str = "changeit";
-pub const SYSTEM_TRUST_STORE_PASSWORD: &str = "changeit";
 // secret vars
 pub const ENV_INTERNAL_SECRET: &str = "INTERNAL_SECRET";
 // TLS

Original file line number	Diff line number	Diff line change
`@@ -184,7 +184,7 @@ impl TrinoOidcAuthentication {`
`184`	`184`	`oauth2_authentication_config.add_commands(`
`185`	`185`	`TrinoRole::Coordinator,`
`186`	`186`	`crate::crd::Container::Prepare,`
`187`		`- command::add_cert_to_truststore(&path, STACKABLE_CLIENT_TLS_DIR, "oidc-idp"),`
	`187`	`+ command::add_cert_to_truststore(&path, STACKABLE_CLIENT_TLS_DIR),`
`188`	`188`	`);`
`189`	`189`	`}`
`190`	`190`	`}`
Original file line number	Diff line number	Diff line change
`@@ -545,11 +545,7 @@ impl ResolvedFaultTolerantExecutionConfig {`
`545`	`545`	`}) => {`
`546`	`546`	`if let Some(ca_cert) = s3_connection.tls.tls_ca_cert_mount_path() {`
`547`	`547`	`self.init_container_extra_start_commands.extend(`
`548`		`- command::add_cert_to_truststore(`
`549`		`- &ca_cert,`
`550`		`- STACKABLE_CLIENT_TLS_DIR,`
`551`		`- "exchange-s3-ca-cert",`
`552`		`- ),`
	`548`	`+ command::add_cert_to_truststore(&ca_cert, STACKABLE_CLIENT_TLS_DIR),`
`553`	`549`	`);`
`554`	`550`	`}`
`555`	`551`	`}`