From 836b356fe0ac642cefebcc7dd333685c2ae66eff Mon Sep 17 00:00:00 2001 From: Luke Bond Date: Thu, 26 Mar 2026 18:21:25 +0000 Subject: [PATCH] feat: support trusted CA certificates in RestateCluster - add `trustedCaCerts` field to `spec.security` referencing Secrets containing PEM-encoded CA certs - add init container that concatenates system CA bundle with custom certs into a single file - set `SSL_CERT_FILE` on the restate container to point to the combined bundle - use the canary image (not the restate image) for the init container - hash secret references as a pod annotation to trigger rollout on config change - update CRD, pkl schema, README, and release notes --- AGENTS.md | 4 + CLAUDE.md | 2 +- README.md | 30 ++- charts/restate-operator-helm/values.yaml | 2 +- crd/RestateCluster.pkl | 13 ++ crd/restateclusters.yaml | 18 ++ release-notes/unreleased/trusted-ca-certs.md | 15 ++ .../restatecluster/reconcilers/compute.rs | 213 +++++++++++++++++- src/resources/restateclusters.rs | 12 + 9 files changed, 298 insertions(+), 11 deletions(-) create mode 100644 release-notes/unreleased/trusted-ca-certs.md diff --git a/AGENTS.md b/AGENTS.md index b3a146c..3cd5e28 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,3 +5,7 @@ Instructions for AI coding agents working on this repository. ## Release Notes When making changes, check whether the change warrants a release note by reviewing the guidelines in `release-notes/README.md`. If it does, create a release note file in `release-notes/unreleased/` as part of the same change. + +## Trusted CA Certs Init Container + +The trusted CA certs feature (`spec.security.trustedCaCerts`) uses an init container that reads the system CA bundle from `/etc/ssl/certs/ca-certificates.crt` (Debian/Alpine path). If the Restate server base image is changed to a different distro (e.g. RHEL uses `/etc/pki/tls/certs/ca-bundle.crt`), the `SYSTEM_CA_BUNDLE` constant in `src/controllers/restatecluster/reconcilers/compute.rs` must be updated. diff --git a/CLAUDE.md b/CLAUDE.md index cb95829..522de6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -261,7 +261,7 @@ kubectl delete pod -n restate-operator -l app=restate-operator - `image.pullPolicy` - Pull policy (default: `IfNotPresent`) - `awsPodIdentityAssociationCluster` - Enables EKS Pod Identity support - `gcpWorkloadIdentity` - Enables GCP Workload Identity via Config Connector -- `canaryImage` - Container image for canary jobs (default: `busybox:uclibc`); must provide `grep` and `wget` +- `canaryImage` - Container image for canary jobs and the trusted CA certs init container (default: `busybox:uclibc`); must provide `cat`, `grep` and `wget` - `operatorNamespace` - Namespace where operator runs - `operatorLabelName/Value` - Labels for network policy selectors diff --git a/README.md b/README.md index 555209c..97e3778 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,7 @@ This feature is particularly useful for Raft-based metadata clusters where manua | `awsPodIdentityAssociationRoleArn` | `string` | **Use this to grant your Restate cluster fine-grained access to other AWS resources (like S3) without managing static credentials.** Creates a `PodIdentityAssociation` to grant the cluster an IAM role. Requires the ACK EKS controller. | | `awsPodSecurityGroups` | `array` | **Use this to isolate your Restate cluster within specific AWS Security Groups for enhanced network control and auditing.** Creates a `SecurityGroupPolicy` to place pods into these security groups. Requires the Security Groups for Pods CRD. | | `requestSigningPrivateKey` | `object` | Configures a private key to sign outbound requests from this cluster. Can be sourced from a `secret` or a CSI `secretProvider`. See details below. | +| `trustedCaCerts` | `array` | Optional list of Secrets containing trusted CA certificates. Each cert is appended to the system CA bundle via an init container. See details below. | --- @@ -193,6 +194,30 @@ This feature is particularly useful for Raft-based metadata clusters where manua --- +#### `spec.security.trustedCaCerts` + +Use this to trust custom CA certificates (e.g. for calling SDK services behind an internal load balancer with a private certificate, or for object store access via a private CA) without building a custom Restate image. +The operator adds an init container that concatenates the system CA bundle with your custom certificates, and sets `SSL_CERT_FILE` to point to the combined bundle. + +Each entry references a Kubernetes Secret: + +| Field | Type | Description | +|---|---|---| +| `secretName` | `string` | **Required**. Name of the Secret containing the CA certificate. | +| `key` | `string` | **Required**. Key within the Secret that contains the PEM-encoded certificate. | + +**Example:** + +```yaml +spec: + security: + trustedCaCerts: + - secretName: internal-ca + key: ca.pem +``` + +--- + #### `spec.config` This field allows you to provide a TOML-encoded configuration string for the Restate server. This maps directly to the Restate server's configuration file. You can use this to configure aspects like roles, metadata storage, snapshotting, and more. @@ -645,7 +670,8 @@ the `RestateCluster` spec. ### Canary Image Both EKS Pod Identity and GCP Workload Identity use a canary job to validate that credentials are available before -starting the Restate cluster. By default, this uses the `busybox:uclibc` image from Docker Hub. In environments where +starting the Restate cluster. The trusted CA certs feature also uses this image for its init container. +By default, this uses the `busybox:uclibc` image from Docker Hub. In environments where nodes cannot pull from Docker Hub (e.g. air-gapped or restricted registries), you can override this with the `canaryImage` Helm value: @@ -661,7 +687,7 @@ docker tag busybox:uclibc my-private-registry.example.com/busybox:uclibc docker push my-private-registry.example.com/busybox:uclibc ``` -If using a different base image, it must provide `grep` and `wget`. +If using a different base image, it must provide `cat`, `grep` and `wget`. ### EKS Security Groups for Pods diff --git a/charts/restate-operator-helm/values.yaml b/charts/restate-operator-helm/values.yaml index 7123deb..e3185cc 100644 --- a/charts/restate-operator-helm/values.yaml +++ b/charts/restate-operator-helm/values.yaml @@ -16,7 +16,7 @@ podAnnotations: {} awsPodIdentityAssociationCluster: null gcpWorkloadIdentity: null clusterDns: null # defaults to "cluster.local" in the operator binary -canaryImage: null # defaults to "busybox:uclibc"; image must provide grep and wget +canaryImage: null # defaults to "busybox:uclibc"; image must provide cat, grep and wget podSecurityContext: fsGroup: 2000 diff --git a/crd/RestateCluster.pkl b/crd/RestateCluster.pkl index 03083ed..a96138f 100644 --- a/crd/RestateCluster.pkl +++ b/crd/RestateCluster.pkl @@ -150,6 +150,10 @@ class Security { /// Annotations to set on the ServiceAccount created for Restate serviceAccountAnnotations: Mapping? + /// Optional list of Secrets containing trusted CA certificates. + /// Each cert is appended to the system CA bundle via an init container. + trustedCaCerts: Listing? + /// Annotations to set on the Service created for Restate serviceAnnotations: Mapping? } @@ -202,6 +206,15 @@ class SecretProvider { provider: String? } +/// A reference to a Secret containing a trusted CA certificate. +class TrustedCACert { + /// Name of the Secret containing the CA certificate + secretName: String + + /// Key within the Secret that contains the PEM-encoded certificate + key: String +} + /// Storage configuration class Storage { /// storageClassName is the name of the StorageClass required by the claim. More info: diff --git a/crd/restateclusters.yaml b/crd/restateclusters.yaml index 61ea069..e1ad001 100644 --- a/crd/restateclusters.yaml +++ b/crd/restateclusters.yaml @@ -1470,6 +1470,24 @@ spec: description: Annotations to set on the Service created for Restate nullable: true type: object + trustedCaCerts: + description: |- + Optional list of Secrets containing trusted CA certificates. + Each cert is appended to the system CA bundle via an init container. + items: + properties: + key: + description: Key within the Secret that contains the PEM-encoded certificate + type: string + secretName: + description: Name of the Secret containing the CA certificate + type: string + required: + - key + - secretName + type: object + nullable: true + type: array type: object storage: description: Storage configuration diff --git a/release-notes/unreleased/trusted-ca-certs.md b/release-notes/unreleased/trusted-ca-certs.md new file mode 100644 index 0000000..5b30ddf --- /dev/null +++ b/release-notes/unreleased/trusted-ca-certs.md @@ -0,0 +1,15 @@ +## Trusted CA Certificates + +You can now configure custom trusted CA certificates for RestateCluster via `spec.security.trustedCaCerts`. +This is useful when Restate needs to trust internal CAs, for example when accessing an object store with a private certificate authority. + +The operator adds an init container that concatenates the system CA bundle with your custom certificates into a single PEM file, +and sets `SSL_CERT_FILE` on the Restate container to point to the combined bundle. Changing the Secret references (name or key) triggers a pod rollout. + +```yaml +spec: + security: + trustedCaCerts: + - secretName: internal-ca + key: ca.pem +``` diff --git a/src/controllers/restatecluster/reconcilers/compute.rs b/src/controllers/restatecluster/reconcilers/compute.rs index 78137ff..211b526 100644 --- a/src/controllers/restatecluster/reconcilers/compute.rs +++ b/src/controllers/restatecluster/reconcilers/compute.rs @@ -6,11 +6,11 @@ use std::time::Duration; use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetStatus}; use k8s_openapi::api::batch::v1::{Job, JobSpec}; use k8s_openapi::api::core::v1::{ - ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EnvVar, EnvVarSource, - HTTPGetAction, ObjectFieldSelector, PersistentVolumeClaim, PersistentVolumeClaimSpec, Pod, - PodSecurityContext, PodSpec, PodTemplateSpec, Probe, SeccompProfile, SecurityContext, Service, - ServiceAccount, ServicePort, ServiceSpec, Toleration, Volume, VolumeMount, - VolumeResourceRequirements, + ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EmptyDirVolumeSource, EnvVar, + EnvVarSource, HTTPGetAction, KeyToPath, ObjectFieldSelector, PersistentVolumeClaim, + PersistentVolumeClaimSpec, Pod, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, + SeccompProfile, SecretVolumeSource, SecurityContext, Service, ServiceAccount, ServicePort, + ServiceSpec, Toleration, Volume, VolumeMount, VolumeResourceRequirements, }; use k8s_openapi::api::policy::v1::{PodDisruptionBudget, PodDisruptionBudgetSpec}; use k8s_openapi::apimachinery::pkg::api::resource::Quantity; @@ -35,7 +35,7 @@ use crate::resources::podidentityassociations::{ PodIdentityAssociation, PodIdentityAssociationSpec, }; use crate::resources::restateclusters::{ - RestateClusterSpec, RestateClusterStatus, RestateClusterStorage, + RestateClusterSpec, RestateClusterStatus, RestateClusterStorage, TrustedCACert, }; use crate::resources::securitygrouppolicies::{ SecurityGroupPolicy, SecurityGroupPolicySecurityGroups, SecurityGroupPolicySpec, @@ -290,6 +290,93 @@ fn env(cluster_name: &str, custom: Option<&[EnvVar]>) -> Vec { } } +// Debian/Alpine system CA bundle path. If the Restate server base image changes to a +// different distro (e.g. RHEL uses /etc/pki/tls/certs/ca-bundle.crt), this must be updated. +const SYSTEM_CA_BUNDLE: &str = "/etc/ssl/certs/ca-certificates.crt"; +const COMBINED_CA_VOLUME: &str = "combined-ca-certs"; +const COMBINED_CA_MOUNT: &str = "/combined-certs"; +const COMBINED_CA_BUNDLE: &str = "/combined-certs/ca-certificates.crt"; + +/// Build volumes and an init container that concatenates system CA certs with custom trusted CAs. +/// Returns (additional_volumes, init_container). +fn trusted_ca_init_container( + certs: &[TrustedCACert], + canary_image: &str, +) -> (Vec, Container) { + let mut ca_volumes = Vec::with_capacity(certs.len() + 1); + let mut init_volume_mounts = Vec::with_capacity(certs.len() + 1); + let mut cat_sources = vec![SYSTEM_CA_BUNDLE.to_string()]; + + for (i, cert) in certs.iter().enumerate() { + let vol_name = format!("trusted-ca-{i}"); + let mount_path = format!("/trusted-ca-{i}"); + + ca_volumes.push(Volume { + name: vol_name.clone(), + secret: Some(SecretVolumeSource { + secret_name: Some(cert.secret_name.clone()), + items: Some(vec![KeyToPath { + key: cert.key.clone(), + path: "ca.pem".into(), + mode: Some(0o444), + }]), + ..Default::default() + }), + ..Default::default() + }); + + init_volume_mounts.push(VolumeMount { + name: vol_name, + mount_path: mount_path.clone(), + read_only: Some(true), + ..Default::default() + }); + + cat_sources.push(format!("{mount_path}/ca.pem")); + } + + // emptyDir for the combined bundle + ca_volumes.push(Volume { + name: COMBINED_CA_VOLUME.into(), + empty_dir: Some(EmptyDirVolumeSource::default()), + ..Default::default() + }); + + init_volume_mounts.push(VolumeMount { + name: COMBINED_CA_VOLUME.into(), + mount_path: COMBINED_CA_MOUNT.into(), + ..Default::default() + }); + + let cat_command = format!("cat {} > {COMBINED_CA_BUNDLE}", cat_sources.join(" ")); + + let init_container = Container { + name: "combine-ca-certs".into(), + image: Some(canary_image.into()), + command: Some(vec!["sh".into(), "-c".into(), cat_command]), + volume_mounts: Some(init_volume_mounts), + security_context: Some(SecurityContext { + allow_privilege_escalation: Some(false), + ..Default::default() + }), + ..Default::default() + }; + + (ca_volumes, init_container) +} + +/// Compute a hash of trusted CA cert references for use as a pod annotation to trigger rollouts. +fn hash_trusted_ca_cert_refs(certs: &[TrustedCACert]) -> String { + let mut hasher = sha2::Sha256::new(); + for cert in certs { + hasher.update(cert.secret_name.as_bytes()); + hasher.update(b":"); + hasher.update(cert.key.as_bytes()); + hasher.update(b","); + } + format!("{:x}", hasher.finalize()) +} + const RESTATE_STATEFULSET_NAME: &str = "restate"; fn restate_statefulset( @@ -298,6 +385,7 @@ fn restate_statefulset( pod_annotations: Option>, signing_key: Option<(Volume, PathBuf)>, cm_name: String, + canary_image: &str, ) -> StatefulSet { let metadata = object_meta(base_metadata, RESTATE_STATEFULSET_NAME); let labels = metadata.labels.clone(); @@ -371,6 +459,36 @@ fn restate_statefulset( }) } + let trusted_ca_certs = spec + .security + .as_ref() + .and_then(|s| s.trusted_ca_certs.as_deref()) + .unwrap_or_default(); + + let init_containers = if !trusted_ca_certs.is_empty() { + let (ca_volumes, init_container) = + trusted_ca_init_container(trusted_ca_certs, canary_image); + volumes.extend(ca_volumes); + + // Mount combined bundle in main container + volume_mounts.push(VolumeMount { + name: COMBINED_CA_VOLUME.into(), + mount_path: COMBINED_CA_MOUNT.into(), + read_only: Some(true), + ..Default::default() + }); + + env.push(EnvVar { + name: "SSL_CERT_FILE".into(), + value: Some(COMBINED_CA_BUNDLE.into()), + value_from: None, + }); + + Some(vec![init_container]) + } else { + None + }; + StatefulSet { metadata, spec: Some(StatefulSetSpec { @@ -389,6 +507,7 @@ fn restate_statefulset( dns_policy: spec.compute.dns_policy.clone(), dns_config: spec.compute.dns_config.clone(), image_pull_secrets: spec.compute.image_pull_secrets.clone(), + init_containers, containers: vec![Container { name: "restate".into(), image: Some(spec.compute.image.clone()), @@ -695,6 +814,20 @@ pub async fn reconcile_compute( (false, None) => {} } + // Add pod annotation for trusted CA certs to trigger rollout on change + if let Some(certs) = spec + .security + .as_ref() + .and_then(|s| s.trusted_ca_certs.as_deref()) + .filter(|c| !c.is_empty()) + { + let pod_annotations = pod_annotations.get_or_insert_with(Default::default); + pod_annotations.insert( + "restate.dev/trusted-ca-certs".into(), + hash_trusted_ca_cert_refs(certs), + ); + } + let restate_service = restate_service( base_metadata, spec.security @@ -723,7 +856,14 @@ pub async fn reconcile_compute( let ss = apply_stateful_set( name, &ss_api, - restate_statefulset(base_metadata, spec, pod_annotations, signing_key, cm_name), + restate_statefulset( + base_metadata, + spec, + pod_annotations, + signing_key, + cm_name, + &ctx.canary_image, + ), ) .await?; @@ -1758,4 +1898,63 @@ mod tests { let pod_spec = job.spec.unwrap().template.spec.unwrap(); assert_eq!(pod_spec.tolerations.unwrap().len(), 1); } + + #[test] + fn test_trusted_ca_init_container_single_cert() { + let certs = vec![TrustedCACert { + secret_name: "my-ca".into(), + key: "ca.pem".into(), + }]; + let (volumes, init) = trusted_ca_init_container(&certs, "busybox:uclibc"); + + // 1 secret volume + 1 emptyDir + assert_eq!(volumes.len(), 2); + assert_eq!(volumes[0].name, "trusted-ca-0"); + assert!(volumes[0].secret.is_some()); + assert_eq!(volumes[1].name, COMBINED_CA_VOLUME); + assert!(volumes[1].empty_dir.is_some()); + + // init container uses canary image, not restate image + assert_eq!(init.image.as_deref(), Some("busybox:uclibc")); + + // init container cat command includes system certs and custom cert + let cmd = &init.command.as_ref().unwrap()[2]; + assert!(cmd.starts_with(&format!("cat {SYSTEM_CA_BUNDLE} /trusted-ca-0/ca.pem >"))); + + // volume mount names match volume names + let mounts = init.volume_mounts.as_ref().unwrap(); + let mount_names: Vec<&str> = mounts.iter().map(|m| m.name.as_str()).collect(); + assert_eq!(mount_names, vec!["trusted-ca-0", COMBINED_CA_VOLUME]); + } + + #[test] + fn test_trusted_ca_init_container_multiple_certs() { + let certs = vec![ + TrustedCACert { + secret_name: "ca-one".into(), + key: "cert.pem".into(), + }, + TrustedCACert { + secret_name: "ca-two".into(), + key: "root.pem".into(), + }, + ]; + let (volumes, init) = trusted_ca_init_container(&certs, "busybox:uclibc"); + + // 2 secret volumes + 1 emptyDir + assert_eq!(volumes.len(), 3); + + let cmd = &init.command.as_ref().unwrap()[2]; + assert!(cmd.contains("/trusted-ca-0/ca.pem")); + assert!(cmd.contains("/trusted-ca-1/ca.pem")); + + // secret volume references correct secret names and keys + let secret0 = volumes[0].secret.as_ref().unwrap(); + assert_eq!(secret0.secret_name.as_deref(), Some("ca-one")); + assert_eq!(secret0.items.as_ref().unwrap()[0].key, "cert.pem"); + + let secret1 = volumes[1].secret.as_ref().unwrap(); + assert_eq!(secret1.secret_name.as_deref(), Some("ca-two")); + assert_eq!(secret1.items.as_ref().unwrap()[0].key, "root.pem"); + } } diff --git a/src/resources/restateclusters.rs b/src/resources/restateclusters.rs index aad5481..24f2474 100644 --- a/src/resources/restateclusters.rs +++ b/src/resources/restateclusters.rs @@ -203,6 +203,18 @@ pub struct RestateClusterSecurity { pub network_egress_rules: Option>, /// If set, configure the use of a private key to sign outbound requests from this cluster pub request_signing_private_key: Option, + /// Optional list of Secrets containing trusted CA certificates. + /// Each cert is appended to the system CA bundle via an init container. + pub trusted_ca_certs: Option>, +} + +#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct TrustedCACert { + /// Name of the Secret containing the CA certificate + pub secret_name: String, + /// Key within the Secret that contains the PEM-encoded certificate + pub key: String, } #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)]