Skip to content

Commit 1de0787

Browse files
sdminonneclaude
sdminonne
and
claude
committed
fix(cpo): propagate additionalTrustBundle to AWS control plane components
AWS control plane components fail TLS verification when calling AWS endpoints in isolated environments (e.g. US-ISO regions) because they do not honor the additionalTrustBundle from the HostedCluster spec. The AWS SDK replaces the system CA bundle when AWS_CA_BUNDLE is set, rather than appending to it (both v1 and v2 create a new empty x509.CertPool). To handle this, add a DeploymentAddAWSCABundleVolume helper in support/podspec that uses an init container (CPO image) to concatenate the system CA bundle with the user CAs from the additionalTrustBundle ConfigMap into a combined PEM file. AWS_CA_BUNDLE points to this combined file, ensuring the AWS SDK trusts both system and user CAs. The init container runs with a restricted security context (AllowPrivilegeEscalation=false, drop ALL capabilities) and minimal resource requests (cpu: 10m, memory: 10Mi), consistent with other lightweight init containers in the codebase. Wire the helper into all affected AWS components: - aws-cloud-controller-manager - capi-provider - ingress-operator - karpenter - karpenter-operator - aws-node-termination-handler Signed-off-by: Salvatore Dario Minonne <sminonne@redhat.com> Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent e09cc2d commit 1de0787

16 files changed

Lines changed: 1042 additions & 0 deletions

File tree

‎control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep
6262
}
6363
})
6464

65+
if hcp.Spec.AdditionalTrustBundle != nil {
66+
podspec.DeploymentAddAWSCABundleVolume(hcp.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName))
67+
}
68+
6569
// Set replicas based on whether termination handler is needed
6670
// If the disable annotation is present, scale to 0 replicas
6771
deployment.Spec.Replicas = ptr.To[int32](1)

‎control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler/deployment_test.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,20 @@ import (
99
assets "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/assets"
1010
controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component"
1111

12+
corev1 "k8s.io/api/core/v1"
1213
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1314
)
1415

16+
type fakeReleaseProvider struct{}
17+
18+
func (f *fakeReleaseProvider) GetImage(key string) string { return "test-cpo-image" }
19+
func (f *fakeReleaseProvider) ImageExist(key string) (string, bool) { return "", false }
20+
func (f *fakeReleaseProvider) Version() string { return "4.17.0" }
21+
func (f *fakeReleaseProvider) ComponentVersions() (map[string]string, error) {
22+
return nil, nil
23+
}
24+
func (f *fakeReleaseProvider) ComponentImages() map[string]string { return nil }
25+
1526
func TestAdaptDeployment(t *testing.T) {
1627
testCases := []struct {
1728
name string
@@ -181,3 +192,91 @@ func TestGetTerminationHandlerQueueURL(t *testing.T) {
181192
})
182193
}
183194
}
195+
196+
func TestAdaptDeploymentAWSCABundle(t *testing.T) {
197+
testCases := []struct {
198+
name string
199+
additionalTrust *corev1.LocalObjectReference
200+
expectCABundle bool
201+
}{
202+
{
203+
name: "When additional trust bundle is set it should add combined CA bundle with init container",
204+
additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"},
205+
expectCABundle: true,
206+
},
207+
{
208+
name: "When no additional trust bundle is set it should not add CA bundle resources",
209+
additionalTrust: nil,
210+
expectCABundle: false,
211+
},
212+
}
213+
214+
for _, tc := range testCases {
215+
t.Run(tc.name, func(t *testing.T) {
216+
g := NewGomegaWithT(t)
217+
218+
hcp := &hyperv1.HostedControlPlane{
219+
ObjectMeta: metav1.ObjectMeta{
220+
Name: "test-cluster",
221+
Namespace: "clusters-test-cluster",
222+
},
223+
Spec: hyperv1.HostedControlPlaneSpec{
224+
InfraID: "test-infra-id",
225+
Platform: hyperv1.PlatformSpec{
226+
Type: hyperv1.AWSPlatform,
227+
AWS: &hyperv1.AWSPlatformSpec{
228+
Region: "us-east-1",
229+
},
230+
},
231+
AdditionalTrustBundle: tc.additionalTrust,
232+
},
233+
}
234+
235+
cpContext := controlplanecomponent.WorkloadContext{
236+
Context: t.Context(),
237+
HCP: hcp,
238+
ReleaseImageProvider: &fakeReleaseProvider{},
239+
}
240+
241+
deployment, err := assets.LoadDeploymentManifest(ComponentName)
242+
g.Expect(err).ToNot(HaveOccurred())
243+
244+
err = adaptDeployment(cpContext, deployment)
245+
g.Expect(err).ToNot(HaveOccurred())
246+
247+
volumes := deployment.Spec.Template.Spec.Volumes
248+
initContainers := deployment.Spec.Template.Spec.InitContainers
249+
container := deployment.Spec.Template.Spec.Containers[0]
250+
251+
if tc.expectCABundle {
252+
g.Expect(volumes).To(ContainElement(SatisfyAll(
253+
HaveField("Name", "user-ca-bundle"),
254+
HaveField("VolumeSource.ConfigMap.Name", "user-ca-bundle"),
255+
)))
256+
g.Expect(volumes).To(ContainElement(SatisfyAll(
257+
HaveField("Name", "aws-ca-bundle"),
258+
HaveField("VolumeSource.EmptyDir", Not(BeNil())),
259+
)))
260+
g.Expect(initContainers).To(ContainElement(SatisfyAll(
261+
HaveField("Name", "setup-aws-ca-bundle"),
262+
HaveField("Image", "test-cpo-image"),
263+
)))
264+
g.Expect(container.VolumeMounts).To(ContainElement(SatisfyAll(
265+
HaveField("Name", "aws-ca-bundle"),
266+
HaveField("MountPath", "/etc/pki/ca-trust/extracted/hypershift"),
267+
HaveField("ReadOnly", true),
268+
)))
269+
g.Expect(container.Env).To(ContainElement(SatisfyAll(
270+
HaveField("Name", "AWS_CA_BUNDLE"),
271+
HaveField("Value", "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem"),
272+
)))
273+
} else {
274+
g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "user-ca-bundle")))
275+
g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle")))
276+
g.Expect(initContainers).ToNot(ContainElement(HaveField("Name", "setup-aws-ca-bundle")))
277+
g.Expect(container.VolumeMounts).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle")))
278+
g.Expect(container.Env).ToNot(ContainElement(HaveField("Name", "AWS_CA_BUNDLE")))
279+
}
280+
})
281+
}
282+
}

‎control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package capiprovider
22

33
import (
4+
hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
45
component "github.com/openshift/hypershift/support/controlplane-component"
56
"github.com/openshift/hypershift/support/k8sutil"
7+
"github.com/openshift/hypershift/support/podspec"
68
"github.com/openshift/hypershift/support/proxy"
79

810
appsv1 "k8s.io/api/apps/v1"
@@ -20,6 +22,10 @@ func (capi *CAPIProviderOptions) adaptDeployment(cpContext component.WorkloadCon
2022

2123
proxy.SetEnvVars(&deployment.Spec.Template.Spec.Containers[0].Env)
2224

25+
if cpContext.HCP.Spec.Platform.Type == hyperv1.AWSPlatform && cpContext.HCP.Spec.AdditionalTrustBundle != nil {
26+
podspec.DeploymentAddAWSCABundleVolume(cpContext.HCP.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName))
27+
}
28+
2329
if deployment.Annotations == nil {
2430
deployment.Annotations = make(map[string]string)
2531
}

‎control-plane-operator/controllers/hostedcontrolplane/v2/capi_provider/deployment_test.go‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,3 +231,130 @@ func TestAdaptDeployment_WithNilAnnotations(t *testing.T) {
231231
g.Expect(deployment.Annotations).ToNot(BeNil())
232232
g.Expect(deployment.Annotations[k8sutil.HostedClusterAnnotation]).To(Equal("test-namespace/test-cluster"))
233233
}
234+
235+
type fakeReleaseProvider struct{}
236+
237+
func (f *fakeReleaseProvider) GetImage(key string) string { return "test-cpo-image" }
238+
func (f *fakeReleaseProvider) ImageExist(key string) (string, bool) { return "", false }
239+
func (f *fakeReleaseProvider) Version() string { return "4.17.0" }
240+
func (f *fakeReleaseProvider) ComponentVersions() (map[string]string, error) {
241+
return nil, nil
242+
}
243+
func (f *fakeReleaseProvider) ComponentImages() map[string]string { return nil }
244+
245+
func TestAdaptDeploymentAWSCABundle(t *testing.T) {
246+
testCases := []struct {
247+
name string
248+
platformType hyperv1.PlatformType
249+
additionalTrust *corev1.LocalObjectReference
250+
expectCABundle bool
251+
}{
252+
{
253+
name: "When AWS platform with additional trust bundle it should add combined CA bundle",
254+
platformType: hyperv1.AWSPlatform,
255+
additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"},
256+
expectCABundle: true,
257+
},
258+
{
259+
name: "When AWS platform without additional trust bundle it should not add CA bundle",
260+
platformType: hyperv1.AWSPlatform,
261+
additionalTrust: nil,
262+
expectCABundle: false,
263+
},
264+
{
265+
name: "When non-AWS platform with additional trust bundle it should not add CA bundle",
266+
platformType: hyperv1.KubevirtPlatform,
267+
additionalTrust: &corev1.LocalObjectReference{Name: "user-ca-bundle"},
268+
expectCABundle: false,
269+
},
270+
}
271+
272+
for _, tc := range testCases {
273+
t.Run(tc.name, func(t *testing.T) {
274+
g := NewWithT(t)
275+
276+
hcp := &hyperv1.HostedControlPlane{
277+
ObjectMeta: metav1.ObjectMeta{
278+
Name: "test-cluster",
279+
Namespace: "clusters-test-cluster",
280+
Annotations: map[string]string{
281+
"hypershift.openshift.io/cluster": "clusters/test-cluster",
282+
},
283+
},
284+
Spec: hyperv1.HostedControlPlaneSpec{
285+
Platform: hyperv1.PlatformSpec{
286+
Type: tc.platformType,
287+
},
288+
AdditionalTrustBundle: tc.additionalTrust,
289+
},
290+
}
291+
292+
deploymentSpec := &appsv1.DeploymentSpec{
293+
Template: corev1.PodTemplateSpec{
294+
Spec: corev1.PodSpec{
295+
Containers: []corev1.Container{
296+
{
297+
Name: "manager",
298+
Image: "test-image",
299+
},
300+
},
301+
},
302+
},
303+
}
304+
305+
capi := &CAPIProviderOptions{
306+
deploymentSpec: deploymentSpec,
307+
}
308+
309+
cpContext := component.WorkloadContext{
310+
Context: t.Context(),
311+
HCP: hcp,
312+
ReleaseImageProvider: &fakeReleaseProvider{},
313+
}
314+
315+
deployment := &appsv1.Deployment{
316+
ObjectMeta: metav1.ObjectMeta{
317+
Name: ComponentName,
318+
Namespace: hcp.Namespace,
319+
},
320+
}
321+
322+
err := capi.adaptDeployment(cpContext, deployment)
323+
g.Expect(err).ToNot(HaveOccurred())
324+
325+
volumes := deployment.Spec.Template.Spec.Volumes
326+
initContainers := deployment.Spec.Template.Spec.InitContainers
327+
container := deployment.Spec.Template.Spec.Containers[0]
328+
329+
if tc.expectCABundle {
330+
g.Expect(volumes).To(ContainElement(SatisfyAll(
331+
HaveField("Name", "user-ca-bundle"),
332+
HaveField("VolumeSource.ConfigMap.Name", "user-ca-bundle"),
333+
)))
334+
g.Expect(volumes).To(ContainElement(SatisfyAll(
335+
HaveField("Name", "aws-ca-bundle"),
336+
HaveField("VolumeSource.EmptyDir", Not(BeNil())),
337+
)))
338+
g.Expect(initContainers).To(ContainElement(SatisfyAll(
339+
HaveField("Name", "setup-aws-ca-bundle"),
340+
HaveField("Image", "test-cpo-image"),
341+
)))
342+
g.Expect(container.VolumeMounts).To(ContainElement(SatisfyAll(
343+
HaveField("Name", "aws-ca-bundle"),
344+
HaveField("MountPath", "/etc/pki/ca-trust/extracted/hypershift"),
345+
HaveField("ReadOnly", true),
346+
)))
347+
g.Expect(container.Env).To(ContainElement(SatisfyAll(
348+
HaveField("Name", "AWS_CA_BUNDLE"),
349+
HaveField("Value", "/etc/pki/ca-trust/extracted/hypershift/combined-ca-bundle.pem"),
350+
)))
351+
} else {
352+
g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "user-ca-bundle")))
353+
g.Expect(volumes).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle")))
354+
g.Expect(initContainers).ToNot(ContainElement(HaveField("Name", "setup-aws-ca-bundle")))
355+
g.Expect(container.VolumeMounts).ToNot(ContainElement(HaveField("Name", "aws-ca-bundle")))
356+
g.Expect(container.Env).ToNot(ContainElement(HaveField("Name", "AWS_CA_BUNDLE")))
357+
}
358+
})
359+
}
360+
}

‎control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/component.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ func (c *awsOptions) NeedsManagementKASAccess() bool {
3131

3232
func NewComponent() component.ControlPlaneComponent {
3333
return component.NewDeploymentComponent(ComponentName, &awsOptions{}).
34+
WithAdaptFunction(adaptDeployment).
3435
WithPredicate(predicate).
3536
WithManifestAdapter(
3637
"config.yaml",

‎control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws/deployment.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package aws
2+
3+
import (
4+
component "github.com/openshift/hypershift/support/controlplane-component"
5+
"github.com/openshift/hypershift/support/podspec"
6+
7+
appsv1 "k8s.io/api/apps/v1"
8+
)
9+
10+
func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Deployment) error {
11+
if cpContext.HCP.Spec.AdditionalTrustBundle != nil {
12+
podspec.DeploymentAddAWSCABundleVolume(cpContext.HCP.Spec.AdditionalTrustBundle, deployment, cpContext.ReleaseImageProvider.GetImage(podspec.CPOImageName))
13+
}
14+
return nil
15+
}

0 commit comments

Comments
 (0)