Skip to content

Commit ad1461f

Browse files
committed
OCPBUGS-86299: skip upstream LB tests on dual-stack and patch service IPFamilyPolicy
Upstream cloud-provider-aws load balancer tests do not support dual-stack clusters yet. Detect dual-stack configuration from the cloud-config (ipFamilies key) at startup and exclude upstream LB tests when the cluster is dual-stack. When detection fails, upstream LB tests are also excluded to avoid false positives (fail-closed). For the downstream AWSServiceLBNetworkSecurityGroup tests, patch the NLB service with IPFamilyPolicy=RequireDualStack when dual-stack is detected so the service matches the cluster's network configuration.
1 parent 7f6aa93 commit ad1461f

4 files changed

Lines changed: 191 additions & 31 deletions

File tree

openshift-tests/ccm-aws-tests/e2e/aws/loadbalancer.go

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
1010
. "github.com/onsi/ginkgo/v2"
1111
. "github.com/onsi/gomega"
12+
"github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/common"
1213
v1 "k8s.io/api/core/v1"
1314
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1415
"k8s.io/apimachinery/pkg/util/intstr"
@@ -30,10 +31,6 @@ const (
3031
featureGateAWSServiceLBNetworkSecurityGroup = "AWSServiceLBNetworkSecurityGroup"
3132

3233
annotationLBType = "service.beta.kubernetes.io/aws-load-balancer-type"
33-
34-
cloudConfigNamespace = "openshift-cloud-controller-manager"
35-
cloudConfigName = "cloud-conf"
36-
cloudConfigKey = "cloud.conf"
3734
)
3835

3936
// TestAWSServiceLBNetworkSecurityGroup validates the AWSServiceLBNetworkSecurityGroup feature gate functionality.
@@ -82,19 +79,13 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
8279
isNLBFeatureEnabled(ctx)
8380

8481
By("getting cloud-config ConfigMap from openshift-cloud-controller-manager namespace")
85-
cm, err := cs.CoreV1().ConfigMaps(cloudConfigNamespace).Get(ctx, cloudConfigName, metav1.GetOptions{})
82+
cm, err := common.GetCloudConfig(ctx, cs)
8683
framework.ExpectNoError(err, "failed to get cloud-config ConfigMap")
8784

88-
By("checking if cloud.conf key exists in ConfigMap")
89-
cloudConf, exists := cm.Data[cloudConfigKey]
90-
Expect(exists).To(BeTrue(), "cloud.conf key not found in ConfigMap")
91-
92-
By("verifying NLBSecurityGroupMode is present in cloud config")
93-
Expect(cloudConf).To(ContainSubstring("NLBSecurityGroupMode"),
94-
"NLBSecurityGroupMode must be present in cloud-config when feature gate is enabled")
95-
9685
By("verifying NLBSecurityGroupMode is set to Managed")
97-
Expect(cloudConf).To(MatchRegexp(`NLBSecurityGroupMode\s*=\s*Managed`),
86+
managed, err := common.IsNLBSecurityGroupModeManaged(cm)
87+
framework.ExpectNoError(err, "failed to check NLBSecurityGroupMode in cloud-config")
88+
Expect(managed).To(BeTrue(),
9889
"NLBSecurityGroupMode must be set to 'Managed' in cloud-config when feature gate is enabled")
9990

10091
framework.Logf("Successfully validated cloud-config contains NLBSecurityGroupMode = Managed")
@@ -531,7 +522,21 @@ func createServiceNLB(ctx context.Context, cs clientset.Interface, ns *v1.Namesp
531522
},
532523
}
533524

534-
_, err := jig.Client.CoreV1().Services(jig.Namespace).Create(ctx, svc, metav1.CreateOptions{})
525+
cloudCfg, err := common.GetCloudConfig(ctx, cs)
526+
if err != nil {
527+
return nil, nil, fmt.Errorf("failed to get cloud-config: %w", err)
528+
}
529+
isDualStack, err := common.IsDualStack(cloudCfg)
530+
if err != nil {
531+
return nil, nil, fmt.Errorf("failed to detect dual-stack from cloud-config: %w", err)
532+
}
533+
if isDualStack {
534+
framework.Logf("Detected DualStack clusters, patching Service setting IPFamilyPolicy to %q", v1.IPFamilyPolicyRequireDualStack)
535+
dualStack := v1.IPFamilyPolicyRequireDualStack
536+
svc.Spec.IPFamilyPolicy = &dualStack
537+
}
538+
539+
_, err = jig.Client.CoreV1().Services(jig.Namespace).Create(ctx, svc, metav1.CreateOptions{})
535540
framework.ExpectNoError(err, "failed to create LoadBalancer Service")
536541

537542
By("waiting for AWS load balancer provisioning")

openshift-tests/ccm-aws-tests/e2e/common/helper.go

Lines changed: 141 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,49 @@ package common
33
import (
44
"context"
55
"fmt"
6+
"regexp"
7+
"strings"
68

79
configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
10+
v1 "k8s.io/api/core/v1"
811
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
clientset "k8s.io/client-go/kubernetes"
913
"k8s.io/kubernetes/test/e2e/framework"
1014
)
1115

16+
const (
17+
cloudConfigNamespace = "openshift-cloud-controller-manager"
18+
cloudConfigName = "cloud-conf"
19+
)
20+
21+
// GetOcClient returns an OpenShift config/v1 API client (FeatureGates, Infrastructures, etc.).
22+
func GetOcClient(ctx context.Context) (*configv1client.ConfigV1Client, error) {
23+
restConfig, err := framework.LoadConfig()
24+
if err != nil {
25+
return nil, fmt.Errorf("failed to load kubeconfig: %w", err)
26+
}
27+
configClient, err := configv1client.NewForConfig(restConfig)
28+
if err != nil {
29+
return nil, fmt.Errorf("failed to openshift client: %w", err)
30+
}
31+
32+
return configClient, nil
33+
}
34+
35+
// GetKubeClient returns a core Kubernetes client (Pods, ConfigMaps, Services, etc.).
36+
func GetKubeClient(ctx context.Context) (clientset.Interface, error) {
37+
restConfig, err := framework.LoadConfig()
38+
if err != nil {
39+
return nil, fmt.Errorf("failed to load kubeconfig: %w", err)
40+
}
41+
cs, err := clientset.NewForConfig(restConfig)
42+
if err != nil {
43+
return nil, fmt.Errorf("failed to kube clientset: %w", err)
44+
}
45+
46+
return cs, nil
47+
}
48+
1249
// IsFeatureEnabled checks if an OpenShift feature gate is enabled by querying the
1350
// FeatureGate resource named "cluster" using the typed OpenShift config API.
1451
//
@@ -27,22 +64,16 @@ import (
2764
// Note: For HyperShift clusters, this checks the management cluster's feature gates.
2865
// To check hosted cluster feature gates, use the hosted cluster's kubeconfig.
2966
func IsFeatureEnabled(ctx context.Context, featureName string) (bool, error) {
30-
// Get the REST config
31-
restConfig, err := framework.LoadConfig()
32-
if err != nil {
33-
return false, fmt.Errorf("failed to load kubeconfig: %v", err)
34-
}
35-
3667
// Create typed config client (more efficient than dynamic client)
37-
configClient, err := configv1client.NewForConfig(restConfig)
68+
oclient, err := GetOcClient(ctx)
3869
if err != nil {
39-
return false, fmt.Errorf("failed to create config client: %v", err)
70+
return false, fmt.Errorf("failed to create config client: %w", err)
4071
}
4172

4273
// Get the FeatureGate resource using typed API
43-
featureGate, err := configClient.FeatureGates().Get(ctx, "cluster", metav1.GetOptions{})
74+
featureGate, err := oclient.FeatureGates().Get(ctx, "cluster", metav1.GetOptions{})
4475
if err != nil {
45-
return false, fmt.Errorf("failed to get FeatureGate 'cluster': %v", err)
76+
return false, fmt.Errorf("failed to get FeatureGate 'cluster': %w", err)
4677
}
4778

4879
// Iterate through the feature gates status (typed structs)
@@ -68,3 +99,103 @@ func IsFeatureEnabled(ctx context.Context, featureName string) (bool, error) {
6899
framework.Logf("Feature %s not found in FeatureGate status", featureName)
69100
return false, nil
70101
}
102+
103+
// GetCloudConfig retrieves the CCM cloud-config ConfigMap.
104+
// When cs is nil, a clientset is created from the current kubeconfig.
105+
// This function must not call Ginkgo control-flow helpers (Skip, Fail, etc.)
106+
// because it is also called from main.go outside a spec context.
107+
func GetCloudConfig(ctx context.Context, cs clientset.Interface) (*v1.ConfigMap, error) {
108+
var err error
109+
if cs == nil {
110+
cs, err = GetKubeClient(ctx)
111+
if err != nil {
112+
return nil, fmt.Errorf("failed to get kubernetes client: %w", err)
113+
}
114+
}
115+
cm, err := cs.CoreV1().ConfigMaps(cloudConfigNamespace).Get(ctx, cloudConfigName, metav1.GetOptions{})
116+
if err != nil {
117+
return nil, fmt.Errorf("failed to get cloud-config ConfigMap: %w", err)
118+
}
119+
return cm, nil
120+
}
121+
122+
// IsConfigPresentCloudConfig checks if a specific configuration key is present in the
123+
// cloud-config data stored in the given ConfigMap. It searches all data entries for an
124+
// INI-style key=value match. Values are split by comma to support multi-value configs
125+
// e.g.: "ipFamilies = IPv4,IPv6" returns ["IPv4", "IPv6"], and
126+
// "NLBSecurityGroupMode" = "Managed" returns ["Managed"].
127+
func IsConfigPresentCloudConfig(cm *v1.ConfigMap, configKey string) (bool, []string, error) {
128+
if cm == nil {
129+
return false, nil, fmt.Errorf("ConfigMap is nil")
130+
}
131+
if configKey == "" {
132+
return false, nil, fmt.Errorf("configKey is empty")
133+
}
134+
135+
pattern, err := regexp.Compile(`(?m)^\s*` + regexp.QuoteMeta(configKey) + `\s*=\s*(.*)$`)
136+
if err != nil {
137+
return false, nil, fmt.Errorf("failed to compile regex for key %q: %w", configKey, err)
138+
}
139+
140+
for dataKey, content := range cm.Data {
141+
allMatches := pattern.FindAllStringSubmatch(content, -1)
142+
if allMatches == nil {
143+
continue
144+
}
145+
146+
var values []string
147+
for _, matches := range allMatches {
148+
rawValue := strings.TrimSpace(matches[1])
149+
if rawValue == "" {
150+
continue
151+
}
152+
for _, p := range strings.Split(rawValue, ",") {
153+
if v := strings.TrimSpace(p); v != "" {
154+
values = append(values, v)
155+
}
156+
}
157+
}
158+
159+
framework.Logf("Found key %q in ConfigMap data key %q with values: %v", configKey, dataKey, values)
160+
return true, values, nil
161+
}
162+
163+
framework.Logf("Key %q not found in ConfigMap %s/%s", configKey, cm.Namespace, cm.Name)
164+
return false, nil, nil
165+
}
166+
167+
// IsNLBSecurityGroupModeManaged returns true when the cloud-config has
168+
// NLBSecurityGroupMode set to "Managed".
169+
func IsNLBSecurityGroupModeManaged(cm *v1.ConfigMap) (bool, error) {
170+
found, values, err := IsConfigPresentCloudConfig(cm, "NLBSecurityGroupMode")
171+
if err != nil {
172+
return false, err
173+
}
174+
if !found {
175+
return false, nil
176+
}
177+
return len(values) == 1 && values[0] == "Managed", nil
178+
}
179+
180+
// IsDualStack returns true when the cloud-config has NodeIPFamilies containing
181+
// both IPv4 and IPv6.
182+
func IsDualStack(cm *v1.ConfigMap) (bool, error) {
183+
found, values, err := IsConfigPresentCloudConfig(cm, "NodeIPFamilies")
184+
if err != nil {
185+
return false, err
186+
}
187+
if !found {
188+
return false, nil
189+
}
190+
var hasIPv4, hasIPv6 bool
191+
// ToDo: review rules and if order matter
192+
for _, v := range values {
193+
switch v {
194+
case "ipv6":
195+
hasIPv6 = true
196+
case "ipv4":
197+
hasIPv4 = true
198+
}
199+
}
200+
return hasIPv4 && hasIPv6, nil
201+
}

openshift-tests/ccm-aws-tests/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ require (
1010
github.com/onsi/ginkgo/v2 v2.28.1
1111
github.com/onsi/gomega v1.39.1
1212
github.com/openshift-eng/openshift-tests-extension v0.0.0-20250916161632-d81c09058835
13+
github.com/openshift/api v0.0.0-20260429122012-1180c0f5c3e9
1314
github.com/openshift/client-go v0.0.0-20260429123927-c81f86abfa6a
1415
github.com/sirupsen/logrus v1.9.4
1516
github.com/spf13/cobra v1.10.2
@@ -68,7 +69,6 @@ require (
6869
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
6970
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
7071
github.com/opencontainers/go-digest v1.0.0 // indirect
71-
github.com/openshift/api v0.0.0-20260429122012-1180c0f5c3e9 // indirect
7272
github.com/pkg/errors v0.9.1 // indirect
7373
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
7474
github.com/prometheus/client_golang v1.23.2 // indirect

openshift-tests/ccm-aws-tests/main.go

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@ import (
2121

2222
// Importing ginkgo tests from the CCM e2e packages
2323
_ "github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/aws"
24-
_ "github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/common"
24+
"github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/common"
2525
_ "k8s.io/cloud-provider-aws/tests/e2e"
2626
)
2727

2828
var (
2929
// testContext is the global test context that is used to store the test configuration.
3030
testContext = &framework.TestContext
31+
32+
isDualStackCluster bool
33+
dualStackDetectionReady bool
3134
)
3235

3336
func main() {
@@ -44,6 +47,17 @@ func main() {
4447
panic(fmt.Errorf("failed to initialize test framework: %w", err))
4548
}
4649

50+
// Detect dual-stack from cloud-config before building specs.
51+
// Upstream load balancer tests do not support dual-stack yet, so they
52+
// must be excluded when the cluster is configured for dual-stack.
53+
if cm, err := common.GetCloudConfig(context.TODO(), nil); err != nil {
54+
log.Debugf("failed to get cloud-config for dual-stack detection: %v", err)
55+
} else {
56+
isDualStackCluster, _ = common.IsDualStack(cm)
57+
dualStackDetectionReady = true
58+
log.Debugf("Dual-stack cluster detected: %v", isDualStackCluster)
59+
}
60+
4761
// Build the extension test specs
4862
specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
4963
if err != nil {
@@ -54,11 +68,22 @@ func main() {
5468
// We need to filter to prevent adding ECR tests.
5569
// All upstream tests must be runnable on OpenShift, if issues are found, let's try to
5670
// fix in upstream to work well with OpenShift and cloud-provider-aws CI.
57-
specs, err = specs.MustSelectAny([]extensiontests.SelectFunction{
58-
extensiontests.NameContains("[cloud-provider-aws-e2e] loadbalancer"),
71+
specSelectors := []extensiontests.SelectFunction{
5972
extensiontests.NameContains("[cloud-provider-aws-e2e] nodes"),
6073
extensiontests.NameContains("[cloud-provider-aws-e2e-openshift]"),
61-
})
74+
}
75+
// Exclude upstream load balancer tests on dual-stack clusters — upstream
76+
// does not support dual-stack yet. When detection fails, the upstream LB
77+
// tests are also excluded to avoid false positives.
78+
// FIXME when upstream e2e supports Service Dual-stack scenarios:
79+
// https://github.com/kubernetes/cloud-provider-aws/pull/1313
80+
// https://github.com/kubernetes/cloud-provider-aws/pull/1356
81+
if isDualStackCluster {
82+
framework.Logf("Dual-stack environment detected, skipping test name that contains '[cloud-provider-aws-e2e] loadbalancer'")
83+
} else {
84+
specSelectors = append(specSelectors, extensiontests.NameContains("[cloud-provider-aws-e2e] loadbalancer"))
85+
}
86+
specs, err = specs.MustSelectAny(specSelectors)
6287
if err != nil {
6388
panic(fmt.Errorf("failed to select specs: %w", err))
6489
}
@@ -79,7 +104,6 @@ func main() {
79104
spec.Exclude(extensiontests.TopologyEquals("SingleReplica"))
80105
}
81106
}
82-
83107
}).Include(extensiontests.PlatformEquals("aws"))
84108
specs.AddBeforeAll(func() {
85109
if err := initFrameworkForTest(); err != nil {

0 commit comments

Comments
 (0)