Skip to content

Commit 0181640

Browse files
Merge pull request #466 from mtulio/OCPBUGS-86299
OCPBUGS-86299: e2e/ccm-aws-ote: support to dual-stack IPv6 primary
2 parents 7f6aa93 + b0ead8c commit 0181640

3 files changed

Lines changed: 197 additions & 30 deletions

File tree

openshift-tests/ccm-aws-tests/e2e/aws/loadbalancer.go

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
1010
. "github.com/onsi/ginkgo/v2"
1111
. "github.com/onsi/gomega"
12+
"github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/common"
1213
v1 "k8s.io/api/core/v1"
1314
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1415
"k8s.io/apimachinery/pkg/util/intstr"
@@ -30,10 +31,6 @@ const (
3031
featureGateAWSServiceLBNetworkSecurityGroup = "AWSServiceLBNetworkSecurityGroup"
3132

3233
annotationLBType = "service.beta.kubernetes.io/aws-load-balancer-type"
33-
34-
cloudConfigNamespace = "openshift-cloud-controller-manager"
35-
cloudConfigName = "cloud-conf"
36-
cloudConfigKey = "cloud.conf"
3734
)
3835

3936
// TestAWSServiceLBNetworkSecurityGroup validates the AWSServiceLBNetworkSecurityGroup feature gate functionality.
@@ -82,19 +79,13 @@ var _ = Describe(fmt.Sprintf("%s NLB [OCPFeatureGate:%s]", e2eTestPrefixLoadBala
8279
isNLBFeatureEnabled(ctx)
8380

8481
By("getting cloud-config ConfigMap from openshift-cloud-controller-manager namespace")
85-
cm, err := cs.CoreV1().ConfigMaps(cloudConfigNamespace).Get(ctx, cloudConfigName, metav1.GetOptions{})
82+
cm, err := common.GetCloudConfig(ctx, cs)
8683
framework.ExpectNoError(err, "failed to get cloud-config ConfigMap")
8784

88-
By("checking if cloud.conf key exists in ConfigMap")
89-
cloudConf, exists := cm.Data[cloudConfigKey]
90-
Expect(exists).To(BeTrue(), "cloud.conf key not found in ConfigMap")
91-
92-
By("verifying NLBSecurityGroupMode is present in cloud config")
93-
Expect(cloudConf).To(ContainSubstring("NLBSecurityGroupMode"),
94-
"NLBSecurityGroupMode must be present in cloud-config when feature gate is enabled")
95-
9685
By("verifying NLBSecurityGroupMode is set to Managed")
97-
Expect(cloudConf).To(MatchRegexp(`NLBSecurityGroupMode\s*=\s*Managed`),
86+
managed, err := common.IsNLBSecurityGroupModeManaged(cm)
87+
framework.ExpectNoError(err, "failed to check NLBSecurityGroupMode in cloud-config")
88+
Expect(managed).To(BeTrue(),
9889
"NLBSecurityGroupMode must be set to 'Managed' in cloud-config when feature gate is enabled")
9990

10091
framework.Logf("Successfully validated cloud-config contains NLBSecurityGroupMode = Managed")
@@ -531,7 +522,21 @@ func createServiceNLB(ctx context.Context, cs clientset.Interface, ns *v1.Namesp
531522
},
532523
}
533524

534-
_, err := jig.Client.CoreV1().Services(jig.Namespace).Create(ctx, svc, metav1.CreateOptions{})
525+
cloudCfg, err := common.GetCloudConfig(ctx, cs)
526+
if err != nil {
527+
return nil, nil, fmt.Errorf("failed to get cloud-config: %w", err)
528+
}
529+
isDualStack, _, err := common.IsDualStack(cloudCfg)
530+
if err != nil {
531+
return nil, nil, fmt.Errorf("failed to detect dual-stack from cloud-config: %w", err)
532+
}
533+
if isDualStack {
534+
framework.Logf("Detected DualStack clusters, patching Service setting IPFamilyPolicy to %q", v1.IPFamilyPolicyRequireDualStack)
535+
dualStack := v1.IPFamilyPolicyRequireDualStack
536+
svc.Spec.IPFamilyPolicy = &dualStack
537+
}
538+
539+
_, err = jig.Client.CoreV1().Services(jig.Namespace).Create(ctx, svc, metav1.CreateOptions{})
535540
framework.ExpectNoError(err, "failed to create LoadBalancer Service")
536541

537542
By("waiting for AWS load balancer provisioning")

openshift-tests/ccm-aws-tests/e2e/common/helper.go

Lines changed: 144 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,49 @@ package common
33
import (
44
"context"
55
"fmt"
6+
"regexp"
7+
"strings"
68

79
configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
10+
v1 "k8s.io/api/core/v1"
811
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
clientset "k8s.io/client-go/kubernetes"
913
"k8s.io/kubernetes/test/e2e/framework"
1014
)
1115

16+
const (
17+
cloudConfigNamespace = "openshift-cloud-controller-manager"
18+
cloudConfigName = "cloud-conf"
19+
)
20+
21+
// GetOcClient returns an OpenShift config/v1 API client (FeatureGates, Infrastructures, etc.).
22+
func GetOcClient(ctx context.Context) (*configv1client.ConfigV1Client, error) {
23+
restConfig, err := framework.LoadConfig()
24+
if err != nil {
25+
return nil, fmt.Errorf("failed to load kubeconfig: %w", err)
26+
}
27+
configClient, err := configv1client.NewForConfig(restConfig)
28+
if err != nil {
29+
return nil, fmt.Errorf("failed to openshift client: %w", err)
30+
}
31+
32+
return configClient, nil
33+
}
34+
35+
// GetKubeClient returns a core Kubernetes client (Pods, ConfigMaps, Services, etc.).
36+
func GetKubeClient(ctx context.Context) (clientset.Interface, error) {
37+
restConfig, err := framework.LoadConfig()
38+
if err != nil {
39+
return nil, fmt.Errorf("failed to load kubeconfig: %w", err)
40+
}
41+
cs, err := clientset.NewForConfig(restConfig)
42+
if err != nil {
43+
return nil, fmt.Errorf("failed to kube clientset: %w", err)
44+
}
45+
46+
return cs, nil
47+
}
48+
1249
// IsFeatureEnabled checks if an OpenShift feature gate is enabled by querying the
1350
// FeatureGate resource named "cluster" using the typed OpenShift config API.
1451
//
@@ -27,22 +64,16 @@ import (
2764
// Note: For HyperShift clusters, this checks the management cluster's feature gates.
2865
// To check hosted cluster feature gates, use the hosted cluster's kubeconfig.
2966
func IsFeatureEnabled(ctx context.Context, featureName string) (bool, error) {
30-
// Get the REST config
31-
restConfig, err := framework.LoadConfig()
32-
if err != nil {
33-
return false, fmt.Errorf("failed to load kubeconfig: %v", err)
34-
}
35-
3667
// Create typed config client (more efficient than dynamic client)
37-
configClient, err := configv1client.NewForConfig(restConfig)
68+
oclient, err := GetOcClient(ctx)
3869
if err != nil {
39-
return false, fmt.Errorf("failed to create config client: %v", err)
70+
return false, fmt.Errorf("failed to create config client: %w", err)
4071
}
4172

4273
// Get the FeatureGate resource using typed API
43-
featureGate, err := configClient.FeatureGates().Get(ctx, "cluster", metav1.GetOptions{})
74+
featureGate, err := oclient.FeatureGates().Get(ctx, "cluster", metav1.GetOptions{})
4475
if err != nil {
45-
return false, fmt.Errorf("failed to get FeatureGate 'cluster': %v", err)
76+
return false, fmt.Errorf("failed to get FeatureGate 'cluster': %w", err)
4677
}
4778

4879
// Iterate through the feature gates status (typed structs)
@@ -68,3 +99,106 @@ func IsFeatureEnabled(ctx context.Context, featureName string) (bool, error) {
6899
framework.Logf("Feature %s not found in FeatureGate status", featureName)
69100
return false, nil
70101
}
102+
103+
// GetCloudConfig retrieves the CCM cloud-config ConfigMap.
104+
// When cs is nil, a clientset is created from the current kubeconfig.
105+
// This function must not call Ginkgo control-flow helpers (Skip, Fail, etc.)
106+
// because it is also called from main.go outside a spec context.
107+
func GetCloudConfig(ctx context.Context, cs clientset.Interface) (*v1.ConfigMap, error) {
108+
var err error
109+
if cs == nil {
110+
cs, err = GetKubeClient(ctx)
111+
if err != nil {
112+
return nil, fmt.Errorf("failed to get kubernetes client: %w", err)
113+
}
114+
}
115+
cm, err := cs.CoreV1().ConfigMaps(cloudConfigNamespace).Get(ctx, cloudConfigName, metav1.GetOptions{})
116+
if err != nil {
117+
return nil, fmt.Errorf("failed to get cloud-config ConfigMap: %w", err)
118+
}
119+
return cm, nil
120+
}
121+
122+
// IsConfigPresentCloudConfig checks if a specific configuration key is present in the
123+
// cloud-config data stored in the given ConfigMap. It searches all data entries for an
124+
// INI-style key=value match. Values are split by comma to support multi-value configs
125+
// e.g.: "ipFamilies = IPv4,IPv6" returns ["IPv4", "IPv6"], and
126+
// "NLBSecurityGroupMode" = "Managed" returns ["Managed"].
127+
func IsConfigPresentCloudConfig(cm *v1.ConfigMap, configKey string) (bool, []string, error) {
128+
if cm == nil {
129+
return false, nil, fmt.Errorf("ConfigMap is nil")
130+
}
131+
if configKey == "" {
132+
return false, nil, fmt.Errorf("configKey is empty")
133+
}
134+
135+
pattern, err := regexp.Compile(`(?m)^\s*` + regexp.QuoteMeta(configKey) + `\s*=\s*(.*)$`)
136+
if err != nil {
137+
return false, nil, fmt.Errorf("failed to compile regex for key %q: %w", configKey, err)
138+
}
139+
140+
for dataKey, content := range cm.Data {
141+
allMatches := pattern.FindAllStringSubmatch(content, -1)
142+
if allMatches == nil {
143+
continue
144+
}
145+
146+
var values []string
147+
for _, matches := range allMatches {
148+
rawValue := strings.TrimSpace(matches[1])
149+
if rawValue == "" {
150+
continue
151+
}
152+
for _, p := range strings.Split(rawValue, ",") {
153+
if v := strings.TrimSpace(p); v != "" {
154+
values = append(values, v)
155+
}
156+
}
157+
}
158+
159+
framework.Logf("Found key %q in ConfigMap data key %q with values: %v", configKey, dataKey, values)
160+
return true, values, nil
161+
}
162+
163+
framework.Logf("Key %q not found in ConfigMap %s/%s", configKey, cm.Namespace, cm.Name)
164+
return false, nil, nil
165+
}
166+
167+
// IsNLBSecurityGroupModeManaged returns true when the cloud-config has
168+
// NLBSecurityGroupMode set to "Managed".
169+
func IsNLBSecurityGroupModeManaged(cm *v1.ConfigMap) (bool, error) {
170+
found, values, err := IsConfigPresentCloudConfig(cm, "NLBSecurityGroupMode")
171+
if err != nil {
172+
return false, err
173+
}
174+
if !found {
175+
return false, nil
176+
}
177+
return len(values) == 1 && values[0] == "Managed", nil
178+
}
179+
180+
// IsDualStack checks the NodeIPFamilies key in the cloud-config ConfigMap.
181+
// It returns (isDualStack, primaryIPv6, error) where isDualStack is true when
182+
// both IPv4 and IPv6 are present, and primaryIPv6 is true when the first
183+
// entry is IPv6 (e.g. NodeIPFamilies=ipv6 then NodeIPFamilies=ipv4).
184+
// When NodeIPFamilies is absent, both booleans are false with no error.
185+
func IsDualStack(cm *v1.ConfigMap) (bool, bool, error) {
186+
found, values, err := IsConfigPresentCloudConfig(cm, "NodeIPFamilies")
187+
if err != nil {
188+
return false, false, fmt.Errorf("failed to lookup up configuration NodeIPFamilies in cloud-config: %w", err)
189+
}
190+
if !found {
191+
return false, false, nil
192+
}
193+
var hasIPv4, hasIPv6 bool
194+
for _, ipFamily := range values {
195+
switch strings.ToLower(ipFamily) {
196+
case "ipv6":
197+
hasIPv6 = true
198+
case "ipv4":
199+
hasIPv4 = true
200+
}
201+
}
202+
primaryIPv6 := len(values) > 0 && strings.ToLower(values[0]) == "ipv6"
203+
return hasIPv4 && hasIPv6, primaryIPv6, nil
204+
}

openshift-tests/ccm-aws-tests/main.go

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,17 @@ import (
2121

2222
// Importing ginkgo tests from the CCM e2e packages
2323
_ "github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/aws"
24-
_ "github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/common"
24+
"github.com/openshift/cluster-cloud-controller-manager-operator/openshift-tests/ccm-aws-tests/e2e/common"
2525
_ "k8s.io/cloud-provider-aws/tests/e2e"
2626
)
2727

2828
var (
2929
// testContext is the global test context that is used to store the test configuration.
3030
testContext = &framework.TestContext
31+
32+
isDualStackCluster bool
33+
isDualStackPrimaryIpv6 bool
34+
dualStackDetectionReady bool
3135
)
3236

3337
func main() {
@@ -44,6 +48,20 @@ func main() {
4448
panic(fmt.Errorf("failed to initialize test framework: %w", err))
4549
}
4650

51+
// Detect dual-stack from cloud-config before building specs.
52+
// Upstream load balancer tests do not support dual-stack yet, so they
53+
// must be excluded when the cluster is configured for dual-stack.
54+
if cm, err := common.GetCloudConfig(context.TODO(), nil); err != nil {
55+
log.Debugf("failed to get cloud-config for dual-stack detection: %v", err)
56+
} else {
57+
isDualStackCluster, isDualStackPrimaryIpv6, err = common.IsDualStack(cm)
58+
if err != nil {
59+
log.Debugf("failed to evaluate dual-stack configuration, leaving default Service config: %v", err)
60+
}
61+
dualStackDetectionReady = true
62+
log.Debugf("Dual-stack cluster detected: %v", isDualStackCluster)
63+
}
64+
4765
// Build the extension test specs
4866
specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
4967
if err != nil {
@@ -54,11 +72,22 @@ func main() {
5472
// We need to filter to prevent adding ECR tests.
5573
// All upstream tests must be runnable on OpenShift, if issues are found, let's try to
5674
// fix in upstream to work well with OpenShift and cloud-provider-aws CI.
57-
specs, err = specs.MustSelectAny([]extensiontests.SelectFunction{
58-
extensiontests.NameContains("[cloud-provider-aws-e2e] loadbalancer"),
75+
specSelectors := []extensiontests.SelectFunction{
5976
extensiontests.NameContains("[cloud-provider-aws-e2e] nodes"),
6077
extensiontests.NameContains("[cloud-provider-aws-e2e-openshift]"),
61-
})
78+
}
79+
// Exclude upstream load balancer tests on dual-stack clusters — upstream
80+
// does not support dual-stack yet. When detection fails, the upstream LB
81+
// tests are also excluded to avoid false positives.
82+
// FIXME when upstream e2e supports Service Dual-stack scenarios:
83+
// https://github.com/kubernetes/cloud-provider-aws/pull/1313
84+
// https://github.com/kubernetes/cloud-provider-aws/pull/1356
85+
if isDualStackCluster && isDualStackPrimaryIpv6 {
86+
framework.Logf("Dual-stack cluster with Primary IPv6 detected, skipping test name that contains '[cloud-provider-aws-e2e] loadbalancer'")
87+
} else {
88+
specSelectors = append(specSelectors, extensiontests.NameContains("[cloud-provider-aws-e2e] loadbalancer"))
89+
}
90+
specs, err = specs.MustSelectAny(specSelectors)
6291
if err != nil {
6392
panic(fmt.Errorf("failed to select specs: %w", err))
6493
}
@@ -79,7 +108,6 @@ func main() {
79108
spec.Exclude(extensiontests.TopologyEquals("SingleReplica"))
80109
}
81110
}
82-
83111
}).Include(extensiontests.PlatformEquals("aws"))
84112
specs.AddBeforeAll(func() {
85113
if err := initFrameworkForTest(); err != nil {

0 commit comments

Comments
 (0)