Skip to content

Commit 428e96c

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #31293 from arkadeepsen/use-apber-for-ex-gw
OCPBUGS-88719: Use AdminPolicyBasedExternalRoute CR for external gateway test
2 parents 0d8f07f + c389785 commit 428e96c

2 files changed

Lines changed: 229 additions & 39 deletions

File tree

‎test/extended/networking/external_gateway.go‎

Lines changed: 170 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,211 @@
11
package networking
22

33
import (
4+
"context"
5+
"fmt"
6+
"regexp"
7+
"time"
8+
49
g "github.com/onsi/ginkgo/v2"
510
o "github.com/onsi/gomega"
611
exutil "github.com/openshift/origin/test/extended/util"
712
e2e "k8s.io/kubernetes/test/e2e/framework"
13+
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
814
admissionapi "k8s.io/pod-security-admission/api"
915
)
1016

1117
var _ = g.Describe("[sig-network] external gateway address", func() {
12-
oc := exutil.NewCLIWithPodSecurityLevel("ns-global", admissionapi.LevelPrivileged)
18+
oc := exutil.NewCLIWithoutNamespace("ns-global")
1319

1420
InOVNKubernetesContext(func() {
1521
f := oc.KubeFramework()
22+
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
1623

1724
g.It("should match the address family of the pod", func() {
25+
labelKey, labelValue := "test", "external-gateway"
26+
labels := map[string]string{
27+
labelKey: labelValue,
28+
}
29+
30+
// Expected error message for APB policy sync failure
31+
errorLog := "Failed to sync APB policy %s.*gateway specified for namespace %s.*%s"
32+
33+
// Returns true if the ovnkube-controller logs contain the expected error message
34+
checkLogs := func(ovnkubePodInfo ovnKubePodInfo, regex *regexp.Regexp) (bool, error) {
35+
logs, err := e2epod.GetPodLogs(context.TODO(), f.ClientSet, ovnNamespace, ovnkubePodInfo.podName, ovnkubePodInfo.containerName)
36+
if err != nil {
37+
return false, err
38+
}
39+
return regex.MatchString(logs), nil
40+
}
41+
42+
g.By("creating namespace with external-gateway label")
43+
ns, err := f.CreateNamespace(context.TODO(), f.BaseName, labels)
44+
expectNoError(err)
45+
f.Namespace = ns
46+
47+
g.By("determining cluster pod IP address family")
1848
podIPFamily := GetIPFamilyForCluster(f)
1949
o.Expect(podIPFamily).NotTo(o.Equal(Unknown))
50+
2051
// Set external gateway address into an IPv6 address and make sure
2152
// pod ip address matches with IPv6 address family.
22-
setNamespaceExternalGateway(f, "fd00:10:244:2::6")
23-
podIPs, err := createPod(f.ClientSet, f.Namespace.Name, "test-ipv6-pod")
53+
apbPolicyNameIPv6 := "static-egress-route-ipv6"
54+
g.By(fmt.Sprintf("applying IPv6 AdminPolicyBasedExternalRoute %s with gateway fd00:10:244:2::6", apbPolicyNameIPv6))
55+
setNamespaceExternalGateway(apbPolicyNameIPv6, []string{"fd00:10:244:2::6"}, labelKey, labelValue)
56+
57+
podNameIPv6 := "test-ipv6-pod"
58+
g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameIPv6, f.Namespace.Name))
59+
pod, err := createPod(f.ClientSet, f.Namespace.Name, podNameIPv6)
60+
expectNoError(err)
61+
podIPs := pod.Status.PodIPs
2462
e2e.Logf("pod IPs are %v after setting external gw with IPv6 address", podIPs)
63+
64+
g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName))
65+
ovnkubePodInfo, err := ovnkubePod(oc, pod.Spec.NodeName)
66+
expectNoError(err)
67+
68+
regexIPv6, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameIPv6, f.Namespace.Name, podNameIPv6))
69+
expectNoError(err)
2570
switch podIPFamily {
26-
case DualStack:
27-
expectNoError(err)
28-
o.Expect(getIPFamily(podIPs)).To(o.Equal(DualStack))
71+
case DualStack, IPv6:
72+
g.By(fmt.Sprintf("verifying ovnkube-node logs do not report APB sync failure for IPv6 gateway on %s cluster", podIPFamily))
73+
o.Consistently(func() bool {
74+
found, err := checkLogs(ovnkubePodInfo, regexIPv6)
75+
if err != nil {
76+
e2e.Logf("Error checking logs: %v", err)
77+
return true
78+
}
79+
return found
80+
}).
81+
WithPolling(20 * time.Second).
82+
WithTimeout(2 * time.Minute).
83+
Should(o.BeFalse())
2984
case IPv4:
3085
// This is an expected failure when pod network in IPv4 address family
3186
// whereas external gateway is set with IPv6 address
32-
expectError(err)
33-
case IPv6:
34-
expectNoError(err)
35-
o.Expect(getIPFamily(podIPs)).To(o.Equal(IPv6))
87+
g.By("verifying ovnkube-node logs report APB sync failure for mismatched IPv6 gateway on IPv4 cluster")
88+
o.Eventually(func() bool {
89+
found, err := checkLogs(ovnkubePodInfo, regexIPv6)
90+
if err != nil {
91+
e2e.Logf("Error checking logs: %v", err)
92+
return false
93+
}
94+
return found
95+
}).
96+
WithPolling(20 * time.Second).
97+
WithTimeout(2 * time.Minute).
98+
Should(o.BeTrue())
3699
}
100+
101+
g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameIPv6))
102+
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameIPv6, "--ignore-not-found").Execute()
103+
expectNoError(err)
104+
105+
g.By(fmt.Sprintf("deleting pod %s", podNameIPv6))
106+
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute()
107+
expectNoError(err)
108+
37109
// Set external gateway address into an IPv4 address and make sure
38110
// pod ip address matches with IPv4 address family.
39-
setNamespaceExternalGateway(f, "10.10.10.1")
40-
podIPs, err = createPod(f.ClientSet, f.Namespace.Name, "test-ipv4-pod")
111+
apbPolicyNameIPv4 := "static-egress-route-ipv4"
112+
g.By(fmt.Sprintf("applying IPv4 AdminPolicyBasedExternalRoute %s with gateway 10.10.10.1", apbPolicyNameIPv4))
113+
setNamespaceExternalGateway(apbPolicyNameIPv4, []string{"10.10.10.1"}, labelKey, labelValue)
114+
115+
podNameIPv4 := "test-ipv4-pod"
116+
g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameIPv4, f.Namespace.Name))
117+
pod, err = createPod(f.ClientSet, f.Namespace.Name, podNameIPv4)
118+
expectNoError(err)
119+
podIPs = pod.Status.PodIPs
41120
e2e.Logf("pod IPs are %v after setting external gw with IPv4 address", podIPs)
121+
122+
g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName))
123+
ovnkubePodInfo, err = ovnkubePod(oc, pod.Spec.NodeName)
124+
expectNoError(err)
125+
126+
regexIPv4, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameIPv4, f.Namespace.Name, podNameIPv4))
127+
expectNoError(err)
42128
switch podIPFamily {
43-
case DualStack:
44-
expectNoError(err)
45-
o.Expect(getIPFamily(podIPs)).To(o.Equal(DualStack))
46-
case IPv4:
47-
expectNoError(err)
48-
o.Expect(getIPFamily(podIPs)).To(o.Equal(IPv4))
129+
case DualStack, IPv4:
130+
g.By(fmt.Sprintf("verifying ovnkube-node logs do not report APB sync failure for IPv4 gateway on %s cluster", podIPFamily))
131+
o.Consistently(func() bool {
132+
found, err := checkLogs(ovnkubePodInfo, regexIPv4)
133+
if err != nil {
134+
e2e.Logf("Error checking logs: %v", err)
135+
return true
136+
}
137+
return found
138+
}).
139+
WithPolling(20 * time.Second).
140+
WithTimeout(2 * time.Minute).
141+
Should(o.BeFalse())
49142
case IPv6:
50143
// This is an expected failure when pod network in IPv6 address family
51144
// whereas external gateway is set with IPv4 address
52-
expectError(err)
145+
g.By("verifying ovnkube-node logs report APB sync failure for mismatched IPv4 gateway on IPv6 cluster")
146+
o.Eventually(func() bool {
147+
found, err := checkLogs(ovnkubePodInfo, regexIPv4)
148+
if err != nil {
149+
e2e.Logf("Error checking logs: %v", err)
150+
return false
151+
}
152+
return found
153+
}).
154+
WithPolling(20 * time.Second).
155+
WithTimeout(2 * time.Minute).
156+
Should(o.BeTrue())
53157
}
158+
159+
g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameIPv4))
160+
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameIPv4, "--ignore-not-found").Execute()
161+
expectNoError(err)
162+
163+
g.By(fmt.Sprintf("deleting pod %s", podNameIPv4))
164+
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute()
165+
expectNoError(err)
166+
54167
// Set external gateway address supporting Dual Stack and make sure
55168
// pod ip address(es) match with desired address family.
56-
setNamespaceExternalGateway(f, "10.10.10.1,fd00:10:244:2::6")
57-
podIPs, err = createPod(f.ClientSet, f.Namespace.Name, "test-dual-stack-pod")
58-
o.Expect(err).NotTo(o.HaveOccurred())
169+
apbPolicyNameDualStack := "static-egress-route-dual-stack"
170+
g.By(fmt.Sprintf("applying dual-stack AdminPolicyBasedExternalRoute %s with gateways 10.10.10.1 and fd00:10:244:2::6", apbPolicyNameDualStack))
171+
setNamespaceExternalGateway(apbPolicyNameDualStack, []string{"10.10.10.1", "fd00:10:244:2::6"}, labelKey, labelValue)
172+
173+
podNameDualStack := "test-dual-stack-pod"
174+
g.By(fmt.Sprintf("creating pod %s in namespace %s", podNameDualStack, f.Namespace.Name))
175+
pod, err = createPod(f.ClientSet, f.Namespace.Name, podNameDualStack)
176+
expectNoError(err)
177+
podIPs = pod.Status.PodIPs
59178
e2e.Logf("pod IPs are %v after setting external gw with Dual Stack address", podIPs)
179+
180+
g.By("verifying pod IP address family matches cluster")
60181
o.Expect(getIPFamily(podIPs)).To(o.Equal(podIPFamily))
182+
183+
g.By(fmt.Sprintf("finding ovnkube-node pod on node %s", pod.Spec.NodeName))
184+
ovnkubePodInfo, err = ovnkubePod(oc, pod.Spec.NodeName)
185+
expectNoError(err)
186+
187+
regexDualStack, err := regexp.Compile(fmt.Sprintf(errorLog, apbPolicyNameDualStack, f.Namespace.Name, podNameDualStack))
188+
expectNoError(err)
189+
g.By("verifying ovnkube-node logs do not report APB sync failure for dual-stack gateway")
190+
o.Consistently(func() bool {
191+
found, err := checkLogs(ovnkubePodInfo, regexDualStack)
192+
if err != nil {
193+
e2e.Logf("Error checking logs: %v", err)
194+
return true
195+
}
196+
return found
197+
}).
198+
WithPolling(20 * time.Second).
199+
WithTimeout(2 * time.Minute).
200+
Should(o.BeFalse())
201+
202+
g.By(fmt.Sprintf("deleting AdminPolicyBasedExternalRoute %s", apbPolicyNameDualStack))
203+
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("adminpolicybasedexternalroute", apbPolicyNameDualStack, "--ignore-not-found").Execute()
204+
expectNoError(err)
205+
206+
g.By(fmt.Sprintf("deleting pod %s", podNameDualStack))
207+
err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", pod.Name, "-n", f.Namespace.Name, "--ignore-not-found").Execute()
208+
expectNoError(err)
61209
})
62210
})
63211
})

‎test/extended/networking/util.go‎

Lines changed: 59 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package networking
22

33
import (
4+
"bytes"
45
"context"
56
"encoding/json"
67
"errors"
@@ -11,6 +12,7 @@ import (
1112
"path/filepath"
1213
"strconv"
1314
"strings"
15+
"text/template"
1416
"time"
1517

1618
netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
@@ -317,18 +319,54 @@ func modifyNetworkConfig(configClient configv1client.Interface, autoAssignCIDRs,
317319
expectNoError(kubeAPIServerRollout.Err())
318320
}
319321

320-
func setNamespaceExternalGateway(f *e2e.Framework, gatewayIP string) {
321-
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
322-
ns, err := f.ClientSet.CoreV1().Namespaces().Get(context.Background(), f.Namespace.Name, metav1.GetOptions{})
323-
expectNoError(err)
324-
if ns.Annotations == nil {
325-
ns.Annotations = make(map[string]string)
326-
}
327-
ns.Annotations["k8s.ovn.org/routing-external-gws"] = gatewayIP
328-
_, err = f.ClientSet.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{})
329-
return err
322+
const adminPolicyBasedExternalRouteTemplate = `
323+
apiVersion: k8s.ovn.org/v1
324+
kind: AdminPolicyBasedExternalRoute
325+
metadata:
326+
name: {{ .Name }}
327+
spec:
328+
from:
329+
namespaceSelector:
330+
matchLabels:
331+
{{ .LabelKey | quote }}: {{ .LabelValue | quote }}
332+
nextHops:
333+
static:
334+
{{- range .IPs }}
335+
- ip: {{ . | quote }}
336+
{{- end }}`
337+
338+
type adminPolicyBasedExternalRouteParams struct {
339+
Name string
340+
LabelKey string
341+
LabelValue string
342+
IPs []string
343+
}
344+
345+
func renderAdminPolicyBasedExternalRoute(params adminPolicyBasedExternalRouteParams) (string, error) {
346+
tmpl, err := template.New("adminPolicyBasedExternalRoute").Funcs(template.FuncMap{
347+
"quote": func(s string) string {
348+
return fmt.Sprintf("%q", s)
349+
},
350+
}).Parse(adminPolicyBasedExternalRouteTemplate)
351+
if err != nil {
352+
return "", err
353+
}
354+
var manifest bytes.Buffer
355+
if err := tmpl.Execute(&manifest, params); err != nil {
356+
return "", err
357+
}
358+
return manifest.String(), nil
359+
}
360+
361+
func setNamespaceExternalGateway(apbPolicyName string, gatewayIPs []string, labelKey string, labelValue string) {
362+
manifest, err := renderAdminPolicyBasedExternalRoute(adminPolicyBasedExternalRouteParams{
363+
Name: apbPolicyName,
364+
LabelKey: labelKey,
365+
LabelValue: labelValue,
366+
IPs: gatewayIPs,
330367
})
331368
expectNoError(err)
369+
expectNoError(applyManifest("", manifest))
332370
}
333371

334372
// findAppropriateNodes tries to find a source and destination for a type of node connectivity
@@ -464,9 +502,14 @@ func networkAttachmentDefinitionClient(config *rest.Config) (dynamic.Namespaceab
464502
}
465503

466504
func GetIPFamilyForCluster(f *e2e.Framework) IPFamily {
467-
podIPs, err := createPod(f.ClientSet, f.Namespace.Name, "test-ip-family-pod")
505+
pod, err := createPod(f.ClientSet, f.Namespace.Name, "test-ip-family-pod")
506+
expectNoError(err)
507+
podIPFamily := getIPFamily(pod.Status.PodIPs)
508+
// Delete the temporary pod
509+
err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
468510
expectNoError(err)
469-
return getIPFamily(podIPs)
511+
// Return the IP family of the pod
512+
return podIPFamily
470513
}
471514

472515
func getIPFamily(podIPs []corev1.PodIP) IPFamily {
@@ -493,21 +536,20 @@ func getIPFamily(podIPs []corev1.PodIP) IPFamily {
493536
}
494537
}
495538

496-
func createPod(client k8sclient.Interface, ns, generateName string) ([]corev1.PodIP, error) {
539+
func createPod(client k8sclient.Interface, ns, generateName string) (*corev1.Pod, error) {
497540
pod := e2epod.NewAgnhostPod(ns, "", nil, nil, nil)
498541
pod.ObjectMeta.GenerateName = generateName
499542
execPod, err := client.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
500543
expectNoError(err, "failed to create new pod in namespace: %s", ns)
501-
var podIPs []corev1.PodIP
544+
var retrievedPod *corev1.Pod
502545
err = wait.PollImmediate(poll, 2*time.Minute, func() (bool, error) {
503-
retrievedPod, err := client.CoreV1().Pods(execPod.Namespace).Get(context.TODO(), execPod.Name, metav1.GetOptions{})
546+
retrievedPod, err = client.CoreV1().Pods(execPod.Namespace).Get(context.TODO(), execPod.Name, metav1.GetOptions{})
504547
if err != nil {
505548
return false, err
506549
}
507-
podIPs = retrievedPod.Status.PodIPs
508550
return retrievedPod.Status.Phase == corev1.PodRunning, nil
509551
})
510-
return podIPs, err
552+
return retrievedPod, err
511553
}
512554

513555
// SubnetIPs enumerates all IP addresses in an IP subnet (starting with the provided IP address and including the broadcast address).

0 commit comments

Comments
 (0)