Skip to content

Commit f277ff7

Browse files
moshevaynermoshevayner
authored
[test] Stabilize flaky e2e tests by replacing hard sleeps with polling loops (#561)
* fix(test): Stabilize flaky e2e tests by replacing hard sleeps with polling loops Signed-off-by: Moshe Vayner <moshe@vayner.me> * [test]: Additional fixes for flaky tests Signed-off-by: Moshe Vayner <moshe@vayner.me> * [test] Increase timeout and retry attempts for firewall deletion in chainsaw test Signed-off-by: Moshe Vayner <moshe@vayner.me> * [test] Re-add the 180 seconds sleep in `lb-with-node-addition` We need this to allow the new node to come up Signed-off-by: Moshe Vayner <moshe@vayner.me> * [test]: Fix curl error handling in reserved IP HTTP response checks curl -w "%{http_code}" already outputs "000" when a connection fails, then exits non-zero. Without error suppression, `set -euo pipefail` exits the script immediately. Add `|| true` to suppress the exit code while preserving curl's own `"000"` output, allowing the retry loop to continue. Signed-off-by: Moshe Vayner <moshe@vayner.me> --------- Signed-off-by: Moshe Vayner <moshe@vayner.me>
1 parent a8ed016 commit f277ff7

17 files changed

Lines changed: 197 additions & 175 deletions

File tree

‎e2e/test/lb-created-with-invalid-ip/chainsaw-test.yaml‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,21 +73,23 @@ spec:
7373
- script:
7474
content: |
7575
set -euo pipefail
76-
sleep 30
7776
invalid_ip=$(kubectl get configmap invalid-ip-config -o=jsonpath='{.data.InvalidIP}' -n $NAMESPACE)
7877
if [[ -z "$invalid_ip" ]]; then
7978
echo "Error: No invalid ip found in configmap"
8079
fi
8180
8281
annotation="service.beta.kubernetes.io/linode-loadbalancer-reserved-ipv4"
83-
events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed --sort-by='.lastTimestamp' -o json)
84-
message=$(echo $events | jq .items[0].message)
8582
86-
if [[ "$message" == *"Error syncing load balancer: failed to ensure load balancer: [400] Invalid IPv4 address"* ]]; then
87-
echo "Warning event found"
88-
else
89-
echo "Warning event not found"
90-
fi
83+
for i in {1..10}; do
84+
events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed --sort-by='.lastTimestamp' -o json)
85+
message=$(echo $events | jq .items[0].message)
86+
87+
if [[ "$message" == *"Error syncing load balancer: failed to ensure load balancer: [400] Invalid IPv4 address"* ]]; then
88+
echo "Warning event found"
89+
break
90+
fi
91+
sleep 10
92+
done
9193
9294
service_ip=$(kubectl get svc svc-test -n $NAMESPACE -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
9395
if [[ "$service_ip" != "" ]]; then

‎e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -163,29 +163,32 @@ spec:
163163
164164
parallel -j 2 patch_annotation ::: $reserved_ip2 "100.10.10.10"
165165
166-
sleep 20
166+
for i in {1..10}; do
167+
all_events=$(kubectl get events -n $NAMESPACE)
168+
events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
169+
num_events=$(echo $events | jq '.items | length')
167170
168-
all_events=$(kubectl get events -n $NAMESPACE)
171+
#k8s scheduler will flatten the work queue of updates for an object into a single reconcile call.
172+
#If k8s scheduler squashes both the patches into 1 only 1 event is generated
169173
170-
events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
171-
num_events=$(echo $events | jq '.items | length')
172-
173-
#k8s scheduler will flatten the work queue of updates for an object into a single reconcile call.
174-
#If k8s scheduler squashes both the patches into 1 only 1 event is generated
174+
if [[ num_events -eq 0 ]]; then
175+
sleep 10
176+
continue
177+
fi
175178
176-
if [[ num_events -eq 0 ]]; then
177-
echo "Warning event not found"
178-
else
179179
message=$(echo $events | jq .items[0].message)
180180
if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ || "$message" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
181181
echo "First warning event found"
182+
break
182183
elif [[ num_events -eq 2 ]]; then
183184
message2=$(echo $events | jq .items[1].message)
184185
if [[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ || "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
185186
echo "Second warning event found"
187+
break
186188
fi
187189
fi
188-
fi
190+
sleep 10
191+
done
189192
check:
190193
($error == null): true
191194
(contains($stdout, 'No reserved ip found in configmap')): false

‎e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -131,17 +131,16 @@ spec:
131131
echo "Unable to update annotation"
132132
fi
133133
134-
sleep 20
135-
136-
events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
137-
138-
message=$(echo $events | jq .items[0].message)
139-
140-
if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $unreserved_ip,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
141-
echo "Warning event found"
142-
else
143-
echo "Warning event not found"
144-
fi
134+
for i in {1..10}; do
135+
events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
136+
message=$(echo $events | jq .items[0].message)
137+
138+
if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $unreserved_ip,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
139+
echo "Warning event found"
140+
break
141+
fi
142+
sleep 10
143+
done
145144
check:
146145
($error == null): true
147146
(contains($stdout, 'Warning event found')): true

‎e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -159,17 +159,16 @@ spec:
159159
echo "Unable to update annotation"
160160
fi
161161
162-
sleep 20
163-
164-
events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
165-
166-
message=$(echo $events | jq .items[0].message)
167-
168-
if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
169-
echo "Warning event found"
170-
else
171-
echo "Warning event not found"
172-
fi
162+
for i in {1..10}; do
163+
events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
164+
message=$(echo $events | jq .items[0].message)
165+
166+
if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
167+
echo "Warning event found"
168+
break
169+
fi
170+
sleep 10
171+
done
173172
check:
174173
($error == null): true
175174
(contains($stdout, 'Warning event found')): true

‎e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -174,15 +174,16 @@ spec:
174174
echo "IPs do not match"
175175
fi
176176
177-
sleep 30
178177
#Run a curl command to the service ip
179178
URL="http://$service_ip:80/"
180-
HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL")
181-
if [ "$HTTP_RESPONSE" -eq 200 ]; then
182-
echo "Request was successful (HTTP 200)"
183-
else
184-
echo "Request failed with response code: $HTTP_RESPONSE"
185-
fi
179+
for i in {1..10}; do
180+
HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL" || true)
181+
if [ "$HTTP_RESPONSE" -eq 200 ]; then
182+
echo "Request was successful (HTTP 200)"
183+
break
184+
fi
185+
sleep 10
186+
done
186187
check:
187188
($error == null): true
188189
(contains($stdout, 'No reserved ip found in configmap')): false

‎e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml‎

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
apiVersion: chainsaw.kyverno.io/v1alpha1
33
kind: Test
44
metadata:
5-
name: c
5+
name: lb-created-with-reserved-ip-multiple-change-ip
66
labels:
77
all:
88
lke:
@@ -161,8 +161,6 @@ spec:
161161
}
162162
}")
163163
164-
sleep 30
165-
166164
patch2=$(kubectl patch service svc-test -n $NAMESPACE --patch "{
167165
\"metadata\": {
168166
\"annotations\": {
@@ -171,30 +169,25 @@ spec:
171169
}
172170
}")
173171
174-
sleep 30
175172
if [[ "$patch" != "service/svc-test patched" ]]; then
176173
echo "Unable to update annotation"
177174
fi
178175
if [[ "$patch2" != "service/svc-test patched" ]]; then
179176
echo "Unable to update annotation"
180177
fi
181-
182-
events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
183178
184-
message=$(echo $events | jq .items[0].message)
185-
message2=$(echo $events | jq .items[1].message)
179+
for i in {1..10}; do
180+
events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
181+
message=$(echo $events | jq .items[0].message)
182+
message2=$(echo $events | jq .items[1].message)
186183
187-
if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
188-
echo "Warning event found"
189-
else
190-
echo "Warning event not found"
191-
fi
192-
193-
if [[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
194-
echo "Warning event found"
195-
else
196-
echo "Warning event not found"
197-
fi
184+
if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]] && \
185+
[[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
186+
echo "Warning event found"
187+
break
188+
fi
189+
sleep 10
190+
done
198191
check:
199192
($error == null): true
200193
(contains($stdout, 'No reserved ip found in configmap')): false

‎e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -206,15 +206,16 @@ spec:
206206
echo "IPs do not match"
207207
fi
208208
209-
sleep 30
210209
#Run a curl command to the service ip
211210
URL="http://$service_ip:80/"
212-
HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL")
213-
if [ "$HTTP_RESPONSE" -eq 200 ]; then
214-
echo "Request was successful (HTTP 200)"
215-
else
216-
echo "Request failed with response code: $HTTP_RESPONSE"
217-
fi
211+
for i in {1..10}; do
212+
HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL" || true)
213+
if [ "$HTTP_RESPONSE" -eq 200 ]; then
214+
echo "Request was successful (HTTP 200)"
215+
break
216+
fi
217+
sleep 10
218+
done
218219
check:
219220
($error == null): true
220221
(contains($stdout, 'No reserved ip found in configmap')): false

‎e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,6 @@ spec:
170170
else
171171
echo "IPs do not match"
172172
fi
173-
sleep 20
174173
175174
check:
176175
($error == null): true

‎e2e/test/lb-fw-delete-acl/chainsaw-test.yaml‎

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -65,45 +65,45 @@ spec:
6565
- name: Delete ACL and check that firewall no longer exists
6666
try:
6767
- script:
68+
timeout: 5m
6869
content: |
6970
set -euo pipefail
7071
71-
for i in {1..10}; do
72-
nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh)
73-
74-
fw=$(curl -s --request GET \
75-
-H "Authorization: Bearer $LINODE_TOKEN" \
76-
-H "Content-Type: application/json" --fail-early --retry 3 \
77-
-H "accept: application/json" \
78-
"$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true)
79-
80-
fwid=$(echo $fw | jq -r '.data[].id')
81-
82-
# Patch service to remove ACL annotation
83-
kubectl patch service svc-test -n $NAMESPACE --type=json -p='[{"op": "remove", "path": "/metadata/annotations/service.beta.kubernetes.io~1linode-loadbalancer-firewall-acl"}]'
84-
sleep 5
85-
86-
# Check that firewall is no longer attached to nb
87-
fw=$(curl -s --request GET \
88-
-H "Authorization: Bearer $LINODE_TOKEN" \
89-
-H "Content-Type: application/json" --fail-early --retry 3 \
90-
-H "accept: application/json" \
91-
"$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true)
92-
93-
fwCount=$(echo $fw | jq -r '.data | length')
94-
95-
# Check if firewall is deleted
96-
fwRespCode=$(curl -s -o /dev/null -w "%{http_code}" \
97-
--request GET \
98-
-H "Authorization: Bearer $LINODE_TOKEN" \
99-
-H "accept: application/json" \
100-
"$LINODE_URL/v4/networking/firewalls/${fwid}" || true)
101-
102-
if [[ $fwCount -eq 0 && $fwRespCode -eq "404" ]]; then
103-
echo "firewall detatched and deleted"
104-
break
105-
fi
106-
sleep 10
72+
nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh)
73+
74+
fw=$(curl -s --request GET \
75+
-H "Authorization: Bearer $LINODE_TOKEN" \
76+
-H "Content-Type: application/json" --fail-early --retry 3 \
77+
-H "accept: application/json" \
78+
"$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true)
79+
80+
fwid=$(echo $fw | jq -r '.data[].id')
81+
82+
# Patch service to remove ACL annotation
83+
kubectl patch service svc-test -n $NAMESPACE --type=json -p='[{"op": "remove", "path": "/metadata/annotations/service.beta.kubernetes.io~1linode-loadbalancer-firewall-acl"}]'
84+
85+
for i in {1..30}; do
86+
# Check that firewall is no longer attached to nb
87+
fw=$(curl -s --request GET \
88+
-H "Authorization: Bearer $LINODE_TOKEN" \
89+
-H "Content-Type: application/json" --fail-early --retry 3 \
90+
-H "accept: application/json" \
91+
"$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true)
92+
93+
fwCount=$(echo $fw | jq -r '.data | length')
94+
95+
# Check if firewall is deleted
96+
fwRespCode=$(curl -s -o /dev/null -w "%{http_code}" \
97+
--request GET \
98+
-H "Authorization: Bearer $LINODE_TOKEN" \
99+
-H "accept: application/json" \
100+
"$LINODE_URL/v4/networking/firewalls/${fwid}" || true)
101+
102+
if [[ $fwCount -eq 0 && $fwRespCode -eq "404" ]]; then
103+
echo "firewall detatched and deleted"
104+
break
105+
fi
106+
sleep 10
107107
done
108108
check:
109109
($error == null): true

‎e2e/test/lb-update-port/chainsaw-test.yaml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,11 @@ spec:
4141
- script:
4242
content: |
4343
set -euo pipefail
44-
sleep 30
4544
IP=$(kubectl get svc svc-test -n $NAMESPACE -o json | jq -r .status.loadBalancer.ingress[0].ip)
4645
4746
podnames=()
4847
49-
for i in {1..10}; do
48+
for i in {1..20}; do
5049
if [[ ${#podnames[@]} -lt 2 ]]; then
5150
output=$(curl -s $IP:80 | jq -e .podName || true)
5251
@@ -86,12 +85,11 @@ spec:
8685
content: |
8786
set -euo pipefail
8887
#wait for changes to propagate to the LB
89-
sleep 60
9088
IP=$(kubectl get svc svc-test -n $NAMESPACE -o json | jq -r .status.loadBalancer.ingress[0].ip)
9189
9290
podnames=()
9391
94-
for i in {1..20}; do
92+
for i in {1..30}; do
9593
if [[ ${#podnames[@]} -lt 2 ]]; then
9694
output=$(curl -s $IP:8080 | jq -e .podName || true)
9795

0 commit comments

Comments
 (0)