Skip to content

Commit fec2ec6

Browse files
tiationg-khotiationg-kho
authored
Fix e2e tests failure in action of build and test (#1094)
* Fix e2e tests failure in action of build and test
1 parent 9517b9d commit fec2ec6

3 files changed

Lines changed: 89 additions & 79 deletions

File tree

‎test/e2e/emit-events-test‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,25 @@ common_helm_args=()
2525
[[ "${TEST_WINDOWS-}" == "true" ]] && common_helm_args+=(--set targetNodeOs="windows")
2626
[[ -n "${NTH_WORKER_LABEL-}" ]] && common_helm_args+=(--set nodeSelector."$NTH_WORKER_LABEL")
2727

28+
aemm_helm_args=(
29+
upgrade
30+
--install
31+
--namespace default
32+
"$CLUSTER_NAME-aemm"
33+
"$AEMM_DL_URL"
34+
--set aemm.IMDSv2="true"
35+
--set servicePort="$IMDS_PORT"
36+
--wait
37+
)
38+
[[ ${#common_helm_args[@]} -gt 0 ]] &&
39+
aemm_helm_args+=("${common_helm_args[@]}")
40+
41+
set -x
42+
retry 5 helm "${aemm_helm_args[@]}"
43+
set +x
44+
45+
sleep 5
46+
2847
anth_helm_args=(
2948
upgrade
3049
--install
@@ -68,23 +87,6 @@ set -x
6887
helm "${emtp_helm_args[@]}"
6988
set +x
7089

71-
aemm_helm_args=(
72-
upgrade
73-
--install
74-
--namespace default
75-
"$CLUSTER_NAME-aemm"
76-
"$AEMM_DL_URL"
77-
--set aemm.IMDSv2="true"
78-
--set servicePort="$IMDS_PORT"
79-
--wait
80-
)
81-
[[ ${#common_helm_args[@]} -gt 0 ]] &&
82-
aemm_helm_args+=("${common_helm_args[@]}")
83-
84-
set -x
85-
retry 5 helm "${aemm_helm_args[@]}"
86-
set +x
87-
8890
TAINT_CHECK_CYCLES=15
8991
TAINT_CHECK_SLEEP=15
9092

@@ -108,8 +110,8 @@ fi
108110
test_node="${TEST_NODE:-$CLUSTER_NAME-worker}"
109111
for i in $(seq 1 $TAINT_CHECK_CYCLES); do
110112
if kubectl get events | tr -s " " | grep "CordonAndDrain node/${test_node} Node successfully cordoned and drained" >/dev/null; then
111-
echo "✅ Verified CordonAndDrain was emitted as a k8s event!"
112-
echo "✅ K8s Emit Events Test Passed $CLUSTER_NAME! ✅"s
113+
echo "✅ Verified CordonAndDrain was emitted as a k8s event! (success event)"
114+
echo "✅ K8s Emit Events Test Passed $CLUSTER_NAME! ✅"
113115
exit 0
114116
fi
115117

‎test/e2e/prometheus-metrics-test‎

Lines changed: 38 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,26 @@ helm repo add prometheus-community https://prometheus-community.github.io/helm-c
2525
helm repo update
2626
retry 5 helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack --version ${PROMETHEUS_HELM_VERSION} --set prometheusOperator.admissionWebhooks.enabled="false" --set grafana.enabled="false" --set nodeExporter.enabled="false" --set kubeStateMetrics.enabled="false"
2727

28+
aemm_helm_args=(
29+
upgrade
30+
--install
31+
--namespace default
32+
"$CLUSTER_NAME-aemm"
33+
"$AEMM_DL_URL"
34+
--set servicePort="$IMDS_PORT"
35+
--set 'tolerations[0].effect=NoSchedule'
36+
--set 'tolerations[0].operator=Exists'
37+
--wait
38+
)
39+
[[ ${#common_helm_args[@]} -gt 0 ]] &&
40+
aemm_helm_args+=("${common_helm_args[@]}")
41+
42+
set -x
43+
retry 5 helm "${aemm_helm_args[@]}"
44+
set +x
45+
46+
sleep 5
47+
2848
anth_helm_args=(
2949
upgrade
3050
--install
@@ -71,24 +91,6 @@ set -x
7191
helm "${emtp_helm_args[@]}"
7292
set +x
7393

74-
aemm_helm_args=(
75-
upgrade
76-
--install
77-
--namespace default
78-
"$CLUSTER_NAME-aemm"
79-
"$AEMM_DL_URL"
80-
--set servicePort="$IMDS_PORT"
81-
--set 'tolerations[0].effect=NoSchedule'
82-
--set 'tolerations[0].operator=Exists'
83-
--wait
84-
)
85-
[[ ${#common_helm_args[@]} -gt 0 ]] &&
86-
aemm_helm_args+=("${common_helm_args[@]}")
87-
88-
set -x
89-
retry 5 helm "${aemm_helm_args[@]}"
90-
set +x
91-
9294
TAINT_CHECK_CYCLES=15
9395
TAINT_CHECK_SLEEP=15
9496

@@ -109,25 +111,25 @@ fi
109111

110112

111113
EXIT_STATUS=1
114+
cordoned=0
115+
tainted=0
112116
for i in $(seq 1 $TAINT_CHECK_CYCLES); do
113-
if kubectl get nodes "${CLUSTER_NAME}-worker" | grep SchedulingDisabled; then
114-
echo "✅ Verified the worker node was cordoned!"
115-
116-
if kubectl get nodes "${CLUSTER_NAME}-worker" -o json | grep -q "aws-node-termination-handler/spot-itn"; then
117-
echo "✅ Verified the worked node was tainted!"
118-
else
119-
echo "❌ Failed tainting node for spot termination event"
120-
EXIT_STATUS=3
121-
break
122-
fi
123-
124-
if [[ $(kubectl get deployments regular-pod-test -o=jsonpath='{.status.unavailableReplicas}') -eq 1 ]]; then
125-
echo "✅ Verified the regular-pod-test pod was evicted!"
126-
echo "✅ Spot Interruption Test Passed $CLUSTER_NAME! ✅"
127-
EXIT_STATUS=0
128-
break
129-
fi
130-
fi
117+
if [[ $cordoned -eq 0 ]] && kubectl get nodes "${CLUSTER_NAME}-worker" | grep SchedulingDisabled; then
118+
echo "✅ Verified the worker node was cordoned!"
119+
cordoned=1
120+
fi
121+
122+
if [[ $cordoned -eq 1 && $tainted -eq 0 ]] && kubectl get nodes "${CLUSTER_NAME}-worker" -o json | grep -q "aws-node-termination-handler/spot-itn"; then
123+
echo "✅ Verified the worked node was tainted!"
124+
tainted=1
125+
fi
126+
127+
if [[ $tainted -eq 1 && $(kubectl get deployments regular-pod-test -o=jsonpath='{.status.unavailableReplicas}') -eq 1 ]]; then
128+
echo "✅ Verified the regular-pod-test pod was evicted!"
129+
echo "✅ Spot Interruption Test Passed $CLUSTER_NAME! ✅"
130+
EXIT_STATUS=0
131+
break
132+
fi
131133
echo "Assertion Loop $i/$TAINT_CHECK_CYCLES, sleeping for $TAINT_CHECK_SLEEP seconds"
132134
sleep $TAINT_CHECK_SLEEP
133135
done

‎test/e2e/webhook-test‎

Lines changed: 30 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,32 +25,24 @@ common_helm_args=()
2525
[[ "${TEST_WINDOWS-}" == "true" ]] && common_helm_args+=(--set targetNodeOs="windows")
2626
[[ -n "${NTH_WORKER_LABEL-}" ]] && common_helm_args+=(--set nodeSelector."$NTH_WORKER_LABEL")
2727

28-
anth_helm_args=(
28+
aemm_helm_args=(
2929
upgrade
3030
--install
31-
--namespace kube-system
32-
"$CLUSTER_NAME-anth"
33-
"$SCRIPTPATH/../../config/helm/aws-node-termination-handler/"
34-
--set instanceMetadataURL="${INSTANCE_METADATA_URL:-"http://$AEMM_URL:$IMDS_PORT"}"
35-
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO"
36-
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG"
37-
--set webhookURL="${WEBHOOK_URL}"
38-
--set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - Node: \{\{ \.NodeName \}\} - InstanceType: \{\{ \.InstanceType \}\} - AvailabilityZone: \{\{ \.AvailabilityZone \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\}\"\}"
39-
--set enableSpotInterruptionDraining="true"
40-
--set enableScheduledEventDraining="true"
41-
--set daemonsetTolerations=""
31+
--namespace default
32+
"$CLUSTER_NAME-aemm"
33+
"$AEMM_DL_URL"
34+
--set servicePort="$IMDS_PORT"
4235
--wait
43-
--force
4436
)
45-
[[ -n "${NODE_TERMINATION_HANDLER_DOCKER_PULL_POLICY-}" ]] &&
46-
anth_helm_args+=(--set image.pullPolicy="$NODE_TERMINATION_HANDLER_DOCKER_PULL_POLICY")
4737
[[ ${#common_helm_args[@]} -gt 0 ]] &&
48-
anth_helm_args+=("${common_helm_args[@]}")
38+
aemm_helm_args+=("${common_helm_args[@]}")
4939

5040
set -x
51-
helm "${anth_helm_args[@]}"
41+
retry 5 helm "${aemm_helm_args[@]}"
5242
set +x
5343

44+
sleep 5
45+
5446
emtp_helm_args=(
5547
upgrade
5648
--install
@@ -70,22 +62,36 @@ set -x
7062
helm "${emtp_helm_args[@]}"
7163
set +x
7264

73-
aemm_helm_args=(
65+
sleep 5
66+
67+
anth_helm_args=(
7468
upgrade
7569
--install
76-
--namespace default
77-
"$CLUSTER_NAME-aemm"
78-
"$AEMM_DL_URL"
79-
--set servicePort="$IMDS_PORT"
70+
--namespace kube-system
71+
"$CLUSTER_NAME-anth"
72+
"$SCRIPTPATH/../../config/helm/aws-node-termination-handler/"
73+
--set instanceMetadataURL="${INSTANCE_METADATA_URL:-"http://$AEMM_URL:$IMDS_PORT"}"
74+
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO"
75+
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG"
76+
--set webhookURL="${WEBHOOK_URL}"
77+
--set webhookTemplate="\{\"Content\":\"[NTH][Instance Interruption] InstanceId: \{\{ \.InstanceID \}\} - Node: \{\{ \.NodeName \}\} - InstanceType: \{\{ \.InstanceType \}\} - AvailabilityZone: \{\{ \.AvailabilityZone \}\} - Kind: \{\{ \.Kind \}\} - Start Time: \{\{ \.StartTime \}\}\"\}"
78+
--set enableSpotInterruptionDraining="true"
79+
--set enableScheduledEventDraining="true"
80+
--set daemonsetTolerations=""
81+
--force
8082
--wait
8183
)
84+
[[ -n "${NODE_TERMINATION_HANDLER_DOCKER_PULL_POLICY-}" ]] &&
85+
anth_helm_args+=(--set image.pullPolicy="$NODE_TERMINATION_HANDLER_DOCKER_PULL_POLICY")
8286
[[ ${#common_helm_args[@]} -gt 0 ]] &&
83-
aemm_helm_args+=("${common_helm_args[@]}")
87+
anth_helm_args+=("${common_helm_args[@]}")
8488

8589
set -x
86-
retry 5 helm "${aemm_helm_args[@]}"
90+
helm "${anth_helm_args[@]}"
8791
set +x
8892

93+
sleep 5
94+
8995
TAINT_CHECK_CYCLES=15
9096
TAINT_CHECK_SLEEP=15
9197

0 commit comments

Comments
 (0)