Skip to content

Commit 9a90088

Browse files
teoincontattoteoincontatto
committed
e2e: make ssl more resilient
1 parent f4bf603 commit 9a90088

3 files changed

Lines changed: 122 additions & 120 deletions

File tree

  • stackgres-k8s/e2e/spec

‎stackgres-k8s/e2e/spec/previous/1.14/spec/ssl‎

Lines changed: 39 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -39,34 +39,7 @@ generated_ssl_check() {
3939

4040
wait_until kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c patroni -- ls /etc/ssl/tls.key
4141

42-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
43-
then
44-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
45-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
46-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
47-
fi
48-
49-
RESPONSE_PRIMARY="$(wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432)"
50-
51-
if [ "$RESPONSE_PRIMARY" = "1" ]
52-
then
53-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
54-
then
55-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
56-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c postgres-util -- \
57-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
58-
fi
59-
60-
RESPONSE_REPLICA="$(run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432)"
61-
if [ "$RESPONSE_REPLICA" = "1" ]
62-
then
63-
success "Connections are possible using SSL"
64-
else
65-
fail "Cannot connect to replica db using SSL"
66-
fi
67-
else
68-
fail "Cannot connect to primary db using SSL"
69-
fi
42+
ssl_check
7043
}
7144

7245
provided_ssl_check() {
@@ -84,23 +57,49 @@ provided_ssl_check() {
8457
wait_until eval '[ "$(kubectl get secret -n "$CLUSTER_NAME" ssl-provided --template "{{ .data \"tls.key\" | base64decode }}")" = \
8558
"$(kubectl exec -n "$CLUSTER_NAME" "$CLUSTER_NAME-0" -c patroni -- cat /etc/ssl/tls.key)" ]'
8659

87-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432 >/dev/null 2>&1
88-
then
89-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
90-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
91-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
92-
fi
93-
94-
RESPONSE_PRIMARY="$(wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432)"
60+
ssl_check
61+
}
9562

96-
if [ "$RESPONSE_PRIMARY" = "1" ]
97-
then
98-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
63+
ssl_check() {
64+
local RETRY=3
65+
local EXIT_CODE RESULT
66+
until [ "$RETRY" -gt 0 ]
67+
do
68+
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
9969
then
10070
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
101-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c postgres-util -- \
71+
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
10272
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
10373
fi
74+
try_function run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
75+
if "$RESULT"
76+
then
77+
break
78+
fi
79+
RETRY="$((RETRY - 1))"
80+
done
81+
82+
RESPONSE_PRIMARY="$(wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432)"
83+
84+
if [ "$RESPONSE_PRIMARY" = "1" ]
85+
then
86+
local RETRY=3
87+
local EXIT_CODE RESULT
88+
until [ "$RETRY" -gt 0 ]
89+
do
90+
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
91+
then
92+
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
93+
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c postgres-util -- \
94+
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
95+
fi
96+
try_function run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
97+
if "$RESULT"
98+
then
99+
break
100+
fi
101+
RETRY="$((RETRY - 1))"
102+
done
104103

105104
RESPONSE_REPLICA="$(wait_until run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432)"
106105
if [ "$RESPONSE_REPLICA" = "1" ]

‎stackgres-k8s/e2e/spec/previous/1.15/spec/ssl‎

Lines changed: 39 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -39,34 +39,7 @@ generated_ssl_check() {
3939

4040
wait_until kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c patroni -- ls /etc/ssl/tls.key
4141

42-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
43-
then
44-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
45-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
46-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
47-
fi
48-
49-
RESPONSE_PRIMARY="$(wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432)"
50-
51-
if [ "$RESPONSE_PRIMARY" = "1" ]
52-
then
53-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
54-
then
55-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
56-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c postgres-util -- \
57-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
58-
fi
59-
60-
RESPONSE_REPLICA="$(run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432)"
61-
if [ "$RESPONSE_REPLICA" = "1" ]
62-
then
63-
success "Connections are possible using SSL"
64-
else
65-
fail "Cannot connect to replica db using SSL"
66-
fi
67-
else
68-
fail "Cannot connect to primary db using SSL"
69-
fi
42+
ssl_check
7043
}
7144

7245
provided_ssl_check() {
@@ -84,23 +57,49 @@ provided_ssl_check() {
8457
wait_until eval '[ "$(kubectl get secret -n "$CLUSTER_NAME" ssl-provided --template "{{ .data \"tls.key\" | base64decode }}")" = \
8558
"$(kubectl exec -n "$CLUSTER_NAME" "$CLUSTER_NAME-0" -c patroni -- cat /etc/ssl/tls.key)" ]'
8659

87-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432 >/dev/null 2>&1
88-
then
89-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
90-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
91-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
92-
fi
93-
94-
RESPONSE_PRIMARY="$(wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432)"
60+
ssl_check
61+
}
9562

96-
if [ "$RESPONSE_PRIMARY" = "1" ]
97-
then
98-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
63+
ssl_check() {
64+
local RETRY=3
65+
local EXIT_CODE RESULT
66+
until [ "$RETRY" -gt 0 ]
67+
do
68+
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
9969
then
10070
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
101-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c postgres-util -- \
71+
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
10272
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
10373
fi
74+
try_function run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
75+
if "$RESULT"
76+
then
77+
break
78+
fi
79+
RETRY="$((RETRY - 1))"
80+
done
81+
82+
RESPONSE_PRIMARY="$(wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432)"
83+
84+
if [ "$RESPONSE_PRIMARY" = "1" ]
85+
then
86+
local RETRY=3
87+
local EXIT_CODE RESULT
88+
until [ "$RETRY" -gt 0 ]
89+
do
90+
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
91+
then
92+
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
93+
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c postgres-util -- \
94+
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
95+
fi
96+
try_function run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
97+
if "$RESULT"
98+
then
99+
break
100+
fi
101+
RETRY="$((RETRY - 1))"
102+
done
104103

105104
RESPONSE_REPLICA="$(wait_until run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432)"
106105
if [ "$RESPONSE_REPLICA" = "1" ]

‎stackgres-k8s/e2e/spec/ssl‎

Lines changed: 44 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ e2e_test_install() {
1919

2020
wait_pods_running "$CLUSTER_NAMESPACE" 2
2121
wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
22+
switch_cluster_to_first "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
2223
}
2324

2425
e2e_test() {
@@ -39,34 +40,7 @@ generated_ssl_check() {
3940

4041
wait_until kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c patroni -- ls /etc/ssl/tls.key
4142

42-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
43-
then
44-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
45-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
46-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
47-
fi
48-
49-
RESPONSE_PRIMARY="$(wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432)"
50-
51-
if [ "$RESPONSE_PRIMARY" = "1" ]
52-
then
53-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
54-
then
55-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
56-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c postgres-util -- \
57-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
58-
fi
59-
60-
RESPONSE_REPLICA="$(run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432)"
61-
if [ "$RESPONSE_REPLICA" = "1" ]
62-
then
63-
success "Connections are possible using SSL"
64-
else
65-
fail "Cannot connect to replica db using SSL"
66-
fi
67-
else
68-
fail "Cannot connect to primary db using SSL"
69-
fi
43+
ssl_check
7044
}
7145

7246
provided_ssl_check() {
@@ -84,23 +58,53 @@ provided_ssl_check() {
8458
wait_until eval '[ "$(kubectl get secret -n "$CLUSTER_NAME" ssl-provided --template "{{ .data \"tls.key\" | base64decode }}")" = \
8559
"$(kubectl exec -n "$CLUSTER_NAME" "$CLUSTER_NAME-0" -c patroni -- cat /etc/ssl/tls.key)" ]'
8660

87-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432 >/dev/null 2>&1
88-
then
89-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
90-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
91-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
92-
fi
61+
ssl_check
62+
}
63+
64+
ssl_check() {
65+
local RETRY=3
66+
local EXIT_CODE RESULT
67+
while [ "$RETRY" -gt 0 ]
68+
do
69+
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
70+
then
71+
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
72+
kubectl delete pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0"
73+
wait_pods_running "$CLUSTER_NAMESPACE" 2
74+
wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
75+
switch_cluster_to_first "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
76+
fi
77+
try_function wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432
78+
if "$RESULT"
79+
then
80+
break
81+
fi
82+
RETRY="$((RETRY - 1))"
83+
done
9384

9485
RESPONSE_PRIMARY="$(wait_until run_query -m require -h "$CLUSTER_NAME" -i 1 -p 5432)"
9586

9687
if [ "$RESPONSE_PRIMARY" = "1" ]
9788
then
98-
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
99-
then
100-
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
101-
kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c postgres-util -- \
102-
sh -c "ps -ef | grep '[e]nvoy' | tr -s ' ' | cut -d ' ' -f 2 | xargs -I % sh -c 'kill %'"
103-
fi
89+
local RETRY=3
90+
local EXIT_CODE RESULT
91+
while [ "$RETRY" -gt 0 ]
92+
do
93+
if ! wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
94+
then
95+
echo "WARNING: SSL was not updated automatically... seems an envoy bug, will restart it"
96+
kubectl delete pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1"
97+
wait_pods_running "$CLUSTER_NAMESPACE" 2
98+
wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
99+
switch_cluster_to_first "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
100+
fi
101+
try_function wait_until -t "$((E2E_TIMEOUT / 4))" run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432
102+
if "$RESULT"
103+
then
104+
break
105+
fi
106+
RETRY="$((RETRY - 1))"
107+
done
104108

105109
RESPONSE_REPLICA="$(wait_until run_query -m require -h "$CLUSTER_NAME"-replicas -i 0 -p 5432)"
106110
if [ "$RESPONSE_REPLICA" = "1" ]

0 commit comments

Comments
 (0)