Skip to content

Commit a100759

Browse files
teoincontattoteoincontatto
committed
Merge branch '2956-primary-pod-does-not-start-when-restarted-after-a-restore-from-snapshot-backup' into 'main-1.14'
Resolve "Primary Pod does not start when restarted after a restore from snapshot backup" and others See merge request ongresinc/stackgres!1636
2 parents e4bf5d6 + 7ed3661 commit a100759

7 files changed

Lines changed: 265 additions & 90 deletions

File tree

‎stackgres-k8s/e2e/spec/replication‎

Lines changed: 87 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,15 @@ metadata:
3030
spec:
3131
sgCluster: $CLUSTER_NAME
3232
EOF
33-
wait_until eval 'kubectl get sgbackup -n "$CLUSTER_NAMESPACE" "$OLD_BACKUP_NAME" -o json \
34-
| jq ".status.process.status" -r | grep -qxF Completed'
33+
34+
wait_until is_backup_phase "$CLUSTER_NAMESPACE" "$OLD_BACKUP_NAME" "Completed"
35+
}
36+
37+
is_backup_phase() {
38+
local NAMESPACE="$1"
39+
local NAME="$2"
40+
local STATUS="$3"
41+
[ "$(kubectl get sgbackup -n "$NAMESPACE" "$NAME" -o=jsonpath='{.status.process.status}')" = "$STATUS" ]
3542
}
3643

3744
e2e_test() {
@@ -41,6 +48,13 @@ e2e_test() {
4148

4249
run_test "Checking that replication is working from backup" check_replication_is_working_from_backup
4350

51+
if [ "$K8S_DISABLE_VOLUME_SNAPSHOT" != true ]
52+
then
53+
run_test "Checking that replication is working from snapshot backup" check_replication_is_working_from_snapshot_backup
54+
else
55+
echo "Skippping Checking that replication is working from snapshot backup"
56+
fi
57+
4458
run_test "Checking that replication is working from newly created backup" check_replication_is_working_from_newly_created_backup
4559

4660
run_test "Checking that metrics are exported" check_metrics
@@ -150,8 +164,75 @@ metadata:
150164
spec:
151165
sgCluster: $CLUSTER_NAME
152166
EOF
153-
wait_until eval 'kubectl get sgbackup -n "$CLUSTER_NAMESPACE" "$BACKUP_NAME" -o json \
154-
| jq ".status.process.status" -r | grep -qxF Completed'
167+
168+
wait_until is_backup_phase "$CLUSTER_NAMESPACE" "$BACKUP_NAME" "Completed"
169+
170+
create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
171+
--set-string cluster.replication.initialization.mode=FromExistingBackup
172+
wait_pods_terminated "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"
173+
wait_until kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
174+
| jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
175+
| grep -q .
176+
177+
kubectl delete --ignore-not-found -n "$CLUSTER_NAMESPACE" pvc/"$CLUSTER_NAME-data-$CLUSTER_NAME-1" pod/"$CLUSTER_NAME-1" --wait=false
178+
wait_pods_running "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"
179+
180+
PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
181+
try_function wait_until eval '
182+
REPLICA_RESPONSE="$(run_query -p 5432 -i 0 -h "$CLUSTER_NAME-replicas" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
183+
[ "$(echo "$PRIMARY_RESPONSE" | tr -d "\n")" = "$(echo "$REPLICA_RESPONSE" | tr -d "\n")" ]
184+
'
185+
if "$RESULT"
186+
then
187+
success "replication is working"
188+
else
189+
fail "replication is not working. The records don't match between primary and replica for the fibonacci table"
190+
fi
191+
192+
if kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep -q "^[^ ]\+ [^ ]\+ INFO: replica has been created using backup$"
193+
then
194+
success "replication has bootstrapped from backup"
195+
else
196+
kubectl logs -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-1" -c patroni | grep "^[^ ]\+ [^ ]\+ INFO: replica has been created using" || true
197+
echo
198+
fail "replication has not bootstrapped from backup"
199+
fi
200+
201+
LATEST_BACKUP="$(kubectl get sgbackup -n "$CLUSTER_NAMESPACE" --sort-by='{.metadata.creationTimestamp}' -o name \
202+
| tail -n 1 | cut -d / -f 2)"
203+
if [ "x$LATEST_BACKUP" != x ] \
204+
&& kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
205+
| jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
206+
| grep -qxF "$LATEST_BACKUP"
207+
then
208+
success "replication has bootstrapped from latest backup"
209+
else
210+
fail "replication has not bootstrapped from latest backup"
211+
fi
212+
}
213+
214+
check_replication_is_working_from_snapshot_backup() {
215+
create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" 1 \
216+
--set cluster.configurations.backups.useVolumeSnapshot=true
217+
218+
wait_pods_terminated "$CLUSTER_NAMESPACE" "1" "$CLUSTER_NAME-[0-9]"
219+
220+
wait_until eval 'kubectl get cronjob -n "$CLUSTER_NAMESPACE" "$BACKUP_CLUSTER_NAME" -o json \
221+
| jq ".spec.template.spec.containers[0].env | any(.name == \"USE_VOLUME_SNAPSHOT\" and .value == \"true\")"'
222+
223+
kubectl delete sgbackup -n "$CLUSTER_NAMESPACE" "$BACKUP_NAME"
224+
225+
cat << EOF | kubectl create -f -
226+
apiVersion: stackgres.io/v1
227+
kind: SGBackup
228+
metadata:
229+
namespace: $CLUSTER_NAMESPACE
230+
name: $BACKUP_NAME
231+
spec:
232+
sgCluster: $CLUSTER_NAME
233+
EOF
234+
235+
wait_until is_backup_phase "$CLUSTER_NAMESPACE" "$BACKUP_NAME" "Completed"
155236

156237
create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
157238
--set-string cluster.replication.initialization.mode=FromExistingBackup
@@ -160,7 +241,7 @@ EOF
160241
| jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
161242
| grep -q .
162243

163-
kubectl delete -n "$CLUSTER_NAMESPACE" pvc/"$CLUSTER_NAME-data-$CLUSTER_NAME-1" pod/"$CLUSTER_NAME-1" --wait=false
244+
kubectl delete --ignore-not-found -n "$CLUSTER_NAMESPACE" pvc/"$CLUSTER_NAME-data-$CLUSTER_NAME-1" pod/"$CLUSTER_NAME-1" --wait=false
164245
wait_pods_running "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"
165246

166247
PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"
@@ -212,7 +293,7 @@ check_replication_is_working_from_newly_created_backup() {
212293
--set-string cluster.replication.initialization.mode=FromNewlyCreatedBackup \
213294
--set-string cluster.replication.initialization.backupNewerThan="PT$(( NOW - BACKUP_END_TIMESTAMP ))S"
214295

215-
kubectl delete -n "$CLUSTER_NAMESPACE" pvc/"$CLUSTER_NAME-data-$CLUSTER_NAME-1" pod/"$CLUSTER_NAME-1" --wait=false
296+
kubectl delete --ignore-not-found -n "$CLUSTER_NAMESPACE" pvc/"$CLUSTER_NAME-data-$CLUSTER_NAME-1" pod/"$CLUSTER_NAME-1" --wait=false
216297
wait_pods_running "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"
217298

218299
PRIMARY_RESPONSE="$(run_query -p 5432 -i 1 -h "$CLUSTER_NAME" -q "SELECT num FROM fibonacci ORDER BY num;" -d "test")"

‎stackgres-k8s/e2e/spec/restore‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,14 +234,27 @@ check_volume_snapshot_backup_can_be_restored() {
234234
--set configurations.create=false --set instanceProfiles=null
235235

236236
wait_pods_running "$CLUSTER_NAMESPACE" 4
237-
wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
237+
SWITCHOVER_TO_FIRST=true wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
238238

239239
check_init_data_after_restore
240240

241241
check_replica_data_after_restore
242242

243243
check_replication_after_restore
244244

245+
create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" 1 \
246+
--set cluster.initialData.restore.fromBackup.name="$BACKUP_VOLUME_SNAPSHOT_NAME" \
247+
--set-string "cluster.postgres.version=$E2E_POSTGRES_VERSION" \
248+
--set configurations.create=false --set instanceProfiles=null
249+
250+
wait_pods_terminated "$CLUSTER_NAMESPACE" 3
251+
252+
kubectl delete pod -n "$CLUSTER_NAMESPACE" -l app=StackGresCluster,stackgres.io/cluster-name="$CLUSTER_NAME",stackgres.io/cluster=true
253+
254+
wait_pods_running "$CLUSTER_NAMESPACE" 3
255+
256+
wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
257+
245258
remove_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
246259
}
247260

‎stackgres-k8s/src/operator/src/main/java/io/stackgres/operator/conciliation/cluster/ClusterRequiredResourcesGenerator.java‎

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -441,30 +441,44 @@ private Optional<StackGresBackup> getReplicationInitializationBackupToCreate(
441441
labelFactory.replicationInitializationBackupLabels(cluster)))
442442
.filter(backup -> backup.getSpec().getSgCluster().equals(
443443
cluster.getMetadata().getName()))
444-
.filter(backup -> Optional.ofNullable(backup.getStatus())
445-
.map(StackGresBackupStatus::getSgBackupConfig)
446-
.map(StackGresBackupConfigSpec::getStorage)
447-
.equals(backupObjectStorage.map(StackGresObjectStorage::getSpec)))
448-
.filter(backup -> Optional.ofNullable(backup.getStatus())
449-
.map(StackGresBackupStatus::getBackupPath)
450-
.equals(Optional
451-
.ofNullable(cluster.getSpec().getConfigurations().getBackups())
452-
.map(Collection::stream)
453-
.flatMap(Stream::findFirst)
454-
.map(StackGresClusterBackupConfiguration::getPath)))
455-
.filter(backup -> Optional.ofNullable(backup.getStatus())
444+
.filter(backup -> backup.getStatus() == null
445+
|| Optional.ofNullable(backup.getStatus())
446+
.filter(status -> Optional.of(status)
447+
.map(StackGresBackupStatus::getProcess)
448+
.map(StackGresBackupProcess::getStatus)
449+
.filter(Predicate.not(BackupStatus.COMPLETED.toString()::equals)
450+
.and(Predicate.not(BackupStatus.FAILED.toString()::equals)))
451+
.isPresent())
452+
.isPresent()
453+
|| Optional.ofNullable(backup.getStatus())
454+
.filter(status -> Optional.of(status)
455+
.map(StackGresBackupStatus::getSgBackupConfig)
456+
.map(StackGresBackupConfigSpec::getStorage)
457+
.equals(backupObjectStorage.map(StackGresObjectStorage::getSpec)))
458+
.filter(status -> Optional.of(status)
459+
.map(StackGresBackupStatus::getBackupPath)
460+
.equals(Optional
461+
.ofNullable(cluster.getSpec().getConfigurations().getBackups())
462+
.map(Collection::stream)
463+
.flatMap(Stream::findFirst)
464+
.map(StackGresClusterBackupConfiguration::getPath)))
465+
.filter(status -> Optional.of(status)
466+
.map(StackGresBackupStatus::getBackupInformation)
467+
.map(StackGresBackupInformation::getPostgresMajorVersion)
468+
.filter(postgresMajorVersion::equals)
469+
.isPresent())
470+
.filter(status -> Optional.of(status)
471+
.map(StackGresBackupStatus::getProcess)
472+
.map(StackGresBackupProcess::getStatus)
473+
.filter(BackupStatus.COMPLETED.toString()::equals)
474+
.isPresent())
456475
.map(StackGresBackupStatus::getProcess)
457476
.map(StackGresBackupProcess::getTiming)
458477
.map(StackGresBackupTiming::getEnd)
459478
.map(Instant::parse)
460479
.or(() -> Optional.of(now))
461480
.filter(end -> backupNewerThan.map(end::isAfter).orElse(true))
462481
.isPresent())
463-
.filter(backup -> Optional.ofNullable(backup.getStatus())
464-
.map(StackGresBackupStatus::getBackupInformation)
465-
.map(StackGresBackupInformation::getPostgresMajorVersion)
466-
.filter(postgresMajorVersion::equals)
467-
.isPresent())
468482
.findFirst();
469483
}
470484

‎stackgres-k8s/src/operator/src/main/java/io/stackgres/operator/conciliation/config/ConfigRequiredResourcesGenerator.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -203,13 +203,13 @@ public List<PrometheusContext> getPrometheus(StackGresConfig config) {
203203
.stream()
204204
.flatMap(List::stream)
205205
.filter(prometheus -> monitors.stream()
206-
.anyMatch(monitor -> monitor.getMetadata().getNamespace().equals(prometheus.getMetadata().getNamespace())
207-
&& monitor.getMetadata().getName().equals(prometheus.getMetadata().getName())))
206+
.anyMatch(monitor -> monitor.getNamespace().equals(prometheus.getMetadata().getNamespace())
207+
&& monitor.getName().equals(prometheus.getMetadata().getName())))
208208
.map(prometheus -> PrometheusContext.toPrometheusContext(
209209
prometheus,
210210
monitors.stream()
211-
.filter(monitor -> monitor.getMetadata().getNamespace().equals(prometheus.getMetadata().getNamespace())
212-
&& monitor.getMetadata().getName().equals(prometheus.getMetadata().getName()))
211+
.filter(monitor -> monitor.getNamespace().equals(prometheus.getMetadata().getNamespace())
212+
&& monitor.getName().equals(prometheus.getMetadata().getName()))
213213
.findFirst()
214214
.orElseThrow()))
215215
.toList();

‎stackgres-k8s/src/operator/src/main/resources/templates/create-backup.sh‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,7 @@ do_backup() {
381381
-- psql -v ON_ERROR_STOP=1 -t -A > /tmp/backup-psql-out 2>&1 &
382382
echo $! > /tmp/backup-psql-pid
383383

384+
echo "Starting backup"
384385
cat << EOF >> /tmp/backup-psql
385386
SELECT 'Backup bootstrap';
386387
EOF
@@ -392,9 +393,9 @@ EOF
392393
cat /tmp/backup-psql-out
393394
if ! kill -0 "$(cat /tmp/backup-psql-pid)"
394395
then
395-
echo 'Backup failed while connection to primary'
396+
echo 'Backup failed while connecting to primary'
396397
retry kubectl patch "$BACKUP_CRD_NAME" -n "$CLUSTER_NAMESPACE" "$BACKUP_NAME" --type json --patch '[
397-
{"op":"replace","path":"/status/process/failure","value":'"$({ printf 'Backup failed while connection to primary:\n'; cat /tmp/backup-psql-out; } | to_json_string)"'}
398+
{"op":"replace","path":"/status/process/failure","value":'"$({ printf 'Backup failed while connecting to primary:\n'; cat /tmp/backup-psql-out; } | to_json_string)"'}
398399
]'
399400
kill "$(cat /tmp/backup-tail-pid)" || true
400401
exit 1
@@ -455,6 +456,20 @@ spec:
455456
source:
456457
persistentVolumeClaimName: $(cat /tmp/current-primary-pvc)
457458
EOF
459+
echo "Performing cleanup of backup restored files before creating the VolumeSnapshot"
460+
if ! retry kubectl exec -n "$CLUSTER_NAMESPACE" "$(cat /tmp/current-primary)" -c patroni -- \
461+
rm -rf "$PG_DATA_PATH.backup" "$PG_DATA_PATH/.restored_from_volume_snapshot" > /tmp/backup-cleanup 2>&1
462+
then
463+
cat /tmp/backup-cleanup
464+
echo 'Backup failed while cleaning up before snapshot'
465+
retry kubectl patch "$BACKUP_CRD_NAME" -n "$CLUSTER_NAMESPACE" "$BACKUP_NAME" --type json --patch '[
466+
{"op":"replace","path":"/status/process/failure","value":'"$({ printf 'Backup failed while cleaning up before snapshot:\n'; cat /tmp/backup-cleanup; } | to_json_string)"'}
467+
]'
468+
kill "$(cat /tmp/backup-tail-pid)" || true
469+
exit 1
470+
fi
471+
echo "Creating VolumeSnapshot"
472+
cat /tmp/backup-cleanup
458473
if ! retry kubectl create -f /tmp/snapshot-to-create > /tmp/backup-snapshot 2>&1
459474
then
460475
cat /tmp/backup-snapshot
@@ -509,6 +524,7 @@ EOF
509524
sleep 1
510525
done
511526

527+
echo "Stopping backup"
512528
cat << EOF >> /tmp/backup-psql
513529
SET client_min_messages TO WARNING;
514530
SELECT

0 commit comments

Comments
 (0)