Skip to content

Commit 0f95a6d

Browse files
teoincontattoteoincontatto
committed
Merge branch '2883-replication-initialization-from-backup-chose-always-the-oldest-backup-first-instead-of-the' into 'main'
Resolve "Replication initialization from backup chose always the oldest backup first instead of the newer one" Closes #2883 and #2884 See merge request ongresinc/stackgres!1613
2 parents ef44388 + e586ac8 commit 0f95a6d

7 files changed

Lines changed: 99 additions & 74 deletions

File tree

‎stackgres-k8s/e2e/spec/replication‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ e2e_test_extra_hash() {
88
}
99

1010
e2e_test_install() {
11+
OLD_BACKUP_NAME="$(get_sgbackup_name "${SPEC_NAME}-old")"
1112
BACKUP_NAME="$(get_sgbackup_name "${SPEC_NAME}")"
1213

1314
install_minio
@@ -19,6 +20,18 @@ e2e_test_install() {
1920

2021
wait_pods_running "$CLUSTER_NAMESPACE" "4"
2122
wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
23+
24+
cat << EOF | kubectl create -f -
25+
apiVersion: stackgres.io/v1
26+
kind: SGBackup
27+
metadata:
28+
namespace: $CLUSTER_NAMESPACE
29+
name: $OLD_BACKUP_NAME
30+
spec:
31+
sgCluster: $CLUSTER_NAME
32+
EOF
33+
wait_until eval 'kubectl get sgbackup -n "$CLUSTER_NAMESPACE" "$OLD_BACKUP_NAME" -o json \
34+
| jq ".status.process.status" -r | grep -qxF Completed'
2235
}
2336

2437
e2e_test() {
@@ -142,7 +155,7 @@ EOF
142155

143156
create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
144157
--set-string cluster.replication.initialization.mode=FromExistingBackup
145-
wait_pods_terminated "$CLUSTER_NAMESPACE" "5"
158+
wait_pods_terminated "$CLUSTER_NAMESPACE" "2" "$CLUSTER_NAME-[0-9]"
146159
wait_until kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
147160
| jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
148161
| grep -q .
@@ -170,13 +183,31 @@ EOF
170183
echo
171184
fail "replication has not bootstrapped from backup"
172185
fi
186+
187+
LATEST_BACKUP="$(kubectl get sgbackup -n "$CLUSTER_NAMESPACE" --sort-by='{.metadata.creationTimestamp}' -o name \
188+
| tail -n 1 | cut -d / -f 2)"
189+
if [ "x$LATEST_BACKUP" != x ] \
190+
&& kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
191+
| jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
192+
| grep -qxF "$LATEST_BACKUP"
193+
then
194+
success "replication has bootstrapped from latest backup"
195+
else
196+
fail "replication has not bootstrapped from latest backup"
197+
fi
173198
}
174199

175200
check_replication_is_working_from_newly_created_backup() {
176201
BACKUP_END_TIMESTAMP="$(kubectl get sgbackup -n "$CLUSTER_NAMESPACE" "$BACKUP_NAME" -o json \
177202
| jq '.status.process.timing.end' -r \
178203
| date -d "$(cat)" +%s)"
179204
NOW="$(date +%s)"
205+
if [ "$(( NOW - BACKUP_END_TIMESTAMP ))" -lt 120 ]
206+
then
207+
echo "Waiting $(( 120 - NOW + BACKUP_END_TIMESTAMP )) seconds for the existing backups to be enough old"
208+
sleep "$(( 120 - NOW + BACKUP_END_TIMESTAMP ))"
209+
NOW="$(date +%s)"
210+
fi
180211
create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "2" \
181212
--set-string cluster.replication.initialization.mode=FromNewlyCreatedBackup \
182213
--set-string cluster.replication.initialization.backupNewerThan="PT$(( NOW - BACKUP_END_TIMESTAMP ))S"
@@ -205,12 +236,15 @@ check_replication_is_working_from_newly_created_backup() {
205236
fail "replication has not bootstrapped from backup"
206237
fi
207238

208-
if kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
239+
NEWLY_CREATED_BACKUP="$(kubectl get sgbackup -n "$CLUSTER_NAMESPACE" -l "app=StackGresCluster,stackgres.io/cluster-name=$CLUSTER_NAME" -o name \
240+
| cut -d '/' -f 2)"
241+
if [ "x$NEWLY_CREATED_BACKUP" != x ] \
242+
&& kubectl get configmap -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-replication-init" -o json \
209243
| jq '.data.REPLICATION_INITIALIZATION_BACKUP' -r \
210-
| grep -qxF "$CLUSTER_NAME"
244+
| grep -qxF "$NEWLY_CREATED_BACKUP"
211245
then
212-
fail "replication has not bootstrapped from newly created backup"
213-
else
214246
success "replication has bootstrapped from newly created backup"
247+
else
248+
fail "replication has not bootstrapped from newly created backup"
215249
fi
216250
}

‎stackgres-k8s/src/common/src/main/java/io/stackgres/common/crd/sgcluster/StackGresClusterPostgresService.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,18 @@ public StackGresClusterPostgresService(Boolean allocateLoadBalancerNodePorts, St
5757
trafficDistribution, type, nodePorts);
5858
}
5959

60+
public StackGresClusterPostgresService(StackGresPostgresService service) {
61+
super(service.getAllocateLoadBalancerNodePorts(), service.getClusterIP(),
62+
service.getClusterIPs(), service.getExternalIPs(), service.getExternalName(),
63+
service.getExternalTrafficPolicy(), service.getHealthCheckNodePort(),
64+
service.getInternalTrafficPolicy(), service.getIpFamilies(), service.getIpFamilyPolicy(),
65+
service.getLoadBalancerClass(), service.getLoadBalancerIP(),
66+
service.getLoadBalancerSourceRanges(), service.getPorts(),
67+
service.getPublishNotReadyAddresses(), service.getSelector(), service.getSessionAffinity(),
68+
service.getSessionAffinityConfig(), service.getTrafficDistribution(), service.getType(),
69+
service.getNodePorts());
70+
}
71+
6072
public List<CustomServicePort> getCustomPorts() {
6173
return customPorts;
6274
}

‎stackgres-k8s/src/operator/src/main/java/io/stackgres/operator/conciliation/cluster/ClusterRequiredResourcesGenerator.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -378,12 +378,13 @@ private Optional<Tuple2<StackGresBackup, Map<String, Secret>>> getReplicationIni
378378
.map(Instant::parse)
379379
.filter(end -> backupNewerThan.map(end::isAfter).orElse(true))
380380
.isPresent())
381-
.sorted(Comparator.comparing(backup -> Optional.ofNullable(backup.getStatus())
381+
.sorted(Comparator.comparing((StackGresBackup backup) -> Optional.ofNullable(backup.getStatus())
382382
.map(StackGresBackupStatus::getProcess)
383383
.map(StackGresBackupProcess::getTiming)
384384
.map(StackGresBackupTiming::getEnd)
385385
.map(Instant::parse)
386-
.get()))
386+
.get())
387+
.reversed())
387388
.map(backup -> Tuple.tuple(
388389
backup,
389390
Optional.of(backup)

‎stackgres-k8s/src/operator/src/main/java/io/stackgres/operator/conciliation/factory/shardedcluster/StackGresShardedClusterForUtil.java‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import io.stackgres.common.crd.sgcluster.StackGresClusterPatroniCredentials;
3030
import io.stackgres.common.crd.sgcluster.StackGresClusterPostgres;
3131
import io.stackgres.common.crd.sgcluster.StackGresClusterPostgresBuilder;
32+
import io.stackgres.common.crd.sgcluster.StackGresClusterPostgresService;
3233
import io.stackgres.common.crd.sgcluster.StackGresClusterPostgresServicesBuilder;
3334
import io.stackgres.common.crd.sgcluster.StackGresClusterResources;
3435
import io.stackgres.common.crd.sgcluster.StackGresClusterRestoreFromBackupBuilder;
@@ -72,7 +73,11 @@ StackGresCluster getCoordinatorCluster(StackGresShardedCluster cluster) {
7273
StackGresShardedClusterUtil.getCoordinatorClusterName(cluster));
7374
var postgresServices = cluster.getSpec().getPostgresServices();
7475
spec.setPostgresServices(new StackGresClusterPostgresServicesBuilder()
75-
.withNewPrimary()
76+
.withPrimary(new StackGresClusterPostgresService(Optional.ofNullable(postgresServices)
77+
.map(StackGresShardedClusterPostgresServices::getCoordinator)
78+
.map(StackGresShardedClusterPostgresCoordinatorServices::getPrimary)
79+
.orElseGet(StackGresPostgresService::new)))
80+
.editPrimary()
7681
.withEnabled(
7782
Optional.ofNullable(postgresServices)
7883
.map(StackGresShardedClusterPostgresServices::getCoordinator)
@@ -85,8 +90,22 @@ StackGresCluster getCoordinatorCluster(StackGresShardedCluster cluster) {
8590
.map(StackGresShardedClusterPostgresCoordinatorServices::getCustomPorts)
8691
.orElse(null))
8792
.endPrimary()
88-
.withNewReplicas()
89-
.withEnabled(false)
93+
.withReplicas(new StackGresClusterPostgresService(Optional.ofNullable(postgresServices)
94+
.map(StackGresShardedClusterPostgresServices::getCoordinator)
95+
.map(StackGresShardedClusterPostgresCoordinatorServices::getAny)
96+
.orElseGet(StackGresPostgresService::new)))
97+
.editReplicas()
98+
.withEnabled(
99+
Optional.ofNullable(postgresServices)
100+
.map(StackGresShardedClusterPostgresServices::getCoordinator)
101+
.map(StackGresShardedClusterPostgresCoordinatorServices::getAny)
102+
.map(StackGresPostgresService::getEnabled)
103+
.orElse(true))
104+
.withCustomPorts(
105+
Optional.ofNullable(postgresServices)
106+
.map(StackGresShardedClusterPostgresServices::getCoordinator)
107+
.map(StackGresShardedClusterPostgresCoordinatorServices::getCustomPorts)
108+
.orElse(null))
90109
.endReplicas()
91110
.build());
92111
coordinatorCluster.setSpec(spec);
@@ -126,7 +145,11 @@ StackGresCluster getShardsCluster(StackGresShardedCluster cluster, int index) {
126145
StackGresShardedClusterUtil.getShardClusterName(cluster, index));
127146
var postgresServices = cluster.getSpec().getPostgresServices();
128147
spec.setPostgresServices(new StackGresClusterPostgresServicesBuilder()
129-
.withNewPrimary()
148+
.withPrimary(new StackGresClusterPostgresService(Optional.ofNullable(postgresServices)
149+
.map(StackGresShardedClusterPostgresServices::getShards)
150+
.map(StackGresShardedClusterPostgresShardsServices::getPrimaries)
151+
.orElseGet(StackGresPostgresService::new)))
152+
.editPrimary()
130153
.withEnabled(
131154
Optional.ofNullable(postgresServices)
132155
.map(StackGresShardedClusterPostgresServices::getShards)

‎stackgres-k8s/src/operator/src/test/java/io/stackgres/operator/conciliation/factory/shardedcluster/StackGresShardedClusterForCitusUtilTest.java‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
import java.util.ArrayList;
1414
import java.util.List;
15-
import java.util.Optional;
1615

1716
import io.stackgres.common.StackGresShardedClusterUtil;
1817
import io.stackgres.common.crd.SecretKeySelector;
@@ -57,7 +56,7 @@ void givedMinimalShardedCluster_shouldGenerateCoordinatorCluster() {
5756
cluster.getSpec().getPostgresServices().getPrimary());
5857
Assertions.assertEquals(
5958
new StackGresClusterPostgresServiceBuilder()
60-
.withEnabled(false)
59+
.withEnabled(true)
6160
.build(),
6261
cluster.getSpec().getPostgresServices().getReplicas());
6362
}
@@ -87,7 +86,7 @@ void givedMinimalShardedClusterPrimaryEnabled_shouldGenerateCoordinatorCluster()
8786
cluster.getSpec().getPostgresServices().getPrimary());
8887
Assertions.assertEquals(
8988
new StackGresClusterPostgresServiceBuilder()
90-
.withEnabled(false)
89+
.withEnabled(true)
9190
.build(),
9291
cluster.getSpec().getPostgresServices().getReplicas());
9392
}
@@ -120,7 +119,7 @@ void givedMinimalShardedClusterPrimaryDisabled_shouldGenerateCoordinatorCluster(
120119
cluster.getSpec().getPostgresServices().getPrimary());
121120
Assertions.assertEquals(
122121
new StackGresClusterPostgresServiceBuilder()
123-
.withEnabled(false)
122+
.withEnabled(true)
124123
.build(),
125124
cluster.getSpec().getPostgresServices().getReplicas());
126125
}
@@ -236,8 +235,6 @@ void givedShardedClusterWithMinimalCoordinator_shouldCopyGlobalSettings() {
236235
createWithRandomData(String.class),
237236
createWithRandomData(String.class)));
238237
setMinimalCoordinatorAndShards(shardedCluster);
239-
var coordinatorPrimary =
240-
shardedCluster.getSpec().getPostgresServices().getCoordinator().getPrimary();
241238
var cluster = getCoordinatorCluster(JsonUtil.copy(shardedCluster));
242239
checkCoordinatorWithGlobalSettings(
243240
shardedCluster,
@@ -246,16 +243,10 @@ void givedShardedClusterWithMinimalCoordinator_shouldCopyGlobalSettings() {
246243
cluster,
247244
0);
248245
Assertions.assertEquals(
249-
new StackGresClusterPostgresServiceBuilder()
250-
.withEnabled(Optional.ofNullable(coordinatorPrimary.getEnabled()).orElse(true))
251-
.withCustomPorts(
252-
shardedCluster.getSpec().getPostgresServices().getCoordinator().getCustomPorts())
253-
.build(),
246+
shardedCluster.getSpec().getPostgresServices().getCoordinator().getPrimary(),
254247
cluster.getSpec().getPostgresServices().getPrimary());
255248
Assertions.assertEquals(
256-
new StackGresClusterPostgresServiceBuilder()
257-
.withEnabled(false)
258-
.build(),
249+
shardedCluster.getSpec().getPostgresServices().getCoordinator().getAny(),
259250
cluster.getSpec().getPostgresServices().getReplicas());
260251
}
261252

@@ -271,8 +262,6 @@ void givedShardedClusterWithMinimalShards_shouldCopyGlobalSettings() {
271262
createWithRandomData(String.class),
272263
createWithRandomData(String.class)));
273264
setMinimalCoordinatorAndShards(shardedCluster);
274-
var shardsPrimary =
275-
shardedCluster.getSpec().getPostgresServices().getShards().getPrimaries();
276265
var cluster = getShardsCluster(JsonUtil.copy(shardedCluster), 0);
277266
checkClusterWithGlobalSettings(
278267
shardedCluster,
@@ -281,11 +270,7 @@ void givedShardedClusterWithMinimalShards_shouldCopyGlobalSettings() {
281270
cluster,
282271
1);
283272
Assertions.assertEquals(
284-
new StackGresClusterPostgresServiceBuilder()
285-
.withEnabled(Optional.ofNullable(shardsPrimary.getEnabled()).orElse(true))
286-
.withCustomPorts(
287-
shardedCluster.getSpec().getPostgresServices().getShards().getCustomPorts())
288-
.build(),
273+
shardedCluster.getSpec().getPostgresServices().getShards().getPrimaries(),
289274
cluster.getSpec().getPostgresServices().getPrimary());
290275
Assertions.assertEquals(
291276
new StackGresClusterPostgresServiceBuilder()

‎stackgres-k8s/src/operator/src/test/java/io/stackgres/operator/conciliation/factory/shardedcluster/StackGresShardedClusterForDdpUtilTest.java‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
import java.util.ArrayList;
1313
import java.util.List;
14-
import java.util.Optional;
1514

1615
import io.stackgres.common.StackGresShardedClusterUtil;
1716
import io.stackgres.common.crd.SecretKeySelector;
@@ -54,7 +53,7 @@ void givedMinimalShardedCluster_shouldGenerateCoordinatorCluster() {
5453
cluster.getSpec().getPostgresServices().getPrimary());
5554
Assertions.assertEquals(
5655
new StackGresClusterPostgresServiceBuilder()
57-
.withEnabled(false)
56+
.withEnabled(true)
5857
.build(),
5958
cluster.getSpec().getPostgresServices().getReplicas());
6059
}
@@ -84,7 +83,7 @@ void givedMinimalShardedClusterPrimaryEnabled_shouldGenerateCoordinatorCluster()
8483
cluster.getSpec().getPostgresServices().getPrimary());
8584
Assertions.assertEquals(
8685
new StackGresClusterPostgresServiceBuilder()
87-
.withEnabled(false)
86+
.withEnabled(true)
8887
.build(),
8988
cluster.getSpec().getPostgresServices().getReplicas());
9089
}
@@ -117,7 +116,7 @@ void givedMinimalShardedClusterPrimaryDisabled_shouldGenerateCoordinatorCluster(
117116
cluster.getSpec().getPostgresServices().getPrimary());
118117
Assertions.assertEquals(
119118
new StackGresClusterPostgresServiceBuilder()
120-
.withEnabled(false)
119+
.withEnabled(true)
121120
.build(),
122121
cluster.getSpec().getPostgresServices().getReplicas());
123122
}
@@ -233,8 +232,6 @@ void givedShardedClusterWithMinimalCoordinator_shouldCopyGlobalSettings() {
233232
createWithRandomData(String.class),
234233
createWithRandomData(String.class)));
235234
setMinimalCoordinatorAndShards(shardedCluster);
236-
var coordinatorPrimary =
237-
shardedCluster.getSpec().getPostgresServices().getCoordinator().getPrimary();
238235
var cluster = getCoordinatorCluster(JsonUtil.copy(shardedCluster));
239236
checkClusterWithGlobalSettings(
240237
shardedCluster,
@@ -243,16 +240,10 @@ void givedShardedClusterWithMinimalCoordinator_shouldCopyGlobalSettings() {
243240
cluster,
244241
0);
245242
Assertions.assertEquals(
246-
new StackGresClusterPostgresServiceBuilder()
247-
.withEnabled(Optional.ofNullable(coordinatorPrimary.getEnabled()).orElse(true))
248-
.withCustomPorts(
249-
shardedCluster.getSpec().getPostgresServices().getCoordinator().getCustomPorts())
250-
.build(),
243+
shardedCluster.getSpec().getPostgresServices().getCoordinator().getPrimary(),
251244
cluster.getSpec().getPostgresServices().getPrimary());
252245
Assertions.assertEquals(
253-
new StackGresClusterPostgresServiceBuilder()
254-
.withEnabled(false)
255-
.build(),
246+
shardedCluster.getSpec().getPostgresServices().getCoordinator().getAny(),
256247
cluster.getSpec().getPostgresServices().getReplicas());
257248
}
258249

@@ -268,8 +259,6 @@ void givedShardedClusterWithMinimalShards_shouldCopyGlobalSettings() {
268259
createWithRandomData(String.class),
269260
createWithRandomData(String.class)));
270261
setMinimalCoordinatorAndShards(shardedCluster);
271-
var shardsPrimary =
272-
shardedCluster.getSpec().getPostgresServices().getShards().getPrimaries();
273262
var cluster = getShardsCluster(JsonUtil.copy(shardedCluster), 0);
274263
checkClusterWithGlobalSettings(
275264
shardedCluster,
@@ -278,11 +267,7 @@ void givedShardedClusterWithMinimalShards_shouldCopyGlobalSettings() {
278267
cluster,
279268
1);
280269
Assertions.assertEquals(
281-
new StackGresClusterPostgresServiceBuilder()
282-
.withEnabled(Optional.ofNullable(shardsPrimary.getEnabled()).orElse(true))
283-
.withCustomPorts(
284-
shardedCluster.getSpec().getPostgresServices().getShards().getCustomPorts())
285-
.build(),
270+
shardedCluster.getSpec().getPostgresServices().getShards().getPrimaries(),
286271
cluster.getSpec().getPostgresServices().getPrimary());
287272
Assertions.assertEquals(
288273
new StackGresClusterPostgresServiceBuilder()

0 commit comments

Comments
 (0)