Skip to content

Commit 7efd515

Browse files
teoincontattoteoincontatto
committed
feat: allow the patroni container to continue working when the cluster-controller experience failures but was already bootstrapped
1 parent f34fe3f commit 7efd515

4 files changed

Lines changed: 118 additions & 24 deletions

File tree

‎stackgres-k8s/src/cluster-controller/src/main/java/io/stackgres/cluster/app/StackGresClusterControllerMain.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ private void runSingleReconciliationCycle() {
137137
return exception;
138138
},
139139
(u, v) -> v);
140-
if (ex.getSuppressed().length > 0) {
140+
if (!ClusterControllerReconciliationCycle.existsContextResource()) {
141141
throw ex;
142142
}
143143
}

‎stackgres-k8s/src/cluster-controller/src/main/java/io/stackgres/cluster/controller/ClusterControllerReconciliationCycle.java‎

Lines changed: 79 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,15 @@
88
import static io.stackgres.common.ClusterControllerProperty.CLUSTER_NAME;
99
import static io.stackgres.common.ClusterControllerProperty.CLUSTER_NAMESPACE;
1010

11+
import java.nio.file.Files;
12+
import java.nio.file.Path;
13+
import java.nio.file.Paths;
1114
import java.util.List;
1215
import java.util.Optional;
1316
import java.util.function.Consumer;
1417
import java.util.stream.Stream;
1518

19+
import com.fasterxml.jackson.databind.ObjectMapper;
1620
import io.fabric8.kubernetes.api.model.HasMetadata;
1721
import io.fabric8.kubernetes.client.KubernetesClient;
1822
import io.quarkus.runtime.ShutdownEvent;
@@ -23,6 +27,8 @@
2327
import io.stackgres.cluster.configuration.ClusterControllerPropertyContext;
2428
import io.stackgres.cluster.resource.ClusterResourceHandlerSelector;
2529
import io.stackgres.common.CdiUtil;
30+
import io.stackgres.common.ClusterControllerProperty;
31+
import io.stackgres.common.ClusterPath;
2632
import io.stackgres.common.crd.sgcluster.StackGresCluster;
2733
import io.stackgres.common.crd.sgcluster.StackGresClusterStatus;
2834
import io.stackgres.common.labels.LabelFactoryForCluster;
@@ -34,18 +40,23 @@
3440
import jakarta.enterprise.event.Observes;
3541
import jakarta.inject.Inject;
3642
import org.jooq.lambda.tuple.Tuple2;
43+
import org.slf4j.Logger;
44+
import org.slf4j.LoggerFactory;
3745
import org.slf4j.helpers.MessageFormatter;
3846

3947
@ApplicationScoped
4048
public class ClusterControllerReconciliationCycle
4149
extends
4250
ReconciliationCycle<StackGresClusterContext, StackGresCluster, ClusterResourceHandlerSelector> {
4351

52+
private static final Logger LOGGER = LoggerFactory.getLogger(ClusterControllerReconciliationCycle.class);
53+
4454
private final ClusterControllerPropertyContext propertyContext;
4555
private final EventController eventController;
4656
private final LabelFactoryForCluster labelFactory;
4757
private final CustomResourceFinder<StackGresCluster> clusterFinder;
4858
private final Metrics metrics;
59+
private final ObjectMapper objectMapper;
4960
private long reconciliationStart;
5061

5162
@Dependent
@@ -66,6 +77,8 @@ public static class Parameters {
6677
CustomResourceFinder<StackGresCluster> clusterFinder;
6778
@Inject
6879
Metrics metrics;
80+
@Inject
81+
ObjectMapper objectMapper;
6982
}
7083

7184
/**
@@ -81,6 +94,7 @@ public ClusterControllerReconciliationCycle(Parameters parameters) {
8194
this.labelFactory = parameters.labelFactory;
8295
this.clusterFinder = parameters.clusterFinder;
8396
this.metrics = parameters.metrics;
97+
this.objectMapper = parameters.objectMapper;
8498
}
8599

86100
public ClusterControllerReconciliationCycle() {
@@ -91,6 +105,7 @@ public ClusterControllerReconciliationCycle() {
91105
this.labelFactory = null;
92106
this.clusterFinder = null;
93107
this.metrics = null;
108+
this.objectMapper = null;
94109
}
95110

96111
public static ClusterControllerReconciliationCycle create(Consumer<Parameters> consumer) {
@@ -174,22 +189,24 @@ protected StackGresClusterContext getContextWithExistingAndRequiredResources(
174189

175190
@Override
176191
public List<StackGresCluster> getExistingContextResources() {
177-
return clusterFinder.findByNameAndNamespace(
178-
propertyContext.getString(CLUSTER_NAME),
179-
propertyContext.getString(CLUSTER_NAMESPACE))
180-
.stream()
181-
.toList();
192+
return List.of(getExistingCustomResource(
193+
LOGGER,
194+
clusterFinder,
195+
objectMapper,
196+
propertyContext.getString(CLUSTER_NAMESPACE),
197+
propertyContext.getString(CLUSTER_NAME)));
182198
}
183199

184200
@Override
185201
public StackGresCluster getExistingContextResource(StackGresCluster source) {
186202
final String namespace = source.getMetadata().getNamespace();
187203
final String name = source.getMetadata().getName();
188-
return clusterFinder.findByNameAndNamespace(
189-
name,
190-
namespace)
191-
.orElseThrow(() -> new IllegalArgumentException(StackGresCluster.KIND
192-
+ " " + name + "." + namespace + " not found"));
204+
return getExistingCustomResource(
205+
LOGGER,
206+
clusterFinder,
207+
objectMapper,
208+
namespace,
209+
name);
193210
}
194211

195212
@Override
@@ -204,4 +221,56 @@ protected StackGresClusterContext getContextFromResource(
204221
.build();
205222
}
206223

224+
static StackGresCluster getExistingCustomResource(
225+
final Logger logger,
226+
final CustomResourceFinder<StackGresCluster> clusterFinder,
227+
final ObjectMapper objectMapper,
228+
final String namespace,
229+
final String name) {
230+
final Path latestCustomResourcePath = getLatestCustomResourcePath(namespace, name);
231+
try {
232+
return clusterFinder.findByNameAndNamespace(name, namespace)
233+
.orElseThrow(() -> new IllegalArgumentException(StackGresCluster.KIND
234+
+ " " + name + "." + namespace + " not found"));
235+
} catch (Exception ex) {
236+
if (Files.exists(latestCustomResourcePath)) {
237+
try {
238+
return objectMapper.readValue(latestCustomResourcePath.toFile(), StackGresCluster.class);
239+
} catch (Exception jex) {
240+
ex.addSuppressed(jex);
241+
}
242+
}
243+
if (ex instanceof RuntimeException rex) {
244+
throw rex;
245+
}
246+
throw new RuntimeException(ex);
247+
}
248+
}
249+
250+
public static boolean existsContextResource() {
251+
return Files.exists(getLatestCustomResourcePath(
252+
ClusterControllerProperty.CLUSTER_NAMESPACE.getString(),
253+
ClusterControllerProperty.CLUSTER_NAME.getString()));
254+
}
255+
256+
static void writeCustomResource(
257+
final Logger logger,
258+
final ObjectMapper objectMapper,
259+
final StackGresCluster cluster) {
260+
final Path latestCustomResourcePath = getLatestCustomResourcePath(
261+
cluster.getMetadata().getNamespace(),
262+
cluster.getMetadata().getName());
263+
try {
264+
objectMapper.writeValue(latestCustomResourcePath.toFile(), cluster);
265+
} catch (Exception jex) {
266+
logger.warn("Error while trying to store latest value of SGCluster to " + latestCustomResourcePath, jex);
267+
}
268+
}
269+
270+
private static Path getLatestCustomResourcePath(final String namespace, final String name) {
271+
return Paths.get(
272+
ClusterPath.PG_BASE_PATH.path(),
273+
".latest." + namespace + "." + name + ".sgcluster.json");
274+
}
275+
207276
}

‎stackgres-k8s/src/cluster-controller/src/main/java/io/stackgres/cluster/controller/ClusterControllerReconciliator.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,14 @@
55

66
package io.stackgres.cluster.controller;
77

8+
import static io.stackgres.cluster.controller.ClusterControllerReconciliationCycle.writeCustomResource;
9+
810
import java.util.ArrayList;
911
import java.util.List;
1012
import java.util.Objects;
1113
import java.util.Optional;
1214

15+
import com.fasterxml.jackson.databind.ObjectMapper;
1316
import io.fabric8.kubernetes.client.KubernetesClient;
1417
import io.stackgres.cluster.common.StackGresClusterContext;
1518
import io.stackgres.cluster.configuration.ClusterControllerPropertyContext;
@@ -43,6 +46,7 @@ public class ClusterControllerReconciliator
4346
private final PatroniConfigReconciliator patroniConfigReconciliator;
4447
private final PatroniMajorVersionUpgradeReconciliator patroniMajorVersionUpgradeReconciliator;
4548
private final PatroniBackupFailoverRestartReconciliator patroniBackupFailoverRestartReconciliator;
49+
private final ObjectMapper objectMapper;
4650
private final ClusterControllerPropertyContext propertyContext;
4751
private final String podName;
4852
private final Optional<String> nodeName;
@@ -61,6 +65,7 @@ public ClusterControllerReconciliator(Parameters parameters) {
6165
this.patroniConfigReconciliator = parameters.patroniConfigReconciliator;
6266
this.patroniMajorVersionUpgradeReconciliator = parameters.patroniMajorVersionUpgradeReconciliator;
6367
this.patroniBackupFailoverRestartReconciliator = parameters.patroniBackupFailoverRestartReconciliator;
68+
this.objectMapper = parameters.objectMapper;
6469
this.propertyContext = parameters.propertyContext;
6570
this.podName = parameters.propertyContext
6671
.getString(ClusterControllerProperty.CLUSTER_CONTROLLER_POD_NAME);
@@ -83,6 +88,7 @@ public ClusterControllerReconciliator() {
8388
this.patroniConfigReconciliator = null;
8489
this.patroniMajorVersionUpgradeReconciliator = null;
8590
this.patroniBackupFailoverRestartReconciliator = null;
91+
this.objectMapper = null;
8692
this.propertyContext = null;
8793
this.podName = null;
8894
this.nodeName = null;
@@ -141,17 +147,18 @@ public ReconciliationResult<Void> reconcile(KubernetesClient client,
141147
ReconciliationResult<Void> patroniBackupFailoverRestartReconciliatorResult =
142148
patroniBackupFailoverRestartReconciliator.reconcile(client, context);
143149

150+
StackGresCluster updatedCluster = cluster;
144151
if (foundPodStatus.isEmpty()
145152
|| nodeNameChanged
146153
|| postgresBootstrapReconciliatorResult.result().orElse(false)
147154
|| extensionReconciliationResult.result().orElse(false)
148155
|| patroniReconciliationResult.result().orElse(false)) {
149-
clusterScheduler.update(cluster,
156+
updatedCluster = clusterScheduler.update(cluster,
150157
(currentCluster) -> updateClusterPodStatus(currentCluster, cluster));
151158
}
152159

153160
if (extensionReconciliationResult.result().orElse(false)) {
154-
KubernetesClientUtil.retryOnConflict(() -> clusterScheduler.update(cluster,
161+
updatedCluster = KubernetesClientUtil.retryOnConflict(() -> clusterScheduler.update(cluster,
155162
(currentCluster) -> {
156163
Optional.ofNullable(cluster.getStatus())
157164
.map(StackGresClusterStatus::getExtensions)
@@ -178,7 +185,7 @@ public ReconciliationResult<Void> reconcile(KubernetesClient client,
178185

179186
var pvcSizeReconciliatorResult = pvcSizeReconciliator.reconcile(client, propertyContext);
180187

181-
return postgresBootstrapReconciliatorResult
188+
var result = postgresBootstrapReconciliatorResult
182189
.join(extensionReconciliationResult)
183190
.join(pgbouncerReconciliationResult)
184191
.join(patroniReconciliationResult)
@@ -189,6 +196,10 @@ public ReconciliationResult<Void> reconcile(KubernetesClient client,
189196
.join(patroniMajorVersionUpgradeReconciliatorResult)
190197
.join(patroniBackupFailoverRestartReconciliatorResult)
191198
.join(pvcSizeReconciliatorResult);
199+
if (result.success()) {
200+
writeCustomResource(logger, objectMapper, updatedCluster);
201+
}
202+
return result;
192203
}
193204

194205
private void updateClusterPodStatus(StackGresCluster currentCluster,
@@ -238,6 +249,7 @@ public static class Parameters {
238249
@Inject PatroniConfigReconciliator patroniConfigReconciliator;
239250
@Inject PatroniMajorVersionUpgradeReconciliator patroniMajorVersionUpgradeReconciliator;
240251
@Inject PatroniBackupFailoverRestartReconciliator patroniBackupFailoverRestartReconciliator;
252+
@Inject ObjectMapper objectMapper;
241253
}
242254

243255
}

‎stackgres-k8s/src/cluster-controller/src/main/java/io/stackgres/cluster/controller/PatroniExternalCdsControllerReconciliationCycle.java‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package io.stackgres.cluster.controller;
77

8+
import static io.stackgres.cluster.controller.ClusterControllerReconciliationCycle.getExistingCustomResource;
89
import static io.stackgres.common.ClusterControllerProperty.CLUSTER_NAME;
910
import static io.stackgres.common.ClusterControllerProperty.CLUSTER_NAMESPACE;
1011

@@ -13,6 +14,7 @@
1314
import java.util.function.Consumer;
1415
import java.util.stream.Stream;
1516

17+
import com.fasterxml.jackson.databind.ObjectMapper;
1618
import io.fabric8.kubernetes.api.model.HasMetadata;
1719
import io.fabric8.kubernetes.client.KubernetesClient;
1820
import io.quarkus.runtime.ShutdownEvent;
@@ -34,18 +36,23 @@
3436
import jakarta.enterprise.event.Observes;
3537
import jakarta.inject.Inject;
3638
import org.jooq.lambda.tuple.Tuple2;
39+
import org.slf4j.Logger;
40+
import org.slf4j.LoggerFactory;
3741
import org.slf4j.helpers.MessageFormatter;
3842

3943
@ApplicationScoped
4044
public class PatroniExternalCdsControllerReconciliationCycle
4145
extends
4246
ReconciliationCycle<StackGresClusterContext, StackGresCluster, ClusterResourceHandlerSelector> {
4347

48+
private static final Logger LOGGER = LoggerFactory.getLogger(PatroniExternalCdsControllerReconciliationCycle.class);
49+
4450
private final ClusterControllerPropertyContext propertyContext;
4551
private final EventController eventController;
4652
private final LabelFactoryForCluster labelFactory;
4753
private final CustomResourceFinder<StackGresCluster> clusterFinder;
4854
private final Metrics metrics;
55+
private final ObjectMapper objectMapper;
4956
private long reconciliationStart;
5057

5158
@Dependent
@@ -66,6 +73,8 @@ public static class Parameters {
6673
CustomResourceFinder<StackGresCluster> clusterFinder;
6774
@Inject
6875
Metrics metrics;
76+
@Inject
77+
ObjectMapper objectMapper;
6978
}
7079

7180
@Inject
@@ -78,6 +87,7 @@ public PatroniExternalCdsControllerReconciliationCycle(Parameters parameters) {
7887
this.labelFactory = parameters.labelFactory;
7988
this.clusterFinder = parameters.clusterFinder;
8089
this.metrics = parameters.metrics;
90+
this.objectMapper = parameters.objectMapper;
8191
}
8292

8393
public PatroniExternalCdsControllerReconciliationCycle() {
@@ -88,6 +98,7 @@ public PatroniExternalCdsControllerReconciliationCycle() {
8898
this.labelFactory = null;
8999
this.clusterFinder = null;
90100
this.metrics = null;
101+
this.objectMapper = null;
91102
}
92103

93104
public static PatroniExternalCdsControllerReconciliationCycle create(Consumer<Parameters> consumer) {
@@ -171,22 +182,24 @@ protected StackGresClusterContext getContextWithExistingAndRequiredResources(
171182

172183
@Override
173184
public List<StackGresCluster> getExistingContextResources() {
174-
return clusterFinder.findByNameAndNamespace(
175-
propertyContext.getString(CLUSTER_NAME),
176-
propertyContext.getString(CLUSTER_NAMESPACE))
177-
.stream()
178-
.toList();
185+
return List.of(getExistingCustomResource(
186+
LOGGER,
187+
clusterFinder,
188+
objectMapper,
189+
propertyContext.getString(CLUSTER_NAMESPACE),
190+
propertyContext.getString(CLUSTER_NAME)));
179191
}
180192

181193
@Override
182194
public StackGresCluster getExistingContextResource(StackGresCluster source) {
183195
final String namespace = source.getMetadata().getNamespace();
184196
final String name = source.getMetadata().getName();
185-
return clusterFinder.findByNameAndNamespace(
186-
name,
187-
namespace)
188-
.orElseThrow(() -> new IllegalArgumentException(StackGresCluster.KIND
189-
+ " " + name + "." + namespace + " not found"));
197+
return getExistingCustomResource(
198+
LOGGER,
199+
clusterFinder,
200+
objectMapper,
201+
namespace,
202+
name);
190203
}
191204

192205
@Override

0 commit comments

Comments
 (0)