feat(server): Kubernetes health probes (#4464) (#4473)

Author: robfrank

engine/src/main/java/com/arcadedb/GlobalConfiguration.java

@@ -545,6 +545,10 @@ Enable diagnostic logging during vector graph build progress (heap/off-heap memo
 
   SERVER_METRICS_LOGGING("arcadedb.serverMetrics.logging", SCOPE.SERVER, "True to enable metrics logging", Boolean.class, false),
 
+  SERVER_READINESS_REQUIRES_HA("arcadedb.server.readinessRequiresHA", SCOPE.SERVER,
+      "When true and HA is active, /api/v1/ready also requires the node to have joined the Raft group and be caught up. Default false preserves current readiness behavior.",
+      Boolean.class, false),
+
   //paths
   SERVER_ROOT_PATH("arcadedb.server.rootPath", SCOPE.SERVER,
       "Root path in the file system where the server is looking for files. By default is the current directory", String.class,

engine/src/test/java/com/arcadedb/GlobalConfigurationReadinessHATest.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class GlobalConfigurationReadinessHATest {
+  @Test
+  void keyExistsAndDefaultsToFalse() {
+    assertThat(GlobalConfiguration.SERVER_READINESS_REQUIRES_HA.getKey()).isEqualTo("arcadedb.server.readinessRequiresHA");
+    assertThat(GlobalConfiguration.SERVER_READINESS_REQUIRES_HA.getValueAsBoolean()).isFalse();
+  }
+
+  @Test
+  void lookupByKeyResolvesConstant() {
+    assertThat(GlobalConfiguration.findByKey("arcadedb.server.readinessRequiresHA"))
+        .isSameAs(GlobalConfiguration.SERVER_READINESS_REQUIRES_HA);
+  }
+}

k8s/README.md

@@ -11,3 +11,35 @@ helm repo add arcadedb https://helm.arcadedb.com/
 helm repo update
 helm install my-arcadedb arcadedb/arcadedb
 ```
+
+## Health probes
+
+ArcadeDB exposes two HTTP probe endpoints on the API port (default `2480`):
+
+- `GET /api/v1/health` - liveness. Returns `204` when the process and HTTP layer are up.
+  Performs no database I/O and requires no authentication. Use this for `livenessProbe` so
+  Kubernetes never restarts a node that is merely warming up.
+- `GET /api/v1/ready` - readiness. Returns `204` when the server is `ONLINE`. When
+  `arcadedb.server.readinessRequiresHA=true` and HA is active, it returns `503` until the node
+  has joined the Raft group and caught up. Default (`false`) preserves single-node readiness
+  behavior.
+
+```yaml
+livenessProbe:
+  httpGet:
+    path: /api/v1/health
+    port: 2480
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  failureThreshold: 3
+readinessProbe:
+  httpGet:
+    path: /api/v1/ready
+    port: 2480
+  initialDelaySeconds: 5
+  periodSeconds: 5
+  failureThreshold: 3
+```
+
+To make readiness HA-aware in a clustered deployment, set the environment variable
+`ARCADEDB_SERVER_READINESSREQUIRESHA=true` (or pass `-Darcadedb.server.readinessRequiresHA=true`).

load-tests/src/test/java/com/arcadedb/test/support/ContainersTestTemplate.java

@@ -389,7 +389,7 @@ protected GenericContainer<?> createArcadeContainer(String name, Network network
             """, name))
         .withEnv("ARCADEDB_OPTS_MEMORY", "-Xms2G -Xmx2G")
         .withCreateContainerCmdModifier(cmd -> cmd.getHostConfig().withMemory(3L * 1024 * 1024 * 1024))
-        .waitingFor(Wait.forHttp("/api/v1/ready").forPort(2480).forStatusCode(204));
+        .waitingFor(Wait.forHttp("/api/v1/health").forPort(2480).forStatusCode(204));
     containers.add(container);
     return container;
   }
@@ -428,14 +428,14 @@ protected GenericContainer<?> createArcadeContainer(
             -Darcadedb.backup.enabled=false
             -Darcadedb.typeDefaultBuckets=10
             -Darcadedb.ha.enabled=true
-            -Darcadedb.ha.implementation=raft
             -Darcadedb.ha.quorum=%s
+            -Darcadedb.server.readinessRequiresHA=true
             -Darcadedb.ha.raft.port=2434
             -Darcadedb.ha.serverList=%s
             """, name, quorum, serverList))
         .withEnv("ARCADEDB_OPTS_MEMORY", "-Xms2G -Xmx2G")
         .withCreateContainerCmdModifier(cmd -> cmd.getHostConfig().withMemory(3L * 1024 * 1024 * 1024))
-        .waitingFor(Wait.forHttp("/api/v1/ready").forPort(2480).forStatusCode(204));
+        .waitingFor(Wait.forHttp("/api/v1/health").forPort(2480).forStatusCode(204));
     containers.add(container);
     return container;
   }
@@ -472,14 +472,14 @@ protected GenericContainer<?> createPersistentArcadeContainer(
             -Darcadedb.backup.enabled=false
             -Darcadedb.typeDefaultBuckets=10
             -Darcadedb.ha.enabled=true
-            -Darcadedb.ha.implementation=raft
+            -Darcadedb.server.readinessRequiresHA=true
             -Darcadedb.ha.quorum=%s
             -Darcadedb.ha.raft.port=2434
             -Darcadedb.ha.serverList=%s
             """, name, quorum, serverList))
         .withEnv("ARCADEDB_OPTS_MEMORY", "-Xms2G -Xmx2G")
         .withCreateContainerCmdModifier(cmd -> cmd.getHostConfig().withMemory(3L * 1024 * 1024 * 1024))
-        .waitingFor(Wait.forHttp("/api/v1/ready").forPort(2480).forStatusCode(204));
+        .waitingFor(Wait.forHttp("/api/v1/health").forPort(2480).forStatusCode(204));
     containers.add(container);
     return container;
   }

server/src/main/java/com/arcadedb/server/http/HttpServer.java

@@ -35,6 +35,7 @@
 import com.arcadedb.server.http.handler.GetUsersHandler;
 import com.arcadedb.server.http.handler.GetDynamicContentHandler;
 import com.arcadedb.server.http.handler.GetExistsDatabaseHandler;
+import com.arcadedb.server.http.handler.GetHealthHandler;
 import com.arcadedb.server.http.handler.GetOpenApiHandler;
 import com.arcadedb.server.http.handler.GetQueryHandler;
 import com.arcadedb.server.http.handler.GetReadyHandler;
@@ -230,6 +231,7 @@ private PathHandler setupRoutes() {
         .get("/server", new GetServerHandler(this))
         .post("/server", new PostServerCommandHandler(this))
         .get("/ready", new GetReadyHandler(this))
+        .get("/health", new GetHealthHandler(this))
         .get("/openapi.json", new GetOpenApiHandler(this))
         .get("/docs", new GetApiDocsHandler(this))
         .get("/server/api-tokens", new GetApiTokensHandler(this))

server/src/main/java/com/arcadedb/server/http/handler/GetHealthHandler.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.server.http.handler;
+
+import com.arcadedb.serializer.json.JSONObject;
+import com.arcadedb.server.http.HttpServer;
+import com.arcadedb.server.security.ServerSecurityUser;
+import io.micrometer.core.instrument.Metrics;
+import io.undertow.server.HttpServerExchange;
+
+/**
+ * Kubernetes liveness probe. Reports whether the server process and HTTP layer are up.
+ * Performs no database I/O and requires no authentication so an orchestrator can poll it
+ * cheaply. Distinct from readiness: a node that is merely warming up is still live and must
+ * not be killed by the orchestrator.
+ */
+public class GetHealthHandler extends AbstractServerHttpHandler {
+  public GetHealthHandler(final HttpServer httpServer) {
+    super(httpServer);
+  }
+
+  @Override
+  public ExecutionResponse execute(final HttpServerExchange exchange, final ServerSecurityUser user, final JSONObject payload) {
+    Metrics.counter("http.health").increment();
+
+    // Liveness only: reaching this handler proves the HTTP layer is up, so the process is live.
+    // It deliberately does not consult server status, so a node still warming up is not killed.
+    return new ExecutionResponse(204, "");
+  }
+
+  @Override
+  public boolean isRequireAuthentication() {
+    return false;
+  }
+}

server/src/main/java/com/arcadedb/server/http/handler/GetReadyHandler.java

@@ -18,8 +18,10 @@
  */
 package com.arcadedb.server.http.handler;
 
+import com.arcadedb.GlobalConfiguration;
 import com.arcadedb.serializer.json.JSONObject;
 import com.arcadedb.server.ArcadeDBServer;
+import com.arcadedb.server.HAServerPlugin;
 import com.arcadedb.server.http.HttpServer;
 import com.arcadedb.server.security.ServerSecurityUser;
 import io.micrometer.core.instrument.Metrics;
@@ -34,9 +36,20 @@ public GetReadyHandler(final HttpServer httpServer) {
   public ExecutionResponse execute(final HttpServerExchange exchange, final ServerSecurityUser user, final JSONObject payload) {
     Metrics.counter("http.ready").increment();
 
-    if (httpServer.getServer().getStatus() == ArcadeDBServer.STATUS.ONLINE)
-      return new ExecutionResponse(204, "");
-    return new ExecutionResponse(503, "Server not started yet");
+    final ArcadeDBServer server = httpServer.getServer();
+    if (server.getStatus() != ArcadeDBServer.STATUS.ONLINE)
+      return new ExecutionResponse(503, "Server not started yet");
+
+    if (server.getConfiguration().getValueAsBoolean(GlobalConfiguration.SERVER_READINESS_REQUIRES_HA)
+        && server.getConfiguration().getValueAsBoolean(GlobalConfiguration.HA_ENABLED)) {
+      final HAServerPlugin ha = server.getHA();
+      // ELECTION_STATUS.DONE means a leader is known; it does not guarantee this follower has
+      // replicated all committed log entries, so a slow follower may report ready before catch-up.
+      if (ha == null || ha.getElectionStatus() != HAServerPlugin.ELECTION_STATUS.DONE)
+        return new ExecutionResponse(503, "Node has not yet joined the Raft group");
+    }
+
+    return new ExecutionResponse(204, "");
   }
 
   @Override

server/src/main/java/com/arcadedb/server/http/handler/OpenApiSpecGenerator.java

@@ -141,6 +141,7 @@ private Paths createPaths() {
     // Server endpoints
     paths.addPathItem("/api/v1/server", createServerPath());
     paths.addPathItem("/api/v1/ready", createReadyPath());
+    paths.addPathItem("/api/v1/health", createHealthPath());
     paths.addPathItem("/api/v1/databases", createDatabasesPath());
 
     // Database endpoints
@@ -216,6 +217,32 @@ private PathItem createReadyPath() {
     return pathItem;
   }
 
+  private PathItem createHealthPath() {
+    final PathItem pathItem = new PathItem();
+
+    final Operation getOp = new Operation();
+    getOp.setSummary("Check server liveness");
+    getOp.setDescription("Liveness probe: returns 204 when the server process and HTTP layer are up. Performs no database I/O and requires no authentication.");
+    getOp.setOperationId("checkHealth");
+    getOp.addTagsItem("Health");
+    getOp.setResponses(createHealthResponses());
+    pathItem.setGet(getOp);
+
+    return pathItem;
+  }
+
+  private ApiResponses createHealthResponses() {
+    final ApiResponses responses = new ApiResponses();
+
+    // Liveness only ever responds with a 2xx when reachable; it never returns 503 (unlike readiness).
+    final ApiResponse liveResponse = new ApiResponse();
+    liveResponse.setDescription("Server process and HTTP layer are up");
+    responses.addApiResponse("200", liveResponse);
+    responses.addApiResponse("204", liveResponse);
+
+    return responses;
+  }
+
   private PathItem createDatabasesPath() {
     final PathItem pathItem = new PathItem();
 

server/src/test/java/com/arcadedb/server/http/GetHealthHandlerTest.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.server.http;
+
+import com.arcadedb.server.http.handler.ExecutionResponse;
+import com.arcadedb.server.http.handler.GetHealthHandler;
+import com.arcadedb.server.security.ServerSecurityUser;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+
+class GetHealthHandlerTest {
+
+  @Test
+  void alwaysReturns204() throws Exception {
+    // Liveness only: if the handler runs at all, the HTTP layer is up. Server status is irrelevant.
+    final GetHealthHandler handler = new GetHealthHandler(mock(HttpServer.class));
+
+    final ExecutionResponse response = handler.execute(null, (ServerSecurityUser) null, null);
+    assertThat(response.getCode()).isEqualTo(204);
+  }
+
+  @Test
+  void doesNotRequireAuthentication() {
+    final GetHealthHandler handler = new GetHealthHandler(mock(HttpServer.class));
+
+    assertThat(handler.isRequireAuthentication()).isFalse();
+  }
+}