feat: [ha] force resync button from studio

lvca · lvca · commit 048b0ef68111 · 2026-06-02T10:08:52.000-04:00
New red "Emergency Recovery" card on the Cluster tab. It lists each database with a "Resync from Leader" button, shown only when the local node is a follower. Clicking confirms (warning that local-only changes are discarded), POSTs to the local node's resync endpoint, and refreshes.
diff --git a/ha-raft/src/main/java/com/arcadedb/server/ha/raft/ArcadeStateMachine.java b/ha-raft/src/main/java/com/arcadedb/server/ha/raft/ArcadeStateMachine.java
@@ -996,6 +996,62 @@ private void installFromLeaderForBootstrap(final String dbName) {
     }
   }
 
+  /**
+   * Operator-triggered emergency recovery: drop the local copy of {@code dbName} and re-acquire a
+   * fresh full snapshot from the current leader. This is the manual equivalent of the automatic
+   * snapshot install path ({@link #notifyInstallSnapshotFromLeader}) and uses the same crash-safe
+   * {@link SnapshotInstaller} machinery as {@link #installFromLeaderForBootstrap}.
+   * <p>
+   * The intended use case is a follower that has diverged from the leader (e.g. a
+   * {@link WALVersionGapException} reported "snapshot resync required"): the diverged page versions
+   * can never be reconciled by applying further deltas, so the only safe fix is to replace the local
+   * files with the leader's authoritative copy. After install the local database matches the leader's
+   * snapshot point; any Raft log entries replayed afterwards that predate the snapshot are skipped by
+   * the page-version guard in {@code applyChanges}, and forward replication resumes normally.
+   * <p>
+   * Runs synchronously on the caller thread (the HTTP worker thread). Refuses to run on the leader
+   * (it holds the authoritative copy) and when no leader is currently known.
+   *
+   * @param dbName name of the database to resync from the leader
+   * @throws ReplicationException if Raft HA is not enabled, this node is the leader, the leader is
+   *                              unknown, or the snapshot install fails
+   */
+  public void resyncDatabaseFromLeader(final String dbName) {
+    final RaftHAServer raft = raftHAServer;
+    if (raft == null)
+      throw new ReplicationException("Cannot resync database '" + dbName + "': Raft HA is not enabled");
+
+    if (raft.isLeader())
+      throw new ReplicationException("Cannot resync database '" + dbName
+          + "' on the leader: the leader holds the authoritative copy. Run the resync on the diverged follower.");
+
+    if (raft.getLeaderHttpAddress() == null)
+      throw new ReplicationException("Cannot resync database '" + dbName
+          + "': the leader is currently unknown (election in progress?). Retry once a leader is elected.");
+
+    LogManager.instance().log(this, Level.WARNING,
+        "Operator-triggered resync of database '%s' from leader: dropping local copy and re-acquiring full snapshot", dbName);
+
+    try {
+      final String databasePath;
+      if (server.existsDatabase(dbName)) {
+        final DatabaseInternal db = (DatabaseInternal) server.getDatabase(dbName);
+        databasePath = db.getDatabasePath();
+        db.getEmbedded().close();
+        server.removeDatabase(dbName);
+      } else {
+        databasePath = server.getConfiguration().getValueAsString(GlobalConfiguration.SERVER_DATABASE_DIRECTORY)
+            + File.separator + dbName;
+      }
+      // Resolve the leader address on each retry (it can change mid-operation if leadership moves).
+      final String clusterToken = raft.getClusterToken();
+      SnapshotInstaller.install(dbName, databasePath, raft::getLeaderHttpAddress, clusterToken, server);
+      LogManager.instance().log(this, Level.INFO, "Database '%s' resynced from leader on operator request", dbName);
+    } catch (final IOException e) {
+      throw new ReplicationException("Failed to resync database '" + dbName + "' from leader", e);
+    }
+  }
+
   /**
    * Returns the bootstrap baseline committed for {@code dbName}, or {@code null} if no
    * {@link RaftLogEntryType#BOOTSTRAP_FINGERPRINT_ENTRY} has been applied for it. Visible to
diff --git a/ha-raft/src/main/java/com/arcadedb/server/ha/raft/GetClusterHandler.java b/ha-raft/src/main/java/com/arcadedb/server/ha/raft/GetClusterHandler.java
@@ -89,6 +89,21 @@ public ExecutionResponse execute(final HttpServerExchange exchange, final Server
     }
     response.put("peers", peers);
 
+    // Per-database list, used by Studio to render per-database actions (e.g. the emergency
+    // "Resync from Leader" control on followers) and to surface bootstrap baselines when present.
+    final JSONArray databases = new JSONArray();
+    for (final String dbName : httpServer.getServer().getDatabaseNames()) {
+      final JSONObject dbJson = new JSONObject();
+      dbJson.put("name", dbName);
+      final ArcadeStateMachine.BootstrapBaseline baseline = stateMachine.getBootstrapBaseline(dbName);
+      if (baseline != null) {
+        dbJson.put("bootstrapLastTxId", baseline.lastTxId());
+        dbJson.put("bootstrapFingerprint", baseline.fingerprint());
+      }
+      databases.put(dbJson);
+    }
+    response.put("databases", databases);
+
     return new ExecutionResponse(200, response.toString());
   }
 }
diff --git a/ha-raft/src/main/java/com/arcadedb/server/ha/raft/PostResyncDatabaseHandler.java b/ha-raft/src/main/java/com/arcadedb/server/ha/raft/PostResyncDatabaseHandler.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.server.ha.raft;
+
+import com.arcadedb.serializer.json.JSONObject;
+import com.arcadedb.server.http.HttpServer;
+import com.arcadedb.server.http.handler.AbstractServerHttpHandler;
+import com.arcadedb.server.http.handler.ExecutionResponse;
+import com.arcadedb.server.security.ServerSecurityUser;
+import io.undertow.server.HttpServerExchange;
+
+/**
+ * POST /api/v1/cluster/resync/{database} - emergency recovery that forces THIS node to drop its
+ * local copy of {@code database} and re-acquire a fresh full snapshot from the current leader.
+ * <p>
+ * Intended for a follower that has diverged from the leader (e.g. a {@code WALVersionGapException}
+ * reporting "snapshot resync required"). The endpoint always operates on the node that receives the
+ * request, so an operator points Studio (or curl) at the diverged follower and triggers it there.
+ * Refuses to run on the leader, which holds the authoritative copy.
+ *
+ * @author Luca Garulli (l.garulli@arcadedata.com)
+ */
+public class PostResyncDatabaseHandler extends AbstractServerHttpHandler {
+
+  private final RaftHAPlugin plugin;
+
+  public PostResyncDatabaseHandler(final HttpServer httpServer, final RaftHAPlugin plugin) {
+    super(httpServer);
+    this.plugin = plugin;
+  }
+
+  @Override
+  protected boolean mustExecuteOnWorkerThread() {
+    // Closing the database and downloading the snapshot from the leader is blocking I/O.
+    return true;
+  }
+
+  @Override
+  public ExecutionResponse execute(final HttpServerExchange exchange, final ServerSecurityUser user,
+      final JSONObject payload) {
+    checkRootUser(user);
+
+    final RaftHAServer raftHAServer = plugin.getRaftHAServer();
+    if (raftHAServer == null)
+      return new ExecutionResponse(400, new JSONObject().put("error", "Raft HA is not enabled").toString());
+
+    // Extract database name from path: /api/v1/cluster/resync/{database}
+    final String path = exchange.getRelativePath();
+    final String databaseName = (path.startsWith("/") ? path.substring(1) : path).trim();
+
+    if (databaseName.isEmpty())
+      return new ExecutionResponse(400, new JSONObject().put("error", "Database name is required in path").toString());
+
+    if (!PostVerifyDatabaseHandler.VALID_DATABASE_NAME.matcher(databaseName).matches())
+      return new ExecutionResponse(400, new JSONObject().put("error", "Invalid database name").toString());
+
+    if (raftHAServer.isLeader())
+      return new ExecutionResponse(400, new JSONObject().put("error",
+          "Cannot resync database '" + databaseName
+              + "' on the leader: the leader holds the authoritative copy. Run the resync on the diverged follower.").toString());
+
+    if (raftHAServer.getLeaderHttpAddress() == null)
+      return new ExecutionResponse(503, new JSONObject().put("error",
+          "Cannot resync database '" + databaseName
+              + "': the leader is currently unknown (election in progress?). Retry once a leader is elected.").toString());
+
+    try {
+      raftHAServer.getStateMachine().resyncDatabaseFromLeader(databaseName);
+    } catch (final Exception e) {
+      return new ExecutionResponse(500, new JSONObject().put("error",
+          "Resync of database '" + databaseName + "' failed: " + e.getMessage()).toString());
+    }
+
+    return new ExecutionResponse(200, new JSONObject()
+        .put("result", "Database '" + databaseName + "' resynced from leader")
+        .put("database", databaseName)
+        .put("localServer", httpServer.getServer().getServerName()).toString());
+  }
+}
diff --git a/ha-raft/src/main/java/com/arcadedb/server/ha/raft/RaftHAPlugin.java b/ha-raft/src/main/java/com/arcadedb/server/ha/raft/RaftHAPlugin.java
@@ -143,6 +143,7 @@ public void registerAPI(final HttpServer httpServer, final PathHandler routes) {
     routes.addExactPath("/api/v1/cluster/stepdown", new PostStepDownHandler(httpServer, this));
     routes.addExactPath("/api/v1/cluster/leave", new PostLeaveHandler(httpServer, this));
     routes.addPrefixPath("/api/v1/cluster/verify/", new PostVerifyDatabaseHandler(httpServer, this));
+    routes.addPrefixPath("/api/v1/cluster/resync/", new PostResyncDatabaseHandler(httpServer, this));
     // Issue #4147: pre-bootstrap state RPC, used by the bootstrap leader at first cluster
     // formation to collect each peer's (fingerprint, lastTxId) per database.
     routes.addExactPath("/api/v1/cluster/bootstrap-state", new PostBootstrapStateHandler(httpServer, this));
diff --git a/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftForceResyncIT.java b/ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftForceResyncIT.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.server.ha.raft;
+
+import com.arcadedb.database.Database;
+import com.arcadedb.graph.MutableVertex;
+import com.arcadedb.serializer.json.JSONObject;
+import org.junit.jupiter.api.Test;
+
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Integration tests for the operator-triggered emergency resync endpoint
+ * (POST /api/v1/cluster/resync/{database}) and {@link ArcadeStateMachine#resyncDatabaseFromLeader}.
+ */
+class RaftForceResyncIT extends BaseRaftHATest {
+
+  @Override
+  protected int getServerCount() {
+    // 3 nodes so a majority (2) is still available while one follower is being resynced.
+    return 3;
+  }
+
+  @Test
+  void followerResyncFromLeaderKeepsDataAndResumesReplication() throws Exception {
+    final int leaderIndex = findLeaderIndex();
+    assertThat(leaderIndex).as("A Raft leader must be elected").isGreaterThanOrEqualTo(0);
+    final int followerIndex = (leaderIndex + 1) % getServerCount();
+
+    final Database leaderDb = getServerDatabase(leaderIndex, getDatabaseName());
+    leaderDb.transaction(() -> {
+      if (!leaderDb.getSchema().existsType("ResyncTest"))
+        leaderDb.getSchema().createVertexType("ResyncTest");
+      for (int i = 0; i < 25; i++)
+        leaderDb.newVertex("ResyncTest").set("index", i).save();
+    });
+
+    assertClusterConsistency();
+    assertThat(getServerDatabase(followerIndex, getDatabaseName()).countType("ResyncTest", true))
+        .as("Follower has the initial data before resync").isEqualTo(25);
+
+    // Trigger the emergency resync on the follower: it drops its local copy and re-downloads a full
+    // snapshot from the leader.
+    final JSONObject response = resync(followerIndex, getDatabaseName());
+    assertThat(response.getString("result", "")).contains("resynced from leader");
+
+    assertThat(getServerDatabase(followerIndex, getDatabaseName()).countType("ResyncTest", true))
+        .as("Follower has the same data after resync").isEqualTo(25);
+
+    // Forward replication must resume after the resync: new writes on the leader reach the follower.
+    leaderDb.transaction(() -> {
+      for (int i = 25; i < 40; i++)
+        leaderDb.newVertex("ResyncTest").set("index", i).save();
+    });
+
+    assertClusterConsistency();
+    assertThat(getServerDatabase(followerIndex, getDatabaseName()).countType("ResyncTest", true))
+        .as("Follower receives writes committed after the resync").isEqualTo(40);
+  }
+
+  @Test
+  void resyncOnLeaderIsRejected() throws Exception {
+    final int leaderIndex = findLeaderIndex();
+    assertThat(leaderIndex).isGreaterThanOrEqualTo(0);
+
+    final int httpPort = getServer(leaderIndex).getHttpServer().getPort();
+    final HttpURLConnection conn = (HttpURLConnection) new URI(
+        "http://localhost:" + httpPort + "/api/v1/cluster/resync/" + getDatabaseName()).toURL().openConnection();
+    conn.setRequestMethod("POST");
+    conn.setRequestProperty("Authorization", basicAuth());
+
+    assertThat(conn.getResponseCode()).isEqualTo(400);
+    final String body = new String(conn.getErrorStream().readAllBytes(), StandardCharsets.UTF_8);
+    assertThat(new JSONObject(body).getString("error", "")).contains("leader");
+    conn.disconnect();
+  }
+
+  private JSONObject resync(final int serverIndex, final String databaseName) throws Exception {
+    final int httpPort = getServer(serverIndex).getHttpServer().getPort();
+    final HttpURLConnection conn = (HttpURLConnection) new URI(
+        "http://localhost:" + httpPort + "/api/v1/cluster/resync/" + databaseName).toURL().openConnection();
+    conn.setRequestMethod("POST");
+    conn.setRequestProperty("Authorization", basicAuth());
+    try {
+      assertThat(conn.getResponseCode()).isEqualTo(200);
+      return new JSONObject(new String(conn.getInputStream().readAllBytes(), StandardCharsets.UTF_8));
+    } finally {
+      conn.disconnect();
+    }
+  }
+
+  private static String basicAuth() {
+    return "Basic " + Base64.getEncoder().encodeToString(
+        ("root:" + DEFAULT_PASSWORD_FOR_TESTS).getBytes(StandardCharsets.UTF_8));
+  }
+}
diff --git a/studio/src/main/resources/static/cluster.html b/studio/src/main/resources/static/cluster.html
@@ -144,6 +144,29 @@ <h6 style="color: var(--text-secondary); font-weight: 600;">Cluster Nodes</h6>
             </div>
           </div>
 
+          <!-- Emergency recovery: force this follower to drop and re-acquire a database from the
+               leader. Hidden on the leader (it holds the authoritative copy) and toggled by
+               renderResyncRecovery in studio-cluster.js. -->
+          <div class="card mb-3" id="clusterResyncCard" style="display:none; border: 1px solid #fca5a5; border-radius: 10px;">
+            <div class="card-header py-2" style="background: #fef2f2; border-bottom: 1px solid #fca5a5; border-radius: 10px 10px 0 0;">
+              <i class="fas fa-sync-alt" style="color: #ef4444;"></i>
+              <span style="font-weight: 600; font-size: 0.88rem; margin-left: 6px; color: #dc2626;">Emergency Recovery</span>
+              <span style="font-size: 0.72rem; color: var(--text-secondary); margin-left: 8px;">
+                — resync a diverged database from the leader
+              </span>
+            </div>
+            <div class="card-body py-3">
+              <p style="font-size: 0.78rem; color: var(--text-secondary); margin-bottom: 10px;">
+                Drops this node's local copy of the selected database and re-downloads a fresh full
+                snapshot from the leader. Use when this follower has diverged (e.g. a
+                <span style="font-family: monospace;">WALVersionGapException / snapshot resync required</span> error).
+              </p>
+              <div id="clusterResyncList">
+                <!-- Dynamically populated by renderResyncRecovery -->
+              </div>
+            </div>
+          </div>
+
         </div>
 
         <!-- Right column: Danger -->
diff --git a/studio/src/main/resources/static/js/studio-cluster.js b/studio/src/main/resources/static/js/studio-cluster.js
