From d7781df28a86f094598b09b40eb8edefd61587cd Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Tue, 21 Apr 2026 11:53:57 +0530 Subject: [PATCH 1/2] HDDS-15065. Reduce Ratis snapshot gap to fix SCM flush delay. --- .../java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 --------- hadoop-hdds/common/src/main/resources/ozone-default.xml | 6 ------ .../java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 6 +++--- .../recon/AbstractTestStorageDistributionEndpoint.java | 2 -- .../hdds/scm/TestDatanodeSCMNodesReconfiguration.java | 1 - .../hadoop/hdds/scm/TestStorageContainerManagerHA.java | 1 - 6 files changed, 3 insertions(+), 22 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index a90bacc9fee2..e856c927a8a2 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -584,15 +584,6 @@ public final class ScmConfigKeys { public static final long OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD_DEFAULT = 1000L; - /** - * the config will transfer value to ratis config - * raft.server.snapshot.creation.gap, used by ratis to take snapshot - * when manual trigger using api. - */ - public static final String OZONE_SCM_HA_RATIS_SNAPSHOT_GAP - = "ozone.scm.ha.ratis.server.snapshot.creation.gap"; - public static final long OZONE_SCM_HA_RATIS_SNAPSHOT_GAP_DEFAULT = - 1024L; public static final String OZONE_SCM_HA_RATIS_SNAPSHOT_DIR = "ozone.scm.ha.ratis.snapshot.dir"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index ea3f2b0d9f96..766cbef14b20 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4168,12 +4168,6 @@ topology cluster tree from SCM. - - ozone.scm.ha.ratis.server.snapshot.creation.gap - 1024 - SCM, OZONE - Raft snapshot gap index after which snapshot can be taken. - ozone.scm.ha.dbtransactionbuffer.flush.interval 60s diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index f2900e38f405..1d2c29c9193d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -54,6 +54,8 @@ */ public final class RatisUtil { + public static final long SCM_RAFT_SNAPSHOT_GAP = 1L; + private RatisUtil() { } @@ -231,9 +233,7 @@ private static void setRaftSnapshotProperties( Snapshot.setAutoTriggerThreshold(properties, ozoneConf.getLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD, ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD_DEFAULT)); - Snapshot.setCreationGap(properties, - ozoneConf.getLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, - ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP_DEFAULT)); + Snapshot.setCreationGap(properties, SCM_RAFT_SNAPSHOT_GAP); } public static void checkRatisException(IOException e, String port, diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java index 47f4eeb12705..70aaa01a6663 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; @@ -146,7 +145,6 @@ protected static void initializeCluster(int numDatanodes) throws Exception { conf.setTimeDuration(OZONE_DIR_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, TimeUnit.MILLISECONDS); - conf.setLong(OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, 1L); conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 50, TimeUnit.MILLISECONDS); conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL, 500, TimeUnit.MILLISECONDS); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java index 6f38d89fb2b6..eb30586ec1ef 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java @@ -74,7 +74,6 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s"); conf.set(ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL, "5s"); - conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, "1"); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, MILLISECONDS); conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java index 2825683f1ac5..ae10947d52cc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java @@ -53,7 +53,6 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s"); conf.set(ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL, "5s"); - conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, "1"); cluster = MiniOzoneCluster.newHABuilder(conf) .setOMServiceId("om-service-test1") .setSCMServiceId("scm-service-test1") From 9a8d822d8ad6ae5822e00ce3b9557242cb683ccf Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Tue, 28 Apr 2026 16:06:33 +0530 Subject: [PATCH 2/2] addressing review commits --- .../apache/hadoop/hdds/scm/ha/RatisUtil.java | 3 --- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +-- .../ha/SCMHATransactionBufferMonitorTask.java | 19 +++---------------- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index 1d2c29c9193d..eb5c429861bf 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -54,8 +54,6 @@ */ public final class RatisUtil { - public static final long SCM_RAFT_SNAPSHOT_GAP = 1L; - private RatisUtil() { } @@ -233,7 +231,6 @@ private static void setRaftSnapshotProperties( Snapshot.setAutoTriggerThreshold(properties, ozoneConf.getLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD, ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD_DEFAULT)); - Snapshot.setCreationGap(properties, SCM_RAFT_SNAPSHOT_GAP); } public static void checkRatisException(IOException e, String port, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java index 0cc600160e11..01f1fee0d36d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java @@ -142,8 +142,7 @@ private void createStartTransactionBufferMonitor() { OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); SCMHATransactionBufferMonitorTask monitorTask - = new SCMHATransactionBufferMonitorTask( - transactionBuffer, ratisServer, interval); + = new SCMHATransactionBufferMonitorTask(transactionBuffer, interval); trxBufferMonitorService = new BackgroundSCMService.Builder().setClock(scm.getSystemClock()) .setScmContext(scm.getScmContext()) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java index 85faedae1c31..16ed97b0d217 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdds.scm.ha; import java.io.IOException; -import org.apache.ratis.statemachine.SnapshotInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +28,6 @@ public class SCMHATransactionBufferMonitorTask implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(SCMHATransactionBufferMonitorTask.class); - private final SCMRatisServer server; private final SCMHADBTransactionBuffer transactionBuffer; private final long flushInterval; @@ -37,30 +35,19 @@ public class SCMHATransactionBufferMonitorTask implements Runnable { * SCMService related variables. */ public SCMHATransactionBufferMonitorTask( - SCMHADBTransactionBuffer transactionBuffer, - SCMRatisServer server, long flushInterval) { + SCMHADBTransactionBuffer transactionBuffer, long flushInterval) { this.flushInterval = flushInterval; this.transactionBuffer = transactionBuffer; - this.server = server; } @Override public void run() { if (transactionBuffer.shouldFlush(flushInterval)) { LOG.debug("Running TransactionFlushTask"); - // set latest snapshot to null for force snapshot - // the value will be reset again when snapshot is taken - final SnapshotInfo lastSnapshot = transactionBuffer - .getLatestSnapshotRef().getAndSet(null); try { - server.triggerSnapshot(); + transactionBuffer.flush(); } catch (IOException e) { - LOG.error("Snapshot request is failed", e); - } finally { - // under failure case, if unable to take snapshot, its value - // is reset to previous known value - transactionBuffer.getLatestSnapshotRef().compareAndSet( - null, lastSnapshot); + LOG.error("TransactionFlushTask is failed", e); } } }