Skip to content

HDDS-15065. Reduce Ratis snapshot gap to fix SCM flush delay. #10100

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
priyeshkaratha wants to merge 2 commits into
base: master
Choose a base branch
from
Open

HDDS-15065. Reduce Ratis snapshot gap to fix SCM flush delay. #10100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d7781df
HDDS-15065. Reduce Ratis snapshot gap to fix SCM flush delay.
priyeshkaratha Apr 21, 2026
9a8d822
addressing review commits
priyeshkaratha Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -584,15 +584,6 @@ public final class ScmConfigKeys {
public static final long OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD_DEFAULT =
1000L;

/**
* the config will transfer value to ratis config
* raft.server.snapshot.creation.gap, used by ratis to take snapshot
* when manual trigger using api.
*/
public static final String OZONE_SCM_HA_RATIS_SNAPSHOT_GAP
= "ozone.scm.ha.ratis.server.snapshot.creation.gap";
public static final long OZONE_SCM_HA_RATIS_SNAPSHOT_GAP_DEFAULT =
1024L;
public static final String OZONE_SCM_HA_RATIS_SNAPSHOT_DIR =
"ozone.scm.ha.ratis.snapshot.dir";

Expand Down
6 changes: 0 additions & 6 deletions hadoop-hdds/common/src/main/resources/ozone-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4168,12 +4168,6 @@
topology cluster tree from SCM.
</description>
</property>
<property>
<name>ozone.scm.ha.ratis.server.snapshot.creation.gap</name>
<value>1024</value>
<tag>SCM, OZONE</tag>
<description>Raft snapshot gap index after which snapshot can be taken.</description>
</property>
<property>
<name>ozone.scm.ha.dbtransactionbuffer.flush.interval</name>
<value>60s</value>
Expand Down
3 changes: 0 additions & 3 deletions hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,9 +231,6 @@ private static void setRaftSnapshotProperties(
Snapshot.setAutoTriggerThreshold(properties,
ozoneConf.getLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD,
ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD_DEFAULT));
Snapshot.setCreationGap(properties,
ozoneConf.getLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP,
ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP_DEFAULT));
}

public static void checkRatisException(IOException e, String port,
Expand Down
3 changes: 1 addition & 2 deletions hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,7 @@ private void createStartTransactionBufferMonitor() {
OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
SCMHATransactionBufferMonitorTask monitorTask
= new SCMHATransactionBufferMonitorTask(
transactionBuffer, ratisServer, interval);
= new SCMHATransactionBufferMonitorTask(transactionBuffer, interval);
trxBufferMonitorService =
new BackgroundSCMService.Builder().setClock(scm.getSystemClock())
.setScmContext(scm.getScmContext())
Expand Down
19 changes: 3 additions & 16 deletions ...er-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.hadoop.hdds.scm.ha;

import java.io.IOException;
import org.apache.ratis.statemachine.SnapshotInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -29,38 +28,26 @@
public class SCMHATransactionBufferMonitorTask implements Runnable {
private static final Logger LOG =
LoggerFactory.getLogger(SCMHATransactionBufferMonitorTask.class);
private final SCMRatisServer server;
private final SCMHADBTransactionBuffer transactionBuffer;
private final long flushInterval;

/**
* SCMService related variables.
*/
public SCMHATransactionBufferMonitorTask(
SCMHADBTransactionBuffer transactionBuffer,
SCMRatisServer server, long flushInterval) {
SCMHADBTransactionBuffer transactionBuffer, long flushInterval) {
this.flushInterval = flushInterval;
this.transactionBuffer = transactionBuffer;
this.server = server;
}

@Override
public void run() {
if (transactionBuffer.shouldFlush(flushInterval)) {
LOG.debug("Running TransactionFlushTask");
// set latest snapshot to null for force snapshot
// the value will be reset again when snapshot is taken
final SnapshotInfo lastSnapshot = transactionBuffer
.getLatestSnapshotRef().getAndSet(null);
try {
server.triggerSnapshot();
transactionBuffer.flush();
} catch (IOException e) {
LOG.error("Snapshot request is failed", e);
} finally {
// under failure case, if unable to take snapshot, its value
// is reset to previous known value
transactionBuffer.getLatestSnapshotRef().compareAndSet(
null, lastSnapshot);
LOG.error("TransactionFlushTask is failed", e);
}
}
}
Expand Down
2 changes: 0 additions & 2 deletions .../src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
Expand Down Expand Up @@ -146,7 +145,6 @@ protected static void initializeCluster(int numDatanodes) throws Exception {
conf.setTimeDuration(OZONE_DIR_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, TimeUnit.MILLISECONDS);
conf.setLong(OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, 1L);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 50, TimeUnit.MILLISECONDS);
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL, 500, TimeUnit.MILLISECONDS);
Expand Down
1 change: 0 additions & 1 deletion ...on-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ public void init() throws Exception {
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
conf.set(ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL,
"5s");
conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, "1");
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
Expand Down
1 change: 0 additions & 1 deletion ...egration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ public void init() throws Exception {
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
conf.set(ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL,
"5s");
conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, "1");
cluster = MiniOzoneCluster.newHABuilder(conf)
.setOMServiceId("om-service-test1")
.setSCMServiceId("scm-service-test1")
Expand Down