diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index a90bacc9fee..e856c927a8a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -584,15 +584,6 @@ public final class ScmConfigKeys { public static final long OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD_DEFAULT = 1000L; - /** - * the config will transfer value to ratis config - * raft.server.snapshot.creation.gap, used by ratis to take snapshot - * when manual trigger using api. - */ - public static final String OZONE_SCM_HA_RATIS_SNAPSHOT_GAP - = "ozone.scm.ha.ratis.server.snapshot.creation.gap"; - public static final long OZONE_SCM_HA_RATIS_SNAPSHOT_GAP_DEFAULT = - 1024L; public static final String OZONE_SCM_HA_RATIS_SNAPSHOT_DIR = "ozone.scm.ha.ratis.snapshot.dir"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index ea3f2b0d9f9..766cbef14b2 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4168,12 +4168,6 @@ topology cluster tree from SCM. - - ozone.scm.ha.ratis.server.snapshot.creation.gap - 1024 - SCM, OZONE - Raft snapshot gap index after which snapshot can be taken. - ozone.scm.ha.dbtransactionbuffer.flush.interval 60s diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index f2900e38f40..eb5c429861b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -231,9 +231,6 @@ private static void setRaftSnapshotProperties( Snapshot.setAutoTriggerThreshold(properties, ozoneConf.getLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD, ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD_DEFAULT)); - Snapshot.setCreationGap(properties, - ozoneConf.getLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, - ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP_DEFAULT)); } public static void checkRatisException(IOException e, String port, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java index 0cc600160e1..01f1fee0d36 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java @@ -142,8 +142,7 @@ private void createStartTransactionBufferMonitor() { OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); SCMHATransactionBufferMonitorTask monitorTask - = new SCMHATransactionBufferMonitorTask( - transactionBuffer, ratisServer, interval); + = new SCMHATransactionBufferMonitorTask(transactionBuffer, interval); trxBufferMonitorService = new BackgroundSCMService.Builder().setClock(scm.getSystemClock()) .setScmContext(scm.getScmContext()) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java index 85faedae1c3..16ed97b0d21 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHATransactionBufferMonitorTask.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdds.scm.ha; import java.io.IOException; -import org.apache.ratis.statemachine.SnapshotInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +28,6 @@ public class SCMHATransactionBufferMonitorTask implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(SCMHATransactionBufferMonitorTask.class); - private final SCMRatisServer server; private final SCMHADBTransactionBuffer transactionBuffer; private final long flushInterval; @@ -37,30 +35,19 @@ public class SCMHATransactionBufferMonitorTask implements Runnable { * SCMService related variables. */ public SCMHATransactionBufferMonitorTask( - SCMHADBTransactionBuffer transactionBuffer, - SCMRatisServer server, long flushInterval) { + SCMHADBTransactionBuffer transactionBuffer, long flushInterval) { this.flushInterval = flushInterval; this.transactionBuffer = transactionBuffer; - this.server = server; } @Override public void run() { if (transactionBuffer.shouldFlush(flushInterval)) { LOG.debug("Running TransactionFlushTask"); - // set latest snapshot to null for force snapshot - // the value will be reset again when snapshot is taken - final SnapshotInfo lastSnapshot = transactionBuffer - .getLatestSnapshotRef().getAndSet(null); try { - server.triggerSnapshot(); + transactionBuffer.flush(); } catch (IOException e) { - LOG.error("Snapshot request is failed", e); - } finally { - // under failure case, if unable to take snapshot, its value - // is reset to previous known value - transactionBuffer.getLatestSnapshotRef().compareAndSet( - null, lastSnapshot); + LOG.error("TransactionFlushTask is failed", e); } } } diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java index 47f4eeb1270..70aaa01a666 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractTestStorageDistributionEndpoint.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; @@ -146,7 +145,6 @@ protected static void initializeCluster(int numDatanodes) throws Exception { conf.setTimeDuration(OZONE_DIR_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, TimeUnit.MILLISECONDS); - conf.setLong(OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, 1L); conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 50, TimeUnit.MILLISECONDS); conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL, 500, TimeUnit.MILLISECONDS); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java index 6f38d89fb2b..eb30586ec1e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java @@ -74,7 +74,6 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s"); conf.set(ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL, "5s"); - conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, "1"); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, MILLISECONDS); conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java index 2825683f1ac..ae10947d52c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java @@ -53,7 +53,6 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s"); conf.set(ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL, "5s"); - conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, "1"); cluster = MiniOzoneCluster.newHABuilder(conf) .setOMServiceId("om-service-test1") .setSCMServiceId("scm-service-test1")