diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index 374b67488bf..84fd907bc5e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -443,13 +443,17 @@ List getDatanodeUsageInfo(String address, List getDatanodeUsageInfo( boolean mostUsed, int count) throws IOException; + @Deprecated StatusAndMessages finalizeScmUpgrade(String upgradeClientID) throws IOException; + @Deprecated StatusAndMessages queryUpgradeFinalizationProgress( String upgradeClientID, boolean force, boolean readonly) throws IOException; + void finalizeUpgrade() throws IOException; + HddsProtos.UpgradeStatus queryUpgradeStatus() throws IOException; DecommissionScmResponseProto decommissionScm( diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 7cf51afa327..9dda75e51bb 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -477,9 +477,13 @@ List getDatanodeUsageInfo( List getDatanodeUsageInfo( boolean mostUsed, int count, int clientVersion) throws IOException; + @Deprecated StatusAndMessages finalizeScmUpgrade(String upgradeClientID) throws IOException; + void finalizeUpgrade() throws IOException; + + @Deprecated StatusAndMessages queryUpgradeFinalizationProgress( String upgradeClientID, boolean force, boolean readonly) throws IOException; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 4f80bebd3d2..9b5e5fba860 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -69,6 +69,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionScmResponseProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.FinalizeScmUpgradeRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.FinalizeScmUpgradeResponseProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.FinalizeUpgradeRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitSafeModeResponseProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerCountRequestProto; @@ -1110,6 +1111,7 @@ public List getDatanodeUsageInfo( } @Override + @Deprecated public StatusAndMessages finalizeScmUpgrade(String upgradeClientID) throws IOException { FinalizeScmUpgradeRequestProto req = FinalizeScmUpgradeRequestProto. @@ -1129,6 +1131,14 @@ public StatusAndMessages finalizeScmUpgrade(String upgradeClientID) } @Override + public void finalizeUpgrade() throws IOException { + FinalizeUpgradeRequestProto req = FinalizeUpgradeRequestProto.newBuilder() + .build(); + submitRequest(Type.FinalizeUpgrade, builder -> builder.setFinalizeUpgradeRequest(req)); + } + + @Override + @Deprecated public StatusAndMessages queryUpgradeFinalizationProgress( String upgradeClientID, boolean force, boolean readonly) throws IOException { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java index 8d0a31a8a4a..d7f28a17e88 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java @@ -115,6 +115,17 @@ public StatusAndMessages finalize(String upgradeClientID, T service) } } + @Override + public void finalize(T service) throws IOException { + UpgradeFinalization.Status status = versionManager.getUpgradeState(); + if (isFinalized(status)) { + return; + } + if (status == FINALIZATION_REQUIRED) { + finalizationExecutor.execute(service, this); + } + } + @Override public synchronized StatusAndMessages reportStatus( String upgradeClientID, boolean takeover) throws UpgradeException { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalizer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalizer.java index 3109e787c6a..94aa9d34f7e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalizer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalizer.java @@ -57,6 +57,13 @@ public interface UpgradeFinalizer { StatusAndMessages finalize(String upgradeClientID, T service) throws IOException; + /** + * Finalize the metadata upgrade. If finalization is not needed or is already underway, this call is a noop. + * @param service the service on which we run finalization. + * @throws IOException if the finalization fails at any stage. + */ + void finalize(T service) throws IOException; + /** * Finalize the component if needed, and wait until completion. * @param upgradeClientID the initiating client's identifier. diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto index 93d270b51fc..1ef451f5a7f 100644 --- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto +++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto @@ -88,6 +88,7 @@ message ScmContainerLocationRequest { optional ReconcileContainerRequestProto reconcileContainerRequest = 49; optional GetDeletedBlocksTxnSummaryRequestProto getDeletedBlocksTxnSummaryRequest = 50; optional QueryUpgradeStatusRequestProto queryUpgradeStatusRequest = 51; + optional FinalizeUpgradeRequestProto finalizeUpgradeRequest = 52; } message ScmContainerLocationResponse { @@ -147,6 +148,7 @@ message ScmContainerLocationResponse { optional ReconcileContainerResponseProto reconcileContainerResponse = 49; optional GetDeletedBlocksTxnSummaryResponseProto getDeletedBlocksTxnSummaryResponse = 50; optional QueryUpgradeStatusResponseProto queryUpgradeStatusResponse = 51; + optional FinalizeUpgradeResponseProto finalizeUpgradeResponse = 52; enum Status { OK = 1; @@ -205,6 +207,7 @@ enum Type { ReconcileContainer = 45; GetDeletedBlocksTransactionSummary = 46; QueryUpgradeStatus = 47; + FinalizeUpgrade = 48; } /** @@ -580,6 +583,12 @@ message QueryUpgradeStatusResponseProto { required hadoop.hdds.UpgradeStatus status = 1; } +message FinalizeUpgradeRequestProto { +} + +message FinalizeUpgradeResponseProto { +} + message ContainerTokenSecretProto { required string ownerId = 1; required ContainerID containerId = 2; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 0dd23c9e4fc..b761f1c28c4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -754,6 +754,12 @@ public ScmContainerLocationResponse processRequest( .setStatus(Status.OK) .setQueryUpgradeStatusResponse(getQueryUpgradeStatus(request.getQueryUpgradeStatusRequest())) .build(); + case FinalizeUpgrade: + impl.finalizeUpgrade(); + return ScmContainerLocationResponse.newBuilder() + .setCmdType(request.getCmdType()) + .setStatus(Status.OK) + .build(); default: throw new IllegalArgumentException( "Unknown command type: " + request.getCmdType()); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 483d60c182a..3d9b433584f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -29,6 +29,8 @@ import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser; +import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.Status.ALREADY_FINALIZED; +import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.Status.STARTING_FINALIZATION; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; @@ -1102,31 +1104,38 @@ public ReplicationManagerReport getReplicationManagerReport() { } @Override + @Deprecated public StatusAndMessages finalizeScmUpgrade(String upgradeClientID) throws IOException { - final Map auditMap = Maps.newHashMap(); - auditMap.put("upgradeClientID", upgradeClientID); + if (scm.getLayoutVersionManager().getUpgradeState() == ALREADY_FINALIZED) { + return new StatusAndMessages(ALREADY_FINALIZED, Collections.emptyList()); + } + finalizeUpgrade(); + return new StatusAndMessages(STARTING_FINALIZATION, Collections.emptyList()); + } + + @Override + public void finalizeUpgrade() throws IOException { + final Map auditMap = Collections.emptyMap(); try { - // check admin authorization getScm().checkAdminAccess(getRemoteUser(), false); - // TODO HDDS-6762: Return to the client once the FINALIZATION_STARTED - // checkpoint has been crossed and continue finalizing asynchronously. - StatusAndMessages result = scm.getFinalizationManager().finalizeUpgrade(upgradeClientID); - AUDIT.logWriteSuccess(buildAuditMessageForSuccess( - SCMAction.FINALIZE_SCM_UPGRADE, auditMap)); - return result; + scm.getFinalizationManager().finalizeUpgrade(); + AUDIT.logWriteSuccess(buildAuditMessageForSuccess(SCMAction.FINALIZE_SCM_UPGRADE, auditMap)); } catch (Exception ex) { - AUDIT.logWriteFailure(buildAuditMessageForFailure( - SCMAction.FINALIZE_SCM_UPGRADE, auditMap, ex)); + AUDIT.logWriteFailure(buildAuditMessageForFailure(SCMAction.FINALIZE_SCM_UPGRADE, auditMap, ex)); throw ex; } - } @Override + @Deprecated public StatusAndMessages queryUpgradeFinalizationProgress( String upgradeClientID, boolean force, boolean readonly) throws IOException { + + // This method, we change to call the queryUpgradeStatus and create a StatusAndMessages object that reflects + // the state. + Map auditMap = Maps.newHashMap(); auditMap.put("upgradeClientID", upgradeClientID); auditMap.put("force", String.valueOf(force)); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java index d73e94e3909..3bdfa2dbd3e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManager.java @@ -34,6 +34,8 @@ public interface FinalizationManager { UpgradeFinalization.StatusAndMessages finalizeUpgrade(String upgradeClientID) throws IOException; + void finalizeUpgrade() throws IOException; + UpgradeFinalization.StatusAndMessages queryUpgradeFinalizationProgress( String upgradeClientID, boolean takeover, boolean readonly ) throws IOException; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java index d1870ff49c3..ccff5101898 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/FinalizationManagerImpl.java @@ -90,6 +90,12 @@ public UpgradeFinalization.StatusAndMessages finalizeUpgrade( return upgradeFinalizer.finalize(upgradeClientID, context); } + @Override + public void finalizeUpgrade() throws IOException { + Objects.requireNonNull(context, "Cannot finalize upgrade without first building the upgrade context."); + upgradeFinalizer.finalize(context); + } + @Override public UpgradeFinalization.StatusAndMessages queryUpgradeFinalizationProgress( String upgradeClientID, boolean takeover, boolean readonly diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java index 5fca596d892..fed89dc1621 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java @@ -18,15 +18,12 @@ package org.apache.hadoop.hdds.scm.server.upgrade; import java.io.IOException; -import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.ozone.upgrade.BasicUpgradeFinalizer; import org.apache.hadoop.ozone.upgrade.LayoutFeature; import org.apache.hadoop.ozone.upgrade.UpgradeException; import org.apache.hadoop.ozone.upgrade.UpgradeFinalizationExecutor; -import org.apache.ratis.protocol.exceptions.NotLeaderException; /** * UpgradeFinalizer for the Storage Container Manager service. @@ -85,59 +82,4 @@ void replicatedFinalizationSteps(HDDSLayoutFeature lf, lf.scmAction(), context.getStorage()); } - - @Override - public void postFinalizeUpgrade(SCMUpgradeFinalizationContext context) throws IOException { - waitForDatanodesToFinalize(context); - super.postFinalizeUpgrade(context); - } - - /** - * Wait for all HEALTHY datanodes to complete finalization before finishing - * SCM finalization. This ensures that when the client receives a - * FINALIZATION_DONE status, all healthy datanodes have also finalized. - * - * A datanode is considered finalized when its metadata layout version (MLV) - * equals its software layout version (SLV), indicating it has completed - * processing all layout features. - * - * @param context The finalization context containing node manager reference - * @throws SCMException if waiting is interrupted or SCM loses leadership - * @throws NotLeaderException if SCM is no longer the leader - */ - private void waitForDatanodesToFinalize(SCMUpgradeFinalizationContext context) - throws SCMException, NotLeaderException { - NodeManager nodeManager = context.getNodeManager(); - - LOG.info("Waiting for all HEALTHY datanodes to complete finalization before finishing SCM finalization."); - - boolean allDatanodesFinalized = false; - while (!allDatanodesFinalized) { - // Break out of the wait and step down from driving finalization if this - // SCM is no longer the leader by throwing NotLeaderException. - context.getSCMContext().getTermOfLeader(); - - NodeManager.DatanodeFinalizationCounts datanodeFinalizationCounts = nodeManager.getDatanodeFinalizationCounts(); - int finalizedNodes = datanodeFinalizationCounts.getNumFinalizedDatanodes(); - int totalHealthyNodes = datanodeFinalizationCounts.getTotalHealthyDatanodes(); - allDatanodesFinalized = datanodeFinalizationCounts.allNodesFinalized(); - - if (!allDatanodesFinalized) { - int unfinalizedNodes = totalHealthyNodes - finalizedNodes; - LOG.info("Waiting for datanodes to finalize. Status: {}/{} healthy " + - "datanodes have finalized ({} remaining).", - finalizedNodes, totalHealthyNodes, unfinalizedNodes); - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SCMException("Interrupted while waiting for datanodes to " + - "finalize.", SCMException.ResultCodes.INTERNAL_ERROR); - } - } else { - LOG.info("All {} HEALTHY datanodes have completed finalization.", - totalHealthyNodes); - } - } - } } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index c30ec7937c9..40ba3f30c4c 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -582,6 +582,11 @@ public StatusAndMessages queryUpgradeFinalizationProgress( upgradeClientID, force, readonly); } + @Override + public void finalizeUpgrade() throws IOException { + storageContainerLocationClient.finalizeUpgrade(); + } + @Override public HddsProtos.UpgradeStatus queryUpgradeStatus() throws IOException { return storageContainerLocationClient.queryUpgradeStatus(); diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java new file mode 100644 index 00000000000..e882f3e537c --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.upgrade; + +import java.io.IOException; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.cli.ScmSubcommand; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import picocli.CommandLine; + +/** + * Sub command to finalize a cluster upgrade. + */ +@CommandLine.Command( + name = "finalize", + description = "Finalize a cluster upgrade", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class FinalizeSubCommand extends ScmSubcommand { + + @Override + public void execute(ScmClient client) throws IOException { + client.finalizeUpgrade(); + + out().println("Cluster finalization has been started. Monitor progress with `ozone admin upgrade status`"); + } +} + diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeCommands.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeCommands.java index a1372e16951..08f928e464a 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeCommands.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeCommands.java @@ -31,6 +31,7 @@ mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class, subcommands = { + FinalizeSubCommand.class, StatusSubCommand.class }) @MetaInfServices(AdminSubcommand.class) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeUtil.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeUtil.java new file mode 100644 index 00000000000..1e248275fd5 --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/UpgradeUtil.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.upgrade; + +import java.io.IOException; +import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; + +/** + * Static utility methods for upgrade commands. + */ +public final class UpgradeUtil { + + private UpgradeUtil() { } + + /** Utility method to test if the upgrade has completed (finalized) or not. + * + * @param scmClient + * @return True if the cluster has completed the upgrade and is finalized. + * @throws IOException + */ + public static boolean isFinalizationComplete( + StorageContainerLocationProtocol scmClient) throws IOException { + return scmClient.queryUpgradeStatus().getShouldFinalize(); + } + +} diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java new file mode 100644 index 00000000000..a380f5388ca --- /dev/null +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.upgrade; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import picocli.CommandLine; + +/** + * Unit tests for {@link FinalizeSubCommand}. + */ +public class TestFinalizeSubCommand { + + private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name(); + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private FinalizeSubCommand cmd; + + @BeforeEach + public void setup() throws UnsupportedEncodingException { + cmd = new FinalizeSubCommand(); + System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING)); + } + + @AfterEach + public void tearDown() { + System.setOut(originalOut); + } + + @Test + public void testCommandRunsAndPrintsOutput() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + + new CommandLine(cmd).parseArgs(); + cmd.execute(scmClient); + + String output = outContent.toString(DEFAULT_ENCODING); + assertTrue(output.contains("Cluster finalization has been started")); + verify(scmClient).finalizeUpgrade(); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java index d714a955b0a..268f5a2dfc7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java @@ -24,12 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.IOException; import java.time.Duration; import java.util.List; import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfig; @@ -55,17 +52,11 @@ import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Tests upgrade finalization failure scenarios and corner cases specific to DN data distribution feature. */ public class TestDNDataDistributionFinalization { - private static final String CLIENT_ID = UUID.randomUUID().toString(); - private static final Logger LOG = - LoggerFactory.getLogger(TestDNDataDistributionFinalization.class); - private StorageContainerLocationProtocol scmClient; private MiniOzoneHAClusterImpl cluster; @@ -167,20 +158,9 @@ public void testDataDistributionUpgradeScenario() throws Exception { // Validate pre-finalization state validatePreDataDistributionFeatureState(); - // Now trigger finalization - Future finalizationFuture = Executors.newSingleThreadExecutor().submit( - () -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - LOG.info("finalization client failed. This may be expected if the" + - " test injected failures.", ex); - } - }); - // Wait for finalization to complete - finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); + TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Verify finalization completed assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), @@ -215,18 +195,8 @@ public void testMissingPendingDeleteMetadataRecalculation() throws Exception { out.write(data); } bucket.deleteKey(keyName); - Future finalizationFuture = Executors.newSingleThreadExecutor().submit( - () -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - LOG.info("finalization client failed. This may be expected if the" + - " test injected failures.", ex); - } - }); - // Wait for finalization - finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); + TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java index ec319ea07c5..9c2e8c7514a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHddsUpgradeUtils.java @@ -34,8 +34,8 @@ import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.HddsDatanodeService; +import org.apache.hadoop.ozone.admin.upgrade.UpgradeUtil; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.LambdaTestUtils; import org.slf4j.Logger; @@ -50,16 +50,11 @@ public final class TestHddsUpgradeUtils { private TestHddsUpgradeUtils() { } - public static void waitForFinalizationFromClient( - StorageContainerLocationProtocol scmClient, String clientID) - throws Exception { + public static void waitForFinalizationFromClient(StorageContainerLocationProtocol scmClient) throws Exception { LambdaTestUtils.await(60_000, 1_000, () -> { - UpgradeFinalization.Status status = scmClient - .queryUpgradeFinalizationProgress(clientID, true, true) - .status(); - LOG.info("Waiting for upgrade finalization to complete from client." + - " Current status is {}.", status); - return status == FINALIZATION_DONE || status == ALREADY_FINALIZED; + boolean isComplete = UpgradeUtil.isFinalizationComplete(scmClient); + LOG.info("Waiting for upgrade finalization to complete from client.Current status is {}.", isComplete); + return isComplete; }); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java index b09217facc2..71cd47570a7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java @@ -43,8 +43,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.client.BlockID; @@ -90,7 +88,6 @@ * Tests upgrade finalization failure scenarios and corner cases specific to SCM data distribution feature. */ public class TestScmDataDistributionFinalization { - private static final String CLIENT_ID = UUID.randomUUID().toString(); private static final Logger LOG = LoggerFactory.getLogger(TestScmDataDistributionFinalization.class); @@ -98,7 +95,6 @@ public class TestScmDataDistributionFinalization { private MiniOzoneHAClusterImpl cluster; private static final int NUM_DATANODES = 3; private static final int NUM_SCMS = 3; - private Future finalizationFuture; private final String volumeName = UUID.randomUUID().toString(); private final String bucketName = UUID.randomUUID().toString(); private OzoneBucket bucket; @@ -158,22 +154,6 @@ public void init(OzoneConfiguration conf, volume.createBucket(bucketName, builder.build()); bucket = volume.getBucket(bucketName); } - - // Launch finalization from the client. In the current implementation, - // this call will block until finalization completes. If the test - // involves restarts or leader changes the client may be disconnected, - // but finalization should still proceed. - if (doFinalize) { - finalizationFuture = Executors.newSingleThreadExecutor().submit( - () -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - LOG.info("finalization client failed. This may be expected if the" + - " test injected failures.", ex); - } - }); - } } @AfterEach @@ -192,8 +172,8 @@ public void testFinalizationEmptyClusterDataDistribution() throws Exception { init(new OzoneConfiguration(), null, true); assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); - finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); + TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), @@ -297,17 +277,8 @@ public void testFinalizationNonEmptyClusterDataDistribution() throws Exception { flushDBTransactionBuffer(activeSCM); assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); - finalizationFuture = Executors.newSingleThreadExecutor().submit( - () -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - LOG.info("finalization client failed. This may be expected if the" + - " test injected failures.", ex); - } - }); - finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); + TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java index 86cc746ef40..b5b70eda97b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java @@ -19,13 +19,9 @@ import static org.assertj.core.api.Assertions.assertThat; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -53,7 +49,6 @@ * HA. */ public class TestScmHAFinalization { - private static final String CLIENT_ID = UUID.randomUUID().toString(); private static final Logger LOG = LoggerFactory.getLogger(TestScmHAFinalization.class); @@ -61,7 +56,6 @@ public class TestScmHAFinalization { private MiniOzoneHAClusterImpl cluster; private static final int NUM_DATANODES = 3; private static final int NUM_SCMS = 3; - private Future finalizationFuture; public void init(OzoneConfiguration conf, UpgradeFinalizationExecutor executor, @@ -88,20 +82,6 @@ public void init(OzoneConfiguration conf, scmClient = cluster.getStorageContainerLocationClient(); cluster.waitForClusterToBeReady(); - - // Launch finalization from the client. In the current implementation, - // this call will block until finalization completes. If the test - // involves restarts or leader changes the client may be disconnected, - // but finalization should still proceed. - finalizationFuture = Executors.newSingleThreadExecutor().submit( - () -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - LOG.info("finalization client failed. This may be expected if the" + - " test injected failures.", ex); - } - }); } @AfterEach @@ -115,8 +95,8 @@ public void shutdown() { public void testFinalization() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); init(conf, new DefaultUpgradeFinalizationExecutor<>(), 0); - finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); + TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Ensure all SCMs finalize, indicating the message has been propagated across them all waitForScmsToFinalize(cluster.getStorageContainerManagersList()); @@ -153,8 +133,8 @@ public void testSnapshotFinalization() throws Exception { } // Wait for finalization from the client perspective. - finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); + TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); // Wait for two running SCMs to finish finalization. waitForScmsToFinalize(activeScms); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java index dd2bde87ec4..aa14184c2b6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.timeout; @@ -32,8 +31,6 @@ import java.util.HashMap; import java.util.List; import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.hadoop.hdds.client.ECReplicationConfig; @@ -69,7 +66,6 @@ * DeletionService test to Pass Usage from OM to SCM. */ public class TestBlockDeletionService { - private static final String CLIENT_ID = UUID.randomUUID().toString(); private static final String VOLUME_NAME = "vol1"; private static final String BUCKET_NAME = "bucket1"; private static final int KEY_SIZE = 5 * 1024; // 5 KB @@ -142,16 +138,8 @@ public void testDeleteKeyQuotaWithUpgrade() throws Exception { GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000); // UPGRADE SCM (if specified) - // Step 5: wait for finalizing upgrade - Future finalizationFuture = Executors.newSingleThreadExecutor().submit(() -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - fail("finalization client failed", ex); - } - }); - finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); + TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); assertEquals(STORAGE_SPACE_DISTRIBUTION.ordinal(), cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());