From 536b5950b7c154afdd5f22154a0809664bfa6b32 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 3 Dec 2025 11:38:41 +0800 Subject: [PATCH 1/5] HDDS-12791. Make Retention Service suspendable --- .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../om/protocol/OzoneManagerProtocol.java | 11 +++ ...ManagerProtocolClientSideTranslatorPB.java | 16 ++++ .../ozone/reconfig/TestOmReconfiguration.java | 2 + .../hadoop/ozone/shell/TestOzoneShellHA.java | 9 ++ .../src/main/proto/OmClientProtocol.proto | 15 ++++ .../apache/hadoop/ozone/om/OzoneManager.java | 35 +++++++- .../ozone/om/service/KeyLifecycleService.java | 37 +++++++- .../OzoneManagerRequestHandler.java | 7 ++ .../om/service/TestKeyLifecycleService.java | 44 ++++++++++ .../om/FinalizationStatusSubCommand.java | 12 +-- .../admin/om/LifecycleStatusSubCommand.java | 88 +++++++++++++++++++ .../ozone/admin/om/LifecycleSubCommand.java | 45 ++++++++++ .../apache/hadoop/ozone/admin/om/OMAdmin.java | 3 +- 14 files changed, 313 insertions(+), 12 deletions(-) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 9a945dfea87e..ce6638dce07e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -278,6 +278,7 @@ public static boolean isReadOnly( case GetQuotaRepairStatus: case StartQuotaRepair: case GetLifecycleConfiguration: + case GetLifecycleServiceStatus: return true; case CreateVolume: case SetVolumeProperty: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 626bef650446..e42d64e8451c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -66,6 +66,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CancelPrepareResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.EchoRPCResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse.PrepareStatus; @@ -1226,4 +1227,14 @@ default void deleteLifecycleConfiguration(String volumeName, throw new UnsupportedOperationException("OzoneManager does not require " + "this to be implemented, as write requests use a new approach."); } + + /** + * Gets the lifecycle service status. + * @return GetLifecycleServiceStatusResponse + * @throws IOException + */ + default GetLifecycleServiceStatusResponse getLifecycleServiceStatus() throws IOException { + throw new UnsupportedOperationException("OzoneManager does not require " + + "this to be implemented, as write requests use a new approach."); + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 3a2d41c0caf9..dbacce5377cc 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -141,6 +141,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleConfigurationRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleConfigurationResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetObjectTaggingRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetObjectTaggingResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretRequest; @@ -2691,6 +2692,21 @@ public OmLifecycleConfiguration getLifecycleConfiguration(String volumeName, resp.getLifecycleConfiguration()); } + @Override + public GetLifecycleServiceStatusResponse getLifecycleServiceStatus() throws IOException { + OzoneManagerProtocolProtos.GetLifecycleServiceStatusRequest + getLifecycleServiceStatusRequest = + OzoneManagerProtocolProtos.GetLifecycleServiceStatusRequest + .newBuilder().build(); + + OMRequest omRequest = createOMRequest(Type.GetLifecycleServiceStatus) + .setGetLifecycleServiceStatusRequest(getLifecycleServiceStatusRequest) + .build(); + + return handleError(submitRequest(omRequest)) + .getGetLifecycleServiceStatusResponse(); + } + @Override public void setLifecycleConfiguration( OmLifecycleConfiguration omLifecycleConfiguration) throws IOException { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java index 258325ce070c..04746b586647 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_READONLY_ADMINISTRATORS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED_DEFAULT; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -51,6 +52,7 @@ void reconfigurableProperties() { .add(OZONE_KEY_DELETING_LIMIT_PER_TASK) .add(OZONE_OM_VOLUME_LISTALL_ALLOWED) .add(OZONE_READONLY_ADMINISTRATORS) + .add(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED) .addAll(new OmConfig().reconfigurableProperties()) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index d7fbce84f1ee..61dbaf71fd0c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -1834,6 +1834,15 @@ public void testSetEncryptionKey() throws Exception { assertEquals(newEncKey, volume.getBucket("bucket0").getEncryptionKeyName()); } + @Test + public void testLifecycleStatus() throws UnsupportedEncodingException { + String[] args = new String[] {"om", "lifecycle", "status", "--service-id", omServiceId}; + execute(ozoneAdminShell, args); + String output = out.toString(DEFAULT_ENCODING); + assertThat(output).contains("IsEnabled:"); + assertThat(output).contains("IsRunning:"); + } + @Test public void testCreateBucketWithECReplicationConfigWithoutReplicationParam() { getVolume("volume102"); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index ba35dab8470f..05c2cdb2f3d3 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -159,6 +159,7 @@ enum Type { SetLifecycleConfiguration = 150; GetLifecycleConfiguration = 151; DeleteLifecycleConfiguration = 152; + GetLifecycleServiceStatus = 153; } enum SafeMode { @@ -310,6 +311,8 @@ message OMRequest { optional SetLifecycleConfigurationRequest setLifecycleConfigurationRequest = 150; optional GetLifecycleConfigurationRequest getLifecycleConfigurationRequest = 151; optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 152; + + optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 153; } message OMResponse { @@ -447,6 +450,8 @@ message OMResponse { optional SetLifecycleConfigurationResponse setLifecycleConfigurationResponse = 150; optional GetLifecycleConfigurationResponse getLifecycleConfigurationResponse = 151; optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 152; + + optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 153; } enum Status { @@ -2432,3 +2437,13 @@ service OzoneManagerService { rpc submitRequest(OMRequest) returns(OMResponse); } + +message GetLifecycleServiceStatusRequest { +} + +message GetLifecycleServiceStatusResponse { + required bool isEnabled = 1; + required bool isRunning = 2; + repeated string runningBuckets = 3; +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index b78368a441fd..8ee257e7a0c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -53,6 +53,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INITIAL_DELAY_MS_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INITIAL_DELAY_MS_KEY; @@ -283,6 +285,7 @@ import org.apache.hadoop.ozone.om.s3.S3SecretCacheProvider; import org.apache.hadoop.ozone.om.s3.S3SecretStoreProvider; import org.apache.hadoop.ozone.om.service.CompactDBService; +import org.apache.hadoop.ozone.om.service.KeyLifecycleService; import org.apache.hadoop.ozone.om.service.OMRangerBGSyncService; import org.apache.hadoop.ozone.om.service.QuotaRepairTask; import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils; @@ -295,6 +298,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.EchoRPCResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExtendedUserAccessIdInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; @@ -540,7 +544,9 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) this::reconfOzoneReadOnlyAdmins) .register(OZONE_OM_VOLUME_LISTALL_ALLOWED, this::reconfigureAllowListAllVolumes) .register(OZONE_KEY_DELETING_LIMIT_PER_TASK, - this::reconfOzoneKeyDeletingLimitPerTask); + this::reconfOzoneKeyDeletingLimitPerTask) + .register(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, + this::reconfKeyLifecycleServiceEnabled); versionManager = new OMLayoutVersionManager(omStorage.getLayoutVersion()); upgradeFinalizer = new OMUpgradeFinalizer(versionManager); @@ -3159,6 +3165,19 @@ public OmLifecycleConfiguration getLifecycleConfiguration(String volumeName, } } + @Override + public GetLifecycleServiceStatusResponse getLifecycleServiceStatus() { + KeyLifecycleService keyLifecycleService = keyManager.getKeyLifecycleService(); + if (keyLifecycleService == null) { + return GetLifecycleServiceStatusResponse.newBuilder() + .setIsEnabled(getConfiguration().getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, + OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT)) + .setIsRunning(false) + .build(); + } + return keyLifecycleService.status(); + } + private Map buildAuditMap(String volume) { Map auditMap = new LinkedHashMap<>(); auditMap.put(OzoneConsts.VOLUME, volume); @@ -5157,6 +5176,20 @@ private String reconfOzoneKeyDeletingLimitPerTask(String newVal) { return newVal; } + private String reconfKeyLifecycleServiceEnabled(String newVal) { + boolean enabled = StringUtils.isEmpty(newVal) ? + OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT : Boolean.parseBoolean(newVal); + KeyLifecycleService keyLifecycleService = getKeyManager().getKeyLifecycleService(); + if (keyLifecycleService != null) { + keyLifecycleService.setServiceEnabled(enabled); + } else { + LOG.warn("Failed reconfiguration for '{}'. KeyLifecycleService is not initialized.", + OZONE_KEY_LIFECYCLE_SERVICE_ENABLED); + } + getConfiguration().setBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, enabled); + return newVal; + } + private String reconfigureAllowListAllVolumes(String newVal) { getConfiguration().set(OZONE_OM_VOLUME_LISTALL_ALLOWED, newVal); setAllowListAllVolumesFromConfig(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index 680ba2bdbb0c..d6721a320a1a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java @@ -43,6 +43,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -84,6 +85,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyError; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest; @@ -105,8 +107,8 @@ public class KeyLifecycleService extends BackgroundService { private int listMaxSize; private long cachedDirMaxCount; private final AtomicBoolean suspended; + private final AtomicBoolean isServiceEnabled; private KeyLifecycleServiceMetrics metrics; - private boolean isServiceEnabled; // A set of bucket name that have LifecycleActionTask scheduled private final ConcurrentHashMap inFlight; private OMMetadataManager omMetadataManager; @@ -132,8 +134,8 @@ public KeyLifecycleService(OzoneManager ozoneManager, OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT); this.suspended = new AtomicBoolean(false); this.metrics = KeyLifecycleServiceMetrics.create(); - this.isServiceEnabled = conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, - OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT); + this.isServiceEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, + OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT)); this.inFlight = new ConcurrentHashMap(); this.omMetadataManager = ozoneManager.getMetadataManager(); int limit = (int) conf.getStorageSize( @@ -192,7 +194,7 @@ private boolean shouldRun() { // OzoneManager can be null for testing return true; } - return isServiceEnabled && !suspended.get() && getOzoneManager().isLeaderReady(); + return isServiceEnabled.get() && !suspended.get() && getOzoneManager().isLeaderReady(); } public KeyLifecycleServiceMetrics getMetrics() { @@ -219,12 +221,26 @@ public void resume() { suspended.set(false); } + public void setServiceEnabled(boolean enabled) { + this.isServiceEnabled.set(enabled); + LOG.info("KeyLifecycleService is {}", enabled ? "enabled" : "disabled"); + } + @Override public void shutdown() { super.shutdown(); KeyLifecycleServiceMetrics.unregister(); } + public GetLifecycleServiceStatusResponse status() { + Set runningBuckets = new HashSet<>(inFlight.keySet()); + return GetLifecycleServiceStatusResponse.newBuilder() + .setIsRunning(!runningBuckets.isEmpty()) + .setIsEnabled(isServiceEnabled.get()) + .addAllRunningBuckets(runningBuckets) + .build(); + } + /** * A lifecycle action task for one specific bucket, scanning OM DB and evaluating if any existing * object/key qualified for expiration according to bucket's lifecycle configuration, and sending @@ -331,6 +347,11 @@ public BackgroundTaskResult call() { return result; } + if (!shouldRun()) { + onSuccess(bucketKey); + return result; + } + LOG.info("{} expired keys and {} expired dirs found and remained for bucket {}", expiredKeyList.size(), expiredDirList.size(), bucketKey); @@ -470,6 +491,11 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table HashSet deletedDirSet = new HashSet<>(); while (!stack.isEmpty()) { + if (!shouldRun()) { + LOG.info("KeyLifecycleService is suspended or disabled." + + "Stopping LifecycleActionTask for bucket {}.", bucketName); + return; + } PendingEvaluateDirectory item = stack.pop(); OmDirectoryInfo currentDir = item.getDirectoryInfo(); String currentDirPath = item.getDirPath(); @@ -698,6 +724,9 @@ private void evaluateBucket(OmBucketInfo bucketInfo, try (TableIterator> keyTblItr = keyTable.iterator(omMetadataManager.getBucketKey(volumeName, bucketName))) { while (keyTblItr.hasNext()) { + if (!shouldRun()) { + return; + } Table.KeyValue keyValue = keyTblItr.next(); OmKeyInfo key = keyValue.getValue(); numKeyIterated++; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index debbbebf874c..eaa578f61362 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -116,6 +116,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleConfigurationRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleConfigurationResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetObjectTaggingRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetObjectTaggingResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse; @@ -402,6 +403,12 @@ public OMResponse handleReadRequest(OMRequest request) { responseBuilder.setGetLifecycleConfigurationResponse( getLifecycleConfigurationResponse); break; + case GetLifecycleServiceStatus: + GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = + impl.getLifecycleServiceStatus(); + responseBuilder.setGetLifecycleServiceStatusResponse( + getLifecycleServiceStatusResponse); + break; default: responseBuilder.setSuccess(false); responseBuilder.setMessage("Unrecognized Command Type: " + cmdType); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java index 8d9d457ef61f..189e501363f9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java @@ -101,6 +101,7 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.request.key.OMKeysDeleteRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleConfiguration; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; @@ -114,6 +115,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; @@ -1537,6 +1539,48 @@ void testMultipleDirectoriesMatched(String keyPrefix1, String keyPrefix2, String } deleteLifecyclePolicy(volumeName, bucketName); } + + @Test + void testGetLifecycleServiceStatus() throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String prefix = "key"; + + //Sservice should be enabled but not running + OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse status = + om.getLifecycleServiceStatus(); + assertTrue(status.getIsEnabled()); + assertFalse(status.getIsRunning()); + assertEquals(0, status.getRunningBucketsCount()); + + // Create and inject for test + createKeys(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, KEY_COUNT, 1, prefix, null); + ZonedDateTime date = ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(EXPIRE_SECONDS); + KeyLifecycleService.setInjectors( + Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl())); + createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, prefix, null, date.toString(), true); + Thread.sleep(SERVICE_INTERVAL + 100); + + // Verify service is running and processing the bucket + status = om.getLifecycleServiceStatus(); + assertTrue(status.getIsEnabled()); + assertTrue(status.getIsRunning()); + assertEquals(1, status.getRunningBucketsCount()); + assertTrue(status.getRunningBucketsList().contains("/" + volumeName + "/" + bucketName)); + + KeyLifecycleService.getInjector(0).resume(); + KeyLifecycleService.getInjector(1).resume(); + GenericTestUtils.waitFor(() -> !om.getLifecycleServiceStatus().getIsRunning(), + WAIT_CHECK_INTERVAL, 10000); + + // Verify service completed and is no longer running + status = om.getLifecycleServiceStatus(); + assertTrue(status.getIsEnabled()); + assertFalse(status.getIsRunning()); + assertEquals(0, status.getRunningBucketsCount()); + + deleteLifecyclePolicy(volumeName, bucketName); + } } /** diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java index dcde6618967d..580fc160364a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java @@ -52,12 +52,12 @@ public class FinalizationStatusSubCommand implements Callable { @Override public Void call() throws Exception { - OzoneManagerProtocol client = - parent.createOmClient(omServiceId, omHost, false); - String upgradeClientID = "Upgrade-Client-" + UUID.randomUUID().toString(); - UpgradeFinalization.StatusAndMessages progress = - client.queryUpgradeFinalizationProgress(upgradeClientID, false, true); - System.out.println(progress.status()); + try (OzoneManagerProtocol client = parent.createOmClient(omServiceId, omHost, false)) { + String upgradeClientID = "Upgrade-Client-" + UUID.randomUUID().toString(); + UpgradeFinalization.StatusAndMessages progress = + client.queryUpgradeFinalizationProgress(upgradeClientID, false, true); + System.out.println(progress.status()); + } return null; } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java new file mode 100644 index 000000000000..2de1b567aad7 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.om; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +/** + * Handler of ozone admin status command. + */ +@Command( + name = "status", + description = "Check Lifecycle Service status", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class LifecycleStatusSubCommand implements Callable { + + @CommandLine.ParentCommand + private LifecycleSubCommand parent; + + @CommandLine.Option( + names = {"-id", "--service-id"}, + description = "Ozone Manager Service ID" + ) + private String omServiceId; + + @CommandLine.Option( + names = {"-host", "--service-host"}, + description = "Ozone Manager Host" + ) + private String omHost; + + @Override + public Void call() throws Exception { + try (OzoneManagerProtocol ozoneManagerClient = + parent.getParent().createOmClient(omServiceId, omHost, false)) { + GetLifecycleServiceStatusResponse lifecycleServiceStatus = + ozoneManagerClient.getLifecycleServiceStatus(); + output(lifecycleServiceStatus); + } + return null; + } + + protected void output(GetLifecycleServiceStatusResponse status) throws IOException { + PrintStream out = out(); + out.println("========================================"); + out.println(" Lifecycle Service Status"); + out.println("========================================"); + out.printf("IsEnabled: %s%n", status.getIsEnabled()); + out.printf("IsRunning: %s%n", status.getIsRunning()); + + if (status.getRunningBucketsCount() > 0) { + out.println("Running Buckets:"); + for (String bucket : status.getRunningBucketsList()) { + out.printf(" - %s%n", bucket); + } + } else { + out.println("No buckets are currently being processed."); + } + out.println("========================================"); + } + + protected PrintStream out() { + return System.out; + } +} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java new file mode 100644 index 000000000000..a6f2aeac5c18 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.om; + +import org.apache.hadoop.hdds.cli.AdminSubcommand; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import picocli.CommandLine; + +/** + * Subcommand to admin operations related to Lifecycle Service. + */ +@CommandLine.Command( + name = "lifecycle", + description = "Ozone Manager Lifecycle Service specific admin operations", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class, + subcommands = { + LifecycleStatusSubCommand.class, + }) +public class LifecycleSubCommand implements AdminSubcommand { + + @CommandLine.ParentCommand + private OMAdmin parent; + + public OMAdmin getParent() { + return parent; + } + +} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java index d536b81be140..1ba095390d3e 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java @@ -59,7 +59,8 @@ UpdateRangerSubcommand.class, TransferOmLeaderSubCommand.class, FetchKeySubCommand.class, - LeaseSubCommand.class + LeaseSubCommand.class, + LifecycleSubCommand.class }) @MetaInfServices(AdminSubcommand.class) public class OMAdmin implements AdminSubcommand { From b8b52861b6f6ec9dbd32e17c85aa42de3af90aa9 Mon Sep 17 00:00:00 2001 From: "pony.chen" Date: Tue, 16 Dec 2025 16:51:42 +0800 Subject: [PATCH 2/5] Add suspend lifecycle service command --- .../om/protocol/OzoneManagerProtocol.java | 11 +++ ...ManagerProtocolClientSideTranslatorPB.java | 14 ++++ .../hadoop/ozone/shell/TestOzoneShellHA.java | 36 +++++++++ .../src/main/proto/OmClientProtocol.proto | 9 +++ .../apache/hadoop/ozone/audit/OMAction.java | 4 +- .../ozone/om/helpers/OMAuditLogger.java | 1 + .../ratis/utils/OzoneManagerRatisUtils.java | 3 + .../OMLifecycleServiceSuspendRequest.java | 77 ++++++++++++++++++ .../OMLifecycleServiceSuspendResponse.java | 46 +++++++++++ .../ozone/admin/om/LifecycleSubCommand.java | 1 + .../admin/om/LifecycleSuspendSubCommand.java | 78 +++++++++++++++++++ 11 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleServiceSuspendResponse.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index e42d64e8451c..d405b95b6eca 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -1237,4 +1237,15 @@ default GetLifecycleServiceStatusResponse getLifecycleServiceStatus() throws IOE throw new UnsupportedOperationException("OzoneManager does not require " + "this to be implemented, as write requests use a new approach."); } + + /** + * Suspends the lifecycle service. + * The service will remain suspended until OM restarts, at which point it will + * be re-enabled based on the configuration. + * @throws IOException + */ + default void suspendLifecycleService() throws IOException { + throw new UnsupportedOperationException("OzoneManager does not require " + + "this to be implemented, as write requests use a new approach."); + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index dbacce5377cc..87372ed710bc 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -2740,6 +2740,20 @@ public void deleteLifecycleConfiguration(String volumeName, String bucketName) handleError(submitRequest(omRequest)); } + @Override + public void suspendLifecycleService() throws IOException { + OzoneManagerProtocolProtos.SuspendLifecycleServiceRequest + suspendLifecycleServiceRequest = + OzoneManagerProtocolProtos.SuspendLifecycleServiceRequest + .newBuilder().build(); + + OMRequest omRequest = createOMRequest(Type.SuspendLifecycleService) + .setSuspendLifecycleServiceRequest(suspendLifecycleServiceRequest) + .build(); + + handleError(submitRequest(omRequest)); + } + private SafeMode toProtoBuf(SafeModeAction action) { switch (action) { case ENTER: diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 61dbaf71fd0c..2c15faea3ab7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -194,6 +194,7 @@ protected static void startCluster(OzoneConfiguration conf) throws Exception { conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 10); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_LIST_MAX_COUNT, 1); + conf.setBoolean(OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, true); ozoneConfiguration = conf; MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf); builder.setOMServiceId(omServiceId) @@ -1843,6 +1844,41 @@ public void testLifecycleStatus() throws UnsupportedEncodingException { assertThat(output).contains("IsRunning:"); } + @Test + public void testLifecycleSuspend() throws Exception { + List ozoneManagers = cluster.getOzoneManagersList(); + for (OzoneManager om : ozoneManagers) { + assertNotNull(om.getKeyManager().getKeyLifecycleService() ); + assertTrue(om.getLifecycleServiceStatus().getIsEnabled()); + } + + // Execute suspend command + String[] args = new String[] {"om", "lifecycle", "suspend", "--service-id", omServiceId}; + execute(ozoneAdminShell, args); + String output = out.toString(DEFAULT_ENCODING); + assertThat(output).contains("Lifecycle Service has been suspended"); + out.reset(); + + // Wait for the suspend command to propagate through Ratis to all OMs + GenericTestUtils.waitFor(() -> { + for (OzoneManager om : ozoneManagers) { + assertNotNull(om.getKeyManager().getKeyLifecycleService() ); + if (om.getLifecycleServiceStatus().getIsEnabled()) { + return false; + } + } + return true; + }, 100, 10000); + + // Verify lifecycle service is suspended on all OMs + for (OzoneManager om : ozoneManagers) { + if (om.getKeyManager().getKeyLifecycleService() != null) { + assertFalse(om.getLifecycleServiceStatus().getIsEnabled(), + "Lifecycle service should be suspended on OM: " + om.getOMNodeId()); + } + } + } + @Test public void testCreateBucketWithECReplicationConfigWithoutReplicationParam() { getVolume("volume102"); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 05c2cdb2f3d3..b5c8721881be 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -160,6 +160,7 @@ enum Type { GetLifecycleConfiguration = 151; DeleteLifecycleConfiguration = 152; GetLifecycleServiceStatus = 153; + SuspendLifecycleService = 154; } enum SafeMode { @@ -313,6 +314,7 @@ message OMRequest { optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 152; optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 153; + optional SuspendLifecycleServiceRequest suspendLifecycleServiceRequest = 154; } message OMResponse { @@ -452,6 +454,7 @@ message OMResponse { optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 152; optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 153; + optional SuspendLifecycleServiceResponse suspendLifecycleServiceResponse = 154; } enum Status { @@ -2447,3 +2450,9 @@ message GetLifecycleServiceStatusResponse { repeated string runningBuckets = 3; } +message SuspendLifecycleServiceRequest { +} + +message SuspendLifecycleServiceResponse { +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java index 10cd0b212916..d3bb61eab383 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java @@ -117,7 +117,9 @@ public enum OMAction implements AuditAction { GET_LIFECYCLE_CONFIGURATION, SET_LIFECYCLE_CONFIGURATION, - DELETE_LIFECYCLE_CONFIGURATION; + DELETE_LIFECYCLE_CONFIGURATION, + GET_LIFECYCLE_SERVICE_STATUS, + SUSPEND_LIFECYCLE_SERVICE; @Override public String getAction() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java index 80c20f7af6dc..5309733cb2a7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java @@ -93,6 +93,7 @@ private static void init() { CMD_AUDIT_ACTION_MAP.put(Type.GetObjectTagging, OMAction.GET_OBJECT_TAGGING); CMD_AUDIT_ACTION_MAP.put(Type.PutObjectTagging, OMAction.PUT_OBJECT_TAGGING); CMD_AUDIT_ACTION_MAP.put(Type.DeleteObjectTagging, OMAction.DELETE_OBJECT_TAGGING); + CMD_AUDIT_ACTION_MAP.put(Type.SuspendLifecycleService, OMAction.SUSPEND_LIFECYCLE_SERVICE); } private static OMAction getAction(OzoneManagerProtocolProtos.OMRequest request) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 1f8f0e24a278..05085b7f5f94 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -68,6 +68,7 @@ import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest; import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleConfigurationDeleteRequest; import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleConfigurationSetRequest; +import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleServiceSuspendRequest; import org.apache.hadoop.ozone.om.request.s3.multipart.S3ExpiredMultipartUploadsAbortRequest; import org.apache.hadoop.ozone.om.request.s3.security.OMSetSecretRequest; import org.apache.hadoop.ozone.om.request.s3.security.S3GetSecretRequest; @@ -351,6 +352,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMLifecycleConfigurationSetRequest(omRequest); case DeleteLifecycleConfiguration: return new OMLifecycleConfigurationDeleteRequest(omRequest); + case SuspendLifecycleService: + return new OMLifecycleServiceSuspendRequest(omRequest); default: throw new OMException("Unrecognized write command type request " + cmdType, OMException.ResultCodes.INVALID_REQUEST); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java new file mode 100644 index 000000000000..8a1fd556c00d --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.lifecycle; + +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.OMAction; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleServiceSuspendResponse; +import org.apache.hadoop.ozone.om.service.KeyLifecycleService; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SuspendLifecycleServiceResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Handles SuspendLifecycleService Request. + */ +public class OMLifecycleServiceSuspendRequest extends OMClientRequest { + private static final Logger LOG = + LoggerFactory.getLogger(OMLifecycleServiceSuspendRequest.class); + + public OMLifecycleServiceSuspendRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) { + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); + AuditLogger auditLogger = ozoneManager.getAuditLogger(); + UserInfo userInfo = getOmRequest().getUserInfo(); + Map auditMap = new HashMap<>(); + IOException exception = null; + OMClientResponse omClientResponse; + + KeyLifecycleService keyLifecycleService = ozoneManager.getKeyManager().getKeyLifecycleService(); + if (keyLifecycleService != null) { + keyLifecycleService.setServiceEnabled(false); + LOG.info("KeyLifecycleService has been suspended by user: {}", + userInfo != null ? userInfo.getUserName() : "unknown"); + } else { + LOG.warn("KeyLifecycleService is not available"); + } + + omResponse.setSuspendLifecycleServiceResponse( + SuspendLifecycleServiceResponse.newBuilder().build()); + omClientResponse = new OMLifecycleServiceSuspendResponse(omResponse.build()); + markForAudit(auditLogger, buildAuditMessage( + OMAction.SUSPEND_LIFECYCLE_SERVICE, auditMap, exception, userInfo)); + return omClientResponse; + } +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleServiceSuspendResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleServiceSuspendResponse.java new file mode 100644 index 000000000000..c687766f65eb --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleServiceSuspendResponse.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.lifecycle; + +import java.io.IOException; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +/** + * Response for SuspendLifecycleService request. + * This response does not modify any database tables. + */ +@CleanupTableInfo +public class OMLifecycleServiceSuspendResponse extends OMClientResponse { + + public OMLifecycleServiceSuspendResponse(OMResponse omResponse) { + super(omResponse); + } + + @Override + protected void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) + throws IOException { + // No database update required for suspending the lifecycle service. + // The service state is maintained in memory only. + } +} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java index a6f2aeac5c18..a9f0c6cf630d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java @@ -31,6 +31,7 @@ versionProvider = HddsVersionProvider.class, subcommands = { LifecycleStatusSubCommand.class, + LifecycleSuspendSubCommand.class, }) public class LifecycleSubCommand implements AdminSubcommand { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java new file mode 100644 index 000000000000..b0ef9bc5ce59 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.om; + +import java.io.PrintStream; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +/** + * Handler of ozone admin om lifecycle suspend command. + */ +@Command( + name = "suspend", + description = "Suspend Lifecycle Service until OM restarts, after OM restarts, " + + "the Lifecycle Service may be enabled again based on the configuration", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class LifecycleSuspendSubCommand implements Callable { + + @CommandLine.ParentCommand + private LifecycleSubCommand parent; + + @CommandLine.Option( + names = {"-id", "--service-id"}, + description = "Ozone Manager Service ID" + ) + private String omServiceId; + + @CommandLine.Option( + names = {"-host", "--service-host"}, + description = "Ozone Manager Host" + ) + private String omHost; + + @Override + public Void call() throws Exception { + try (OzoneManagerProtocol ozoneManagerClient = + parent.getParent().createOmClient(omServiceId, omHost, false)) { + ozoneManagerClient.suspendLifecycleService(); + output(); + } + return null; + } + + protected void output() { + PrintStream out = out(); + out.println("========================================"); + out.println("Lifecycle Service has been suspended."); + out.println("The Lifecycle Service may be re-enabled after OM restarts"); + out.println("based on the configuration."); + out.println("You can modify the configuration to prevent" + + " Lifecycle Service from starting after OM starts.\n"); + out.println("========================================"); + } + + protected PrintStream out() { + return System.out; + } +} + From 1ba79a3970dca27915fd17917e5300449cab1a46 Mon Sep 17 00:00:00 2001 From: "pony.chen" Date: Tue, 16 Dec 2025 17:20:37 +0800 Subject: [PATCH 3/5] Address some comments --- .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../om/protocol/OzoneManagerProtocol.java | 2 - .../ozone/reconfig/TestOmReconfiguration.java | 2 - .../hadoop/ozone/shell/TestOzoneShellHA.java | 5 +- .../src/main/proto/OmClientProtocol.proto | 3 +- .../apache/hadoop/ozone/om/OzoneManager.java | 19 +------ .../OMLifecycleServiceSuspendRequest.java | 51 +++++++++++++------ .../ozone/om/service/KeyLifecycleService.java | 18 ++++--- .../om/service/TestKeyLifecycleService.java | 7 +-- .../om/FinalizationStatusSubCommand.java | 12 ++--- .../admin/om/LifecycleStatusSubCommand.java | 4 +- 11 files changed, 60 insertions(+), 64 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index ce6638dce07e..4f4af1077675 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -342,6 +342,7 @@ public static boolean isReadOnly( case DeleteObjectTagging: case SetLifecycleConfiguration: case DeleteLifecycleConfiguration: + case SuspendLifecycleService: case UnknownCommand: return false; case EchoRPC: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index d405b95b6eca..102b060ef967 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -1240,8 +1240,6 @@ default GetLifecycleServiceStatusResponse getLifecycleServiceStatus() throws IOE /** * Suspends the lifecycle service. - * The service will remain suspended until OM restarts, at which point it will - * be re-enabled based on the configuration. * @throws IOException */ default void suspendLifecycleService() throws IOException { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java index 04746b586647..258325ce070c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_READONLY_ADMINISTRATORS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED_DEFAULT; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -52,7 +51,6 @@ void reconfigurableProperties() { .add(OZONE_KEY_DELETING_LIMIT_PER_TASK) .add(OZONE_OM_VOLUME_LISTALL_ALLOWED) .add(OZONE_READONLY_ADMINISTRATORS) - .add(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED) .addAll(new OmConfig().reconfigurableProperties()) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 2c15faea3ab7..81632d3a3009 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -1841,14 +1841,13 @@ public void testLifecycleStatus() throws UnsupportedEncodingException { execute(ozoneAdminShell, args); String output = out.toString(DEFAULT_ENCODING); assertThat(output).contains("IsEnabled:"); - assertThat(output).contains("IsRunning:"); } @Test public void testLifecycleSuspend() throws Exception { List ozoneManagers = cluster.getOzoneManagersList(); for (OzoneManager om : ozoneManagers) { - assertNotNull(om.getKeyManager().getKeyLifecycleService() ); + assertNotNull(om.getKeyManager().getKeyLifecycleService()); assertTrue(om.getLifecycleServiceStatus().getIsEnabled()); } @@ -1862,7 +1861,7 @@ public void testLifecycleSuspend() throws Exception { // Wait for the suspend command to propagate through Ratis to all OMs GenericTestUtils.waitFor(() -> { for (OzoneManager om : ozoneManagers) { - assertNotNull(om.getKeyManager().getKeyLifecycleService() ); + assertNotNull(om.getKeyManager().getKeyLifecycleService()); if (om.getLifecycleServiceStatus().getIsEnabled()) { return false; } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index b5c8721881be..6142a16a1fd6 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -2446,8 +2446,7 @@ message GetLifecycleServiceStatusRequest { message GetLifecycleServiceStatusResponse { required bool isEnabled = 1; - required bool isRunning = 2; - repeated string runningBuckets = 3; + repeated string runningBuckets = 2; } message SuspendLifecycleServiceRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 8ee257e7a0c5..19838691c8d6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -544,9 +544,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) this::reconfOzoneReadOnlyAdmins) .register(OZONE_OM_VOLUME_LISTALL_ALLOWED, this::reconfigureAllowListAllVolumes) .register(OZONE_KEY_DELETING_LIMIT_PER_TASK, - this::reconfOzoneKeyDeletingLimitPerTask) - .register(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, - this::reconfKeyLifecycleServiceEnabled); + this::reconfOzoneKeyDeletingLimitPerTask); versionManager = new OMLayoutVersionManager(omStorage.getLayoutVersion()); upgradeFinalizer = new OMUpgradeFinalizer(versionManager); @@ -3172,7 +3170,6 @@ public GetLifecycleServiceStatusResponse getLifecycleServiceStatus() { return GetLifecycleServiceStatusResponse.newBuilder() .setIsEnabled(getConfiguration().getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT)) - .setIsRunning(false) .build(); } return keyLifecycleService.status(); @@ -5176,20 +5173,6 @@ private String reconfOzoneKeyDeletingLimitPerTask(String newVal) { return newVal; } - private String reconfKeyLifecycleServiceEnabled(String newVal) { - boolean enabled = StringUtils.isEmpty(newVal) ? - OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT : Boolean.parseBoolean(newVal); - KeyLifecycleService keyLifecycleService = getKeyManager().getKeyLifecycleService(); - if (keyLifecycleService != null) { - keyLifecycleService.setServiceEnabled(enabled); - } else { - LOG.warn("Failed reconfiguration for '{}'. KeyLifecycleService is not initialized.", - OZONE_KEY_LIFECYCLE_SERVICE_ENABLED); - } - getConfiguration().setBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, enabled); - return newVal; - } - private String reconfigureAllowListAllVolumes(String newVal) { getConfiguration().set(OZONE_OM_VOLUME_LISTALL_ALLOWED, newVal); setAllowListAllVolumesFromConfig(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java index 8a1fd556c00d..11d226923933 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java @@ -17,9 +17,12 @@ package org.apache.hadoop.ozone.om.request.lifecycle; +import java.io.IOException; +import java.util.HashMap; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; @@ -30,15 +33,15 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SuspendLifecycleServiceResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - /** * Handles SuspendLifecycleService Request. + * This request suspends the KeyLifecycleService by setting isServiceEnabled to false. + * The service will remain suspended until OM restarts, at which point it will + * be re-enabled based on the configuration. */ public class OMLifecycleServiceSuspendRequest extends OMClientRequest { private static final Logger LOG = @@ -53,24 +56,40 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); AuditLogger auditLogger = ozoneManager.getAuditLogger(); UserInfo userInfo = getOmRequest().getUserInfo(); - Map auditMap = new HashMap<>(); IOException exception = null; OMClientResponse omClientResponse; - KeyLifecycleService keyLifecycleService = ozoneManager.getKeyManager().getKeyLifecycleService(); - if (keyLifecycleService != null) { - keyLifecycleService.setServiceEnabled(false); - LOG.info("KeyLifecycleService has been suspended by user: {}", - userInfo != null ? userInfo.getUserName() : "unknown"); - } else { - LOG.warn("KeyLifecycleService is not available"); + try { + if (ozoneManager.getAclsEnabled()) { + UserGroupInformation ugi = createUGIForApi(); + if (!ozoneManager.isAdmin(ugi)) { + throw new OMException("Access denied for user " + ugi + ". " + + "Superuser privilege is required to suspend Lifecycle Service.", + OMException.ResultCodes.ACCESS_DENIED); + } + } + + KeyLifecycleService keyLifecycleService = ozoneManager.getKeyManager().getKeyLifecycleService(); + if (keyLifecycleService != null) { + keyLifecycleService.setServiceEnabled(false); + LOG.info("KeyLifecycleService has been suspended by user: {}", + userInfo != null ? userInfo.getUserName() : "unknown"); + } else { + LOG.warn("KeyLifecycleService is not available"); + } + + omResponse.setSuspendLifecycleServiceResponse( + SuspendLifecycleServiceResponse.newBuilder().build()); + omClientResponse = new OMLifecycleServiceSuspendResponse(omResponse.build()); + } catch (IOException ex) { + exception = ex; + LOG.error("Failed to suspend KeyLifecycleService", ex); + omClientResponse = new OMLifecycleServiceSuspendResponse( + createErrorOMResponse(omResponse, ex)); } - omResponse.setSuspendLifecycleServiceResponse( - SuspendLifecycleServiceResponse.newBuilder().build()); - omClientResponse = new OMLifecycleServiceSuspendResponse(omResponse.build()); markForAudit(auditLogger, buildAuditMessage( - OMAction.SUSPEND_LIFECYCLE_SERVICE, auditMap, exception, userInfo)); + OMAction.SUSPEND_LIFECYCLE_SERVICE, new HashMap<>(), exception, userInfo)); return omClientResponse; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index d6721a320a1a..141ee9e01cfa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java @@ -221,6 +221,10 @@ public void resume() { suspended.set(false); } + /** + * Set isServiceEnabled. + * @param enabled whether enable the lifecycle Service + */ public void setServiceEnabled(boolean enabled) { this.isServiceEnabled.set(enabled); LOG.info("KeyLifecycleService is {}", enabled ? "enabled" : "disabled"); @@ -232,10 +236,13 @@ public void shutdown() { KeyLifecycleServiceMetrics.unregister(); } + /** + * Build a GetLifecycleServiceStatusResponse instance. + * @return GetLifecycleServiceStatusResponse instance + */ public GetLifecycleServiceStatusResponse status() { Set runningBuckets = new HashSet<>(inFlight.keySet()); return GetLifecycleServiceStatusResponse.newBuilder() - .setIsRunning(!runningBuckets.isEmpty()) .setIsEnabled(isServiceEnabled.get()) .addAllRunningBuckets(runningBuckets) .build(); @@ -347,11 +354,6 @@ public BackgroundTaskResult call() { return result; } - if (!shouldRun()) { - onSuccess(bucketKey); - return result; - } - LOG.info("{} expired keys and {} expired dirs found and remained for bucket {}", expiredKeyList.size(), expiredDirList.size(), bucketKey); @@ -492,7 +494,7 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table HashSet deletedDirSet = new HashSet<>(); while (!stack.isEmpty()) { if (!shouldRun()) { - LOG.info("KeyLifecycleService is suspended or disabled." + + LOG.info("KeyLifecycleService is suspended or disabled. " + "Stopping LifecycleActionTask for bucket {}.", bucketName); return; } @@ -725,6 +727,8 @@ private void evaluateBucket(OmBucketInfo bucketInfo, keyTable.iterator(omMetadataManager.getBucketKey(volumeName, bucketName))) { while (keyTblItr.hasNext()) { if (!shouldRun()) { + LOG.info("KeyLifecycleService is suspended or disabled. " + + "Stopping LifecycleActionTask for bucket {}.", bucketName); return; } Table.KeyValue keyValue = keyTblItr.next(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java index 189e501363f9..7b1b0e57c7ca 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java @@ -1546,11 +1546,10 @@ void testGetLifecycleServiceStatus() throws Exception { final String bucketName = uniqueObjectName("bucket"); String prefix = "key"; - //Sservice should be enabled but not running + //Service should be enabled but not running OzoneManagerProtocolProtos.GetLifecycleServiceStatusResponse status = om.getLifecycleServiceStatus(); assertTrue(status.getIsEnabled()); - assertFalse(status.getIsRunning()); assertEquals(0, status.getRunningBucketsCount()); // Create and inject for test @@ -1564,19 +1563,17 @@ void testGetLifecycleServiceStatus() throws Exception { // Verify service is running and processing the bucket status = om.getLifecycleServiceStatus(); assertTrue(status.getIsEnabled()); - assertTrue(status.getIsRunning()); assertEquals(1, status.getRunningBucketsCount()); assertTrue(status.getRunningBucketsList().contains("/" + volumeName + "/" + bucketName)); KeyLifecycleService.getInjector(0).resume(); KeyLifecycleService.getInjector(1).resume(); - GenericTestUtils.waitFor(() -> !om.getLifecycleServiceStatus().getIsRunning(), + GenericTestUtils.waitFor(() -> om.getLifecycleServiceStatus().getRunningBucketsCount() == 0, WAIT_CHECK_INTERVAL, 10000); // Verify service completed and is no longer running status = om.getLifecycleServiceStatus(); assertTrue(status.getIsEnabled()); - assertFalse(status.getIsRunning()); assertEquals(0, status.getRunningBucketsCount()); deleteLifecyclePolicy(volumeName, bucketName); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java index 580fc160364a..dcde6618967d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/FinalizationStatusSubCommand.java @@ -52,12 +52,12 @@ public class FinalizationStatusSubCommand implements Callable { @Override public Void call() throws Exception { - try (OzoneManagerProtocol client = parent.createOmClient(omServiceId, omHost, false)) { - String upgradeClientID = "Upgrade-Client-" + UUID.randomUUID().toString(); - UpgradeFinalization.StatusAndMessages progress = - client.queryUpgradeFinalizationProgress(upgradeClientID, false, true); - System.out.println(progress.status()); - } + OzoneManagerProtocol client = + parent.createOmClient(omServiceId, omHost, false); + String upgradeClientID = "Upgrade-Client-" + UUID.randomUUID().toString(); + UpgradeFinalization.StatusAndMessages progress = + client.queryUpgradeFinalizationProgress(upgradeClientID, false, true); + System.out.println(progress.status()); return null; } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java index 2de1b567aad7..be525cfa46e3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone.admin.om; -import java.io.IOException; import java.io.PrintStream; import java.util.concurrent.Callable; import org.apache.hadoop.hdds.cli.HddsVersionProvider; @@ -62,13 +61,12 @@ public Void call() throws Exception { return null; } - protected void output(GetLifecycleServiceStatusResponse status) throws IOException { + protected void output(GetLifecycleServiceStatusResponse status) { PrintStream out = out(); out.println("========================================"); out.println(" Lifecycle Service Status"); out.println("========================================"); out.printf("IsEnabled: %s%n", status.getIsEnabled()); - out.printf("IsRunning: %s%n", status.getIsRunning()); if (status.getRunningBucketsCount() > 0) { out.println("Running Buckets:"); From 2bed61b3a8cee0cfd1a4ca24919f73091b389656 Mon Sep 17 00:00:00 2001 From: "pony.chen" Date: Wed, 17 Dec 2025 14:58:37 +0800 Subject: [PATCH 4/5] fix test --- .../apache/hadoop/ozone/om/response/TestCleanupTableInfo.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 7def54ae5e8b..a6d90d200ace 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -60,6 +60,7 @@ import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.om.response.key.OmKeyResponse; +import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleServiceSuspendResponse; import org.apache.hadoop.ozone.om.response.util.OMEchoRPCWriteResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyRequest; @@ -135,6 +136,7 @@ public void checkAnnotationAndTableName() { // OMEchoRPCWriteResponse does not need CleanupTable. subTypes.remove(OMEchoRPCWriteResponse.class); subTypes.remove(DummyOMClientResponse.class); + subTypes.remove(OMLifecycleServiceSuspendResponse.class); subTypes.forEach(aClass -> { assertTrue(aClass.isAnnotationPresent(CleanupTableInfo.class), aClass + " does not have annotation of" + From f49963757854ddf4a9d594e9fef640190bb63b10 Mon Sep 17 00:00:00 2001 From: XiChen <32928346+xichen01@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:35:22 +0800 Subject: [PATCH 5/5] Add resume command --- .../java/org/apache/hadoop/ozone/OmUtils.java | 2 +- .../om/protocol/OzoneManagerProtocol.java | 9 +++ ...ManagerProtocolClientSideTranslatorPB.java | 26 +++++-- .../hadoop/ozone/shell/TestOzoneShellHA.java | 38 +++++++++- .../src/main/proto/OmClientProtocol.proto | 14 ++-- .../apache/hadoop/ozone/audit/OMAction.java | 2 +- .../ozone/om/helpers/OMAuditLogger.java | 2 +- .../ratis/utils/OzoneManagerRatisUtils.java | 6 +- ...> OMLifecycleSetServiceStatusRequest.java} | 47 +++++++----- ... OMLifecycleSetServiceStatusResponse.java} | 8 +- .../ozone/om/service/KeyLifecycleService.java | 18 ++--- .../om/response/TestCleanupTableInfo.java | 4 +- .../admin/om/LifecycleResumeSubCommand.java | 73 +++++++++++++++++++ .../admin/om/LifecycleStatusSubCommand.java | 5 +- .../ozone/admin/om/LifecycleSubCommand.java | 1 + .../admin/om/LifecycleSuspendSubCommand.java | 10 +-- 16 files changed, 200 insertions(+), 65 deletions(-) rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/{OMLifecycleServiceSuspendRequest.java => OMLifecycleSetServiceStatusRequest.java} (65%) rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/{OMLifecycleServiceSuspendResponse.java => OMLifecycleSetServiceStatusResponse.java} (84%) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleResumeSubCommand.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 4f4af1077675..2287d904da31 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -342,7 +342,7 @@ public static boolean isReadOnly( case DeleteObjectTagging: case SetLifecycleConfiguration: case DeleteLifecycleConfiguration: - case SuspendLifecycleService: + case SetLifecycleServiceStatus: case UnknownCommand: return false; case EchoRPC: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 102b060ef967..f8d686f53ada 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -1246,4 +1246,13 @@ default void suspendLifecycleService() throws IOException { throw new UnsupportedOperationException("OzoneManager does not require " + "this to be implemented, as write requests use a new approach."); } + + /** + * Resumes the lifecycle service. + * @throws IOException + */ + default void resumeLifecycleService() throws IOException { + throw new UnsupportedOperationException("OzoneManager does not require " + + "this to be implemented, as write requests use a new approach."); + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 87372ed710bc..94fbe0e0f060 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -2742,13 +2742,27 @@ public void deleteLifecycleConfiguration(String volumeName, String bucketName) @Override public void suspendLifecycleService() throws IOException { - OzoneManagerProtocolProtos.SuspendLifecycleServiceRequest - suspendLifecycleServiceRequest = - OzoneManagerProtocolProtos.SuspendLifecycleServiceRequest - .newBuilder().build(); + OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest + setLifecycleServiceStatusRequest = + OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest + .newBuilder().setSuspend(true).build(); + + OMRequest omRequest = createOMRequest(Type.SetLifecycleServiceStatus) + .setSetLifecycleServiceStatusRequest(setLifecycleServiceStatusRequest) + .build(); + + handleError(submitRequest(omRequest)); + } + + @Override + public void resumeLifecycleService() throws IOException { + OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest + setLifecycleServiceStatusRequest = + OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest + .newBuilder().setSuspend(false).build(); - OMRequest omRequest = createOMRequest(Type.SuspendLifecycleService) - .setSuspendLifecycleServiceRequest(suspendLifecycleServiceRequest) + OMRequest omRequest = createOMRequest(Type.SetLifecycleServiceStatus) + .setSetLifecycleServiceStatusRequest(setLifecycleServiceStatusRequest) .build(); handleError(submitRequest(omRequest)); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 81632d3a3009..2dad2eef7aca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -1844,11 +1844,12 @@ public void testLifecycleStatus() throws UnsupportedEncodingException { } @Test - public void testLifecycleSuspend() throws Exception { + public void testLifecycleSuspendAndResume() throws Exception { List ozoneManagers = cluster.getOzoneManagersList(); for (OzoneManager om : ozoneManagers) { assertNotNull(om.getKeyManager().getKeyLifecycleService()); assertTrue(om.getLifecycleServiceStatus().getIsEnabled()); + assertFalse(om.getLifecycleServiceStatus().getIsSuspended()); } // Execute suspend command @@ -1862,7 +1863,7 @@ public void testLifecycleSuspend() throws Exception { GenericTestUtils.waitFor(() -> { for (OzoneManager om : ozoneManagers) { assertNotNull(om.getKeyManager().getKeyLifecycleService()); - if (om.getLifecycleServiceStatus().getIsEnabled()) { + if (!om.getLifecycleServiceStatus().getIsSuspended()) { return false; } } @@ -1872,8 +1873,39 @@ public void testLifecycleSuspend() throws Exception { // Verify lifecycle service is suspended on all OMs for (OzoneManager om : ozoneManagers) { if (om.getKeyManager().getKeyLifecycleService() != null) { - assertFalse(om.getLifecycleServiceStatus().getIsEnabled(), + assertTrue(om.getLifecycleServiceStatus().getIsSuspended(), "Lifecycle service should be suspended on OM: " + om.getOMNodeId()); + // isEnabled should still be true (based on configuration) + assertTrue(om.getLifecycleServiceStatus().getIsEnabled(), + "Lifecycle service isEnabled should still be true on OM: " + om.getOMNodeId()); + } + } + + // Execute resume command + args = new String[] {"om", "lifecycle", "resume", "--service-id", omServiceId}; + execute(ozoneAdminShell, args); + output = out.toString(DEFAULT_ENCODING); + assertThat(output).contains("Lifecycle Service has been resumed"); + out.reset(); + + // Wait for the resume command to propagate through Ratis to all OMs + GenericTestUtils.waitFor(() -> { + for (OzoneManager om : ozoneManagers) { + assertNotNull(om.getKeyManager().getKeyLifecycleService()); + if (om.getLifecycleServiceStatus().getIsSuspended()) { + return false; + } + } + return true; + }, 100, 10000); + + // Verify lifecycle service is resumed on all OMs + for (OzoneManager om : ozoneManagers) { + if (om.getKeyManager().getKeyLifecycleService() != null) { + assertFalse(om.getLifecycleServiceStatus().getIsSuspended(), + "Lifecycle service should be resumed on OM: " + om.getOMNodeId()); + assertTrue(om.getLifecycleServiceStatus().getIsEnabled(), + "Lifecycle service isEnabled should be true on OM: " + om.getOMNodeId()); } } } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 6142a16a1fd6..9d28133f1597 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -160,7 +160,7 @@ enum Type { GetLifecycleConfiguration = 151; DeleteLifecycleConfiguration = 152; GetLifecycleServiceStatus = 153; - SuspendLifecycleService = 154; + SetLifecycleServiceStatus = 154; } enum SafeMode { @@ -314,7 +314,7 @@ message OMRequest { optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 152; optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 153; - optional SuspendLifecycleServiceRequest suspendLifecycleServiceRequest = 154; + optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 154; } message OMResponse { @@ -454,7 +454,7 @@ message OMResponse { optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 152; optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 153; - optional SuspendLifecycleServiceResponse suspendLifecycleServiceResponse = 154; + optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 154; } enum Status { @@ -2446,12 +2446,14 @@ message GetLifecycleServiceStatusRequest { message GetLifecycleServiceStatusResponse { required bool isEnabled = 1; - repeated string runningBuckets = 2; + optional bool isSuspended = 2; + repeated string runningBuckets = 3; } -message SuspendLifecycleServiceRequest { +message SetLifecycleServiceStatusRequest { + required bool suspend = 1; } -message SuspendLifecycleServiceResponse { +message SetLifecycleServiceStatusResponse { } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java index d3bb61eab383..fa0452941750 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java @@ -119,7 +119,7 @@ public enum OMAction implements AuditAction { SET_LIFECYCLE_CONFIGURATION, DELETE_LIFECYCLE_CONFIGURATION, GET_LIFECYCLE_SERVICE_STATUS, - SUSPEND_LIFECYCLE_SERVICE; + SET_LIFECYCLE_SERVICE_STATUS; @Override public String getAction() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java index 5309733cb2a7..7673663fa044 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java @@ -93,7 +93,7 @@ private static void init() { CMD_AUDIT_ACTION_MAP.put(Type.GetObjectTagging, OMAction.GET_OBJECT_TAGGING); CMD_AUDIT_ACTION_MAP.put(Type.PutObjectTagging, OMAction.PUT_OBJECT_TAGGING); CMD_AUDIT_ACTION_MAP.put(Type.DeleteObjectTagging, OMAction.DELETE_OBJECT_TAGGING); - CMD_AUDIT_ACTION_MAP.put(Type.SuspendLifecycleService, OMAction.SUSPEND_LIFECYCLE_SERVICE); + CMD_AUDIT_ACTION_MAP.put(Type.SetLifecycleServiceStatus, OMAction.SET_LIFECYCLE_SERVICE_STATUS); } private static OMAction getAction(OzoneManagerProtocolProtos.OMRequest request) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 05085b7f5f94..dcaf47b17542 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -68,7 +68,7 @@ import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest; import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleConfigurationDeleteRequest; import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleConfigurationSetRequest; -import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleServiceSuspendRequest; +import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleSetServiceStatusRequest; import org.apache.hadoop.ozone.om.request.s3.multipart.S3ExpiredMultipartUploadsAbortRequest; import org.apache.hadoop.ozone.om.request.s3.security.OMSetSecretRequest; import org.apache.hadoop.ozone.om.request.s3.security.S3GetSecretRequest; @@ -352,8 +352,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMLifecycleConfigurationSetRequest(omRequest); case DeleteLifecycleConfiguration: return new OMLifecycleConfigurationDeleteRequest(omRequest); - case SuspendLifecycleService: - return new OMLifecycleServiceSuspendRequest(omRequest); + case SetLifecycleServiceStatus: + return new OMLifecycleSetServiceStatusRequest(omRequest); default: throw new OMException("Unrecognized write command type request " + cmdType, OMException.ResultCodes.INVALID_REQUEST); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSetServiceStatusRequest.java similarity index 65% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSetServiceStatusRequest.java index 11d226923933..be9e92b3c1d3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleServiceSuspendRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSetServiceStatusRequest.java @@ -27,27 +27,25 @@ import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleServiceSuspendResponse; +import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleSetServiceStatusResponse; import org.apache.hadoop.ozone.om.service.KeyLifecycleService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SuspendLifecycleServiceResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetLifecycleServiceStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Handles SuspendLifecycleService Request. - * This request suspends the KeyLifecycleService by setting isServiceEnabled to false. - * The service will remain suspended until OM restarts, at which point it will - * be re-enabled based on the configuration. + * Handles SetLifecycleServiceStatus Request. + * This request suspends or resumes the KeyLifecycleService. */ -public class OMLifecycleServiceSuspendRequest extends OMClientRequest { +public class OMLifecycleSetServiceStatusRequest extends OMClientRequest { private static final Logger LOG = - LoggerFactory.getLogger(OMLifecycleServiceSuspendRequest.class); + LoggerFactory.getLogger(OMLifecycleSetServiceStatusRequest.class); - public OMLifecycleServiceSuspendRequest(OMRequest omRequest) { + public OMLifecycleSetServiceStatusRequest(OMRequest omRequest) { super(omRequest); } @@ -56,40 +54,49 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); AuditLogger auditLogger = ozoneManager.getAuditLogger(); UserInfo userInfo = getOmRequest().getUserInfo(); + HashMap auditMap = new HashMap<>(); IOException exception = null; OMClientResponse omClientResponse; + boolean suspend = getOmRequest().getSetLifecycleServiceStatusRequest().getSuspend(); + auditMap.put("suspend", String.valueOf(suspend)); try { if (ozoneManager.getAclsEnabled()) { UserGroupInformation ugi = createUGIForApi(); if (!ozoneManager.isAdmin(ugi)) { throw new OMException("Access denied for user " + ugi + ". " - + "Superuser privilege is required to suspend Lifecycle Service.", + + "Superuser privilege is required to " + (suspend ? "suspend" : "resume") + " Lifecycle Service.", OMException.ResultCodes.ACCESS_DENIED); } } KeyLifecycleService keyLifecycleService = ozoneManager.getKeyManager().getKeyLifecycleService(); if (keyLifecycleService != null) { - keyLifecycleService.setServiceEnabled(false); - LOG.info("KeyLifecycleService has been suspended by user: {}", - userInfo != null ? userInfo.getUserName() : "unknown"); + if (suspend) { + keyLifecycleService.suspend(); + LOG.info("KeyLifecycleService has been suspended by user: {}", + userInfo != null ? userInfo.getUserName() : "unknown"); + } else { + keyLifecycleService.resume(); + LOG.info("KeyLifecycleService resume called by user: {}", + userInfo != null ? userInfo.getUserName() : "unknown"); + } } else { LOG.warn("KeyLifecycleService is not available"); } - omResponse.setSuspendLifecycleServiceResponse( - SuspendLifecycleServiceResponse.newBuilder().build()); - omClientResponse = new OMLifecycleServiceSuspendResponse(omResponse.build()); + omResponse.setSetLifecycleServiceStatusResponse( + SetLifecycleServiceStatusResponse.newBuilder().build()); + omClientResponse = new OMLifecycleSetServiceStatusResponse(omResponse.build()); } catch (IOException ex) { exception = ex; - LOG.error("Failed to suspend KeyLifecycleService", ex); - omClientResponse = new OMLifecycleServiceSuspendResponse( + LOG.error("Failed to " + (suspend ? "suspend" : "resume") + " KeyLifecycleService", ex); + omClientResponse = new OMLifecycleSetServiceStatusResponse( createErrorOMResponse(omResponse, ex)); } - markForAudit(auditLogger, buildAuditMessage( - OMAction.SUSPEND_LIFECYCLE_SERVICE, new HashMap<>(), exception, userInfo)); + markForAudit(auditLogger, buildAuditMessage(OMAction.SET_LIFECYCLE_SERVICE_STATUS, + auditMap, exception, userInfo)); return omClientResponse; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleServiceSuspendResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSetServiceStatusResponse.java similarity index 84% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleServiceSuspendResponse.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSetServiceStatusResponse.java index c687766f65eb..59f8b0963ff9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleServiceSuspendResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSetServiceStatusResponse.java @@ -25,13 +25,13 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; /** - * Response for SuspendLifecycleService request. + * Response for SetLifecycleServiceStatus request. * This response does not modify any database tables. */ @CleanupTableInfo -public class OMLifecycleServiceSuspendResponse extends OMClientResponse { +public class OMLifecycleSetServiceStatusResponse extends OMClientResponse { - public OMLifecycleServiceSuspendResponse(OMResponse omResponse) { + public OMLifecycleSetServiceStatusResponse(OMResponse omResponse) { super(omResponse); } @@ -39,7 +39,7 @@ public OMLifecycleServiceSuspendResponse(OMResponse omResponse) { protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - // No database update required for suspending the lifecycle service. + // No database update required for setting the lifecycle service state. // The service state is maintained in memory only. } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index 141ee9e01cfa..c77b48616067 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java @@ -208,7 +208,6 @@ public OzoneManager getOzoneManager() { /** * Suspend the service. */ - @VisibleForTesting public void suspend() { suspended.set(true); } @@ -216,18 +215,12 @@ public void suspend() { /** * Resume the service if suspended. */ - @VisibleForTesting public void resume() { suspended.set(false); } - /** - * Set isServiceEnabled. - * @param enabled whether enable the lifecycle Service - */ - public void setServiceEnabled(boolean enabled) { - this.isServiceEnabled.set(enabled); - LOG.info("KeyLifecycleService is {}", enabled ? "enabled" : "disabled"); + public boolean isSuspended() { + return suspended.get(); } @Override @@ -244,6 +237,7 @@ public GetLifecycleServiceStatusResponse status() { Set runningBuckets = new HashSet<>(inFlight.keySet()); return GetLifecycleServiceStatusResponse.newBuilder() .setIsEnabled(isServiceEnabled.get()) + .setIsSuspended(suspended.get()) .addAllRunningBuckets(runningBuckets) .build(); } @@ -494,8 +488,10 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table HashSet deletedDirSet = new HashSet<>(); while (!stack.isEmpty()) { if (!shouldRun()) { - LOG.info("KeyLifecycleService is suspended or disabled. " + - "Stopping LifecycleActionTask for bucket {}.", bucketName); + LOG.info("LifecycleActionTask for bucket {} stopping. " + + "Service enabled: {}, suspended: {}, leader ready: {}", + bucketName, isServiceEnabled.get(), suspended.get(), + getOzoneManager() != null ? getOzoneManager().isLeaderReady() : "N/A"); return; } PendingEvaluateDirectory item = stack.pop(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index a6d90d200ace..62bdacb8c4a4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@ -60,7 +60,7 @@ import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.om.response.key.OmKeyResponse; -import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleServiceSuspendResponse; +import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleSetServiceStatusResponse; import org.apache.hadoop.ozone.om.response.util.OMEchoRPCWriteResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyRequest; @@ -136,7 +136,7 @@ public void checkAnnotationAndTableName() { // OMEchoRPCWriteResponse does not need CleanupTable. subTypes.remove(OMEchoRPCWriteResponse.class); subTypes.remove(DummyOMClientResponse.class); - subTypes.remove(OMLifecycleServiceSuspendResponse.class); + subTypes.remove(OMLifecycleSetServiceStatusResponse.class); subTypes.forEach(aClass -> { assertTrue(aClass.isAnnotationPresent(CleanupTableInfo.class), aClass + " does not have annotation of" + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleResumeSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleResumeSubCommand.java new file mode 100644 index 000000000000..69f6c66a90da --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleResumeSubCommand.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.admin.om; + +import java.io.PrintStream; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +/** + * Handler of ozone admin om lifecycle resume command. + */ +@Command( + name = "resume", + description = "Resume Lifecycle Service that was previously suspended", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class LifecycleResumeSubCommand implements Callable { + + @CommandLine.ParentCommand + private LifecycleSubCommand parent; + + @CommandLine.Option( + names = {"-id", "--service-id"}, + description = "Ozone Manager Service ID" + ) + private String omServiceId; + + @CommandLine.Option( + names = {"-host", "--service-host"}, + description = "Ozone Manager Host" + ) + private String omHost; + + @Override + public Void call() throws Exception { + try (OzoneManagerProtocol ozoneManagerClient = + parent.getParent().createOmClient(omServiceId, omHost, false)) { + ozoneManagerClient.resumeLifecycleService(); + output(); + } + return null; + } + + protected void output() { + PrintStream out = out(); + out.println("========================================"); + out.println("Lifecycle Service has been resumed."); + out.println("========================================"); + } + + protected PrintStream out() { + return System.out; + } +} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java index be525cfa46e3..a443471391ac 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java @@ -26,7 +26,7 @@ import picocli.CommandLine.Command; /** - * Handler of ozone admin status command. + * Handler of ozone admin om lifecycle status command. */ @Command( name = "status", @@ -67,6 +67,9 @@ protected void output(GetLifecycleServiceStatusResponse status) { out.println(" Lifecycle Service Status"); out.println("========================================"); out.printf("IsEnabled: %s%n", status.getIsEnabled()); + if (status.getIsEnabled() && status.hasIsSuspended()) { + out.printf("IsSuspended: %s%n", status.getIsSuspended()); + } if (status.getRunningBucketsCount() > 0) { out.println("Running Buckets:"); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java index a9f0c6cf630d..7517618e84bd 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSubCommand.java @@ -32,6 +32,7 @@ subcommands = { LifecycleStatusSubCommand.class, LifecycleSuspendSubCommand.class, + LifecycleResumeSubCommand.class, }) public class LifecycleSubCommand implements AdminSubcommand { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java index b0ef9bc5ce59..8a420c3d41b6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java @@ -29,8 +29,8 @@ */ @Command( name = "suspend", - description = "Suspend Lifecycle Service until OM restarts, after OM restarts, " + - "the Lifecycle Service may be enabled again based on the configuration", + description = "Suspend Lifecycle Service. Use 'resume' command to resume it, " + + "or it will be re-enabled after OM restarts based on the configuration", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) public class LifecycleSuspendSubCommand implements Callable { @@ -64,10 +64,8 @@ protected void output() { PrintStream out = out(); out.println("========================================"); out.println("Lifecycle Service has been suspended."); - out.println("The Lifecycle Service may be re-enabled after OM restarts"); - out.println("based on the configuration."); - out.println("You can modify the configuration to prevent" + - " Lifecycle Service from starting after OM starts.\n"); + out.println("Use 'ozone admin om lifecycle resume' to resume it,"); + out.println("or it will be re-enabled after OM restarts based on the configuration."); out.println("========================================"); }