HDDS-14826. Prepare OM to switch from LayoutVersionManager to ComponentVersionManager (#10098)

Author: errose28

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.upgrade;
+
+import java.util.Map;
+import org.apache.hadoop.hdds.ComponentVersion;
+
+/**
+ * Supplies upgrade actions keyed by {@link ComponentVersion}. Implementations typically perform classpath scanning or
+ * return a fixed map for tests. The component version manager decides when each action is invoked.
+ *
+ * @param <A> concrete upgrade action type (for example OM-specific or HDDS-specific)
+ */
+@FunctionalInterface
+public interface ComponentUpgradeActionProvider<A> {
+
+  /**
+   * Returns all upgrade actions from this provider, keyed by component version.
+   * <p>
+   * Implementations must return a newly allocated map on each call; the caller may retain and use it directly.
+   */
+  Map<ComponentVersion, A> load();
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeException.java

@@ -106,10 +106,8 @@ public String toString() {
   public enum ResultCodes {
     OK,
     INVALID_REQUEST,
-    UPDATE_LAYOUT_VERSION_FAILED,
-    LAYOUT_FEATURE_FINALIZATION_FAILED,
-    PREFINALIZE_ACTION_VALIDATION_FAILED,
-    FIRST_UPGRADE_START_ACTION_FAILED,
-    PREFINALIZE_VALIDATION_FAILED;
+    APPARENT_VERSION_UPDATE_FAILED,
+    UPGRADE_FINALIZATION_FAILED,
+    FINALIZE_UPGRADE_ACTION_FAILED,
   }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalization.java

@@ -49,12 +49,19 @@ public final class UpgradeFinalization {
   );
 
   /**
-   * Default message to provide when the service is in ALREADY_FINALIZED state.
+   * Returned from finalize upgrade commands when finalization is not required.
    */
   public static final StatusAndMessages FINALIZED_MSG = new StatusAndMessages(
       Status.ALREADY_FINALIZED, Collections.emptyList()
   );
 
+  /**
+   * Returned from progress/status queries when finalization is not required.
+   */
+  public static final StatusAndMessages FINALIZATION_DONE_MSG = new StatusAndMessages(
+      Status.FINALIZATION_DONE, Collections.emptyList()
+  );
+
   /**
    * Represents the current state in which the service is with regards to
    * finalization after an upgrade.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java

@@ -261,7 +261,7 @@ private static DatanodeDetailsYaml getDatanodeDetailsYaml(
           HDDSLayoutFeature layoutFeature
               = f.getAnnotation(BelongsToHDDSLayoutVersion.class).value();
           if (layoutFeature.layoutVersion() >
-              datanodeLayoutStorage.getLayoutVersion()) {
+              datanodeLayoutStorage.getApparentVersion()) {
             continue;
           }
         }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java

@@ -168,7 +168,7 @@ public DatanodeStateMachine(HddsDatanodeService hddsDatanodeService,
         datanodeDetails.getUuidString());
 
     layoutVersionManager = new HDDSLayoutVersionManager(
-        layoutStorage.getLayoutVersion());
+        layoutStorage.getApparentVersion());
     upgradeFinalizer = new DataNodeUpgradeFinalizer(layoutVersionManager);
     VersionedDatanodeFeatures.initialize(layoutVersionManager);
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java

@@ -47,7 +47,7 @@ public void finalizeLayoutFeature(LayoutFeature layoutFeature,
       String msg = String.format("Failed to finalize datanode layout feature " +
           "%s. It is not an HDDS Layout Feature.", layoutFeature);
       throw new UpgradeException(msg,
-          UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED);
+          UpgradeException.ResultCodes.UPGRADE_FINALIZATION_FAILED);
     }
   }
 }

hadoop-hdds/docs/content/design/upgrade-dev-primer.md

@@ -36,13 +36,11 @@ Downgrade to a lower version is allowed from the pre-finalized state. This invol
     org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature
 Class to add  a new layout feature being brought in. Layout version is typically 1 + last layout feature in that catalog. 
 
-## LayoutVersionManager
-    org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager
-    org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager
-Every component carries an instance of this interface, which provides APIs to get runtime layout version, and if a feature is allowed based on that or not.
+## Version management (OM vs HDDS)
 
-The LayoutVersionManager interface carries an API that can be used to check if a feature is allowed in the current layout version.
-     org.apache.hadoop.ozone.upgrade.LayoutVersionManager#isAllowed(org.apache.hadoop.ozone.upgrade.LayoutFeature)
+**Ozone Manager** uses [`org.apache.hadoop.ozone.om.upgrade.OMVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions discovered via [`org.apache.hadoop.ozone.om.upgrade.OMUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java). It exposes apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating.
+
+**SCM / DataNode** continue to use [`org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.AbstractLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java)), which provides metadata/software layout integers and `isAllowed(LayoutFeature)`.
 
 ## @DisallowedUntilLayoutVersion Annotation
 Method level annotation used to "disallow" an API if current layout version does not include the associated layout feature. Currently it is added only to the OM module, but can easily be moved down to a common module based on need on the HDDS layer.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java

@@ -20,29 +20,45 @@
 import java.io.IOException;
 import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.HDDSVersion;
+import org.apache.hadoop.ozone.common.Storage;
 import org.apache.hadoop.ozone.upgrade.ComponentVersionManager;
+import org.apache.hadoop.ozone.upgrade.UpgradeException;
 
 /**
  * Component version manager for HDDS (Datanodes and SCM).
  */
 public class HDDSVersionManager extends ComponentVersionManager {
-  public HDDSVersionManager(int serializedApparentVersion) throws IOException {
-    super(computeApparentVersion(serializedApparentVersion), HDDSVersion.SOFTWARE_VERSION);
+  public HDDSVersionManager(Storage storage) throws IOException {
+    super(storage, computeApparentVersion(storage.getApparentVersion()), HDDSVersion.SOFTWARE_VERSION);
   }
 
   /**
-   * If the apparent version stored on the disk is >= 100, it indicates the component has been finalized for the
-   * ZDU feature, and the apparent version corresponds to a version in {@link HDDSVersion}.
-   * If the apparent version stored on the disk is < 100, it indicates the component is not yet finalized for the
-   * ZDU feature, and the apparent version corresponds to a version in {@link HDDSLayoutFeature}.
+   * If the apparent version stored on the disk is &gt;= {@link HDDSVersion#ZDU} serialized, the apparent version is
+   * resolved via {@link HDDSVersion#deserialize(int)}. Values with no matching {@link HDDSVersion} fail startup with
+   * the persisted integer in the exception message.
+   * If the value is below that threshold, the apparent version is resolved as a {@link HDDSLayoutFeature}. Integers in
+   * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values; startup fails
+   * with the persisted integer in the exception message.
    */
-  private static ComponentVersion computeApparentVersion(int serializedApparentVersion) {
-    if (serializedApparentVersion < HDDSVersion.ZDU.serialize()) {
-      return HDDSLayoutFeature.deserialize(serializedApparentVersion);
+  private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException {
+    if (serializedApparentVersion >= HDDSVersion.ZDU.serialize()) {
+      HDDSVersion fromHdds = HDDSVersion.deserialize(serializedApparentVersion);
+      if (fromHdds != HDDSVersion.FUTURE_VERSION) {
+        return fromHdds;
+      }
     } else {
-      return HDDSVersion.deserialize(serializedApparentVersion);
+      ComponentVersion fromLayout = HDDSLayoutFeature.deserialize(serializedApparentVersion);
+      if (fromLayout != null) {
+        return fromLayout;
+      }
     }
+    throw new IOException("Initialization failed. Disk contains unknown apparent version " + serializedApparentVersion +
+        " for software version " + HDDSVersion.SOFTWARE_VERSION + ". Make sure this component was not downgraded" +
+        " after finalization");
   }
 
-  // TODO HDDS-14826: Register upgrade actions based on annotations
+  @Override
+  protected void runUpgradeAction(ComponentVersion componentVersion) throws UpgradeException {
+    // TODO HDDS-14826: Register upgrade actions based on annotations
+  }
 }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/Storage.java

@@ -124,20 +124,12 @@ public void setClusterId(String clusterId) throws IOException {
     }
   }
 
-  public int getLayoutVersion() {
-    return storageInfo.getLayoutVersion();
+  public int getApparentVersion() {
+    return storageInfo.getApparentVersion();
   }
 
-  public void setLayoutVersion(int version) {
-    storageInfo.setLayoutVersion(version);
-  }
-
-  public void setFirstUpgradeActionLayoutVersion(int version) {
-    storageInfo.setFirstUpgradeActionLayoutVersion(version);
-  }
-
-  public int getFirstUpgradeActionLayoutVersion() {
-    return storageInfo.getFirstUpgradeActionLayoutVersion();
+  public void setApparentVersion(int version) {
+    storageInfo.setApparentVersion(version);
   }
 
   /**
@@ -288,8 +280,7 @@ public void persistCurrentState() throws IOException {
     storageInfo.writeTo(getVersionFile());
   }
 
-  protected static int getInitLayoutVersion(OzoneConfiguration conf,
-      String configKey,
+  protected static int getInitApparentVersion(OzoneConfiguration conf, String configKey,
       IntSupplier defaultLvSupplier) {
     int lV = conf.getInt(configKey, OZONE_INIT_DEFAULT_LAYOUT_VERSION_DEFAULT);
     if (lV == OZONE_INIT_DEFAULT_LAYOUT_VERSION_DEFAULT) {

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java

@@ -61,9 +61,6 @@ public class StorageInfo {
    */
   private static final String LAYOUT_VERSION = "layoutVersion";
 
-  private static final String FIRST_UPGRADE_ACTION_LAYOUT_VERSION =
-      "firstUpgradeActionLayoutVersion";
-
   private static final int INVALID_LAYOUT_VERSION = -1;
 
   /**
@@ -93,7 +90,7 @@ public StorageInfo(NodeType type, File propertiesFile)
     verifyNodeType(type);
     verifyClusterId();
     verifyCreationTime();
-    verifyLayoutVersion();
+    verifyApparentVersion();
   }
 
   public NodeType getNodeType() {
@@ -112,15 +109,15 @@ public Long  getCreationTime() {
     return null;
   }
 
-  public int getLayoutVersion() {
+  public int getApparentVersion() {
     String layout = properties.getProperty(LAYOUT_VERSION);
     if (layout != null) {
       return Integer.parseInt(layout);
     }
     return 0;
   }
 
-  private void verifyLayoutVersion() {
+  private void verifyApparentVersion() {
     String layout = getProperty(LAYOUT_VERSION);
     if (layout == null) {
       LOG.warn("Found " + STORAGE_FILE_VERSION + " file without any layout " +
@@ -129,20 +126,6 @@ private void verifyLayoutVersion() {
     }
   }
 
-  public int getFirstUpgradeActionLayoutVersion() {
-    String upgradingTo =
-        properties.getProperty(FIRST_UPGRADE_ACTION_LAYOUT_VERSION);
-    if (upgradingTo != null) {
-      return Integer.parseInt(upgradingTo);
-    }
-    return INVALID_LAYOUT_VERSION;
-  }
-
-  public void setFirstUpgradeActionLayoutVersion(int layoutVersion) {
-    properties.setProperty(
-        FIRST_UPGRADE_ACTION_LAYOUT_VERSION, Integer.toString(layoutVersion));
-  }
-
   public String getProperty(String key) {
     return properties.getProperty(key);
   }
@@ -159,7 +142,7 @@ public void setClusterId(String clusterId) {
     properties.setProperty(CLUSTER_ID, clusterId);
   }
 
-  public void setLayoutVersion(int version) {
+  public void setApparentVersion(int version) {
     properties.setProperty(LAYOUT_VERSION, Integer.toString(version));
   }