HDDS-14939. Implement version file rewrite logic for path migration across metadata history (#10067)

Author: sreejasahithi

hadoop-ozone/dist/src/main/license/bin/LICENSE.txt

@@ -387,6 +387,7 @@ Apache License 2.0
    org.apache.hadoop:hadoop-shaded-protobuf_3_25
    org.apache.httpcomponents:httpcore
    org.apache.iceberg:iceberg-api
+   org.apache.iceberg:iceberg-bundled-guava
    org.apache.iceberg:iceberg-common
    org.apache.iceberg:iceberg-core
    org.apache.kerby:kerb-admin

hadoop-ozone/dist/src/main/license/jar-report.txt

@@ -87,6 +87,7 @@ share/ozone/lib/hppc.jar
 share/ozone/lib/httpclient.jar
 share/ozone/lib/httpcore.jar
 share/ozone/lib/iceberg-api.jar
+share/ozone/lib/iceberg-bundled-guava.jar
 share/ozone/lib/iceberg-common.jar
 share/ozone/lib/iceberg-core.jar
 share/ozone/lib/istack-commons-runtime.jar

hadoop-ozone/iceberg/pom.xml

@@ -37,12 +37,6 @@
     <dependency>
       <groupId>org.apache.iceberg</groupId>
       <artifactId>iceberg-api</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.iceberg</groupId>
-          <artifactId>iceberg-bundled-guava</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.iceberg</groupId>
@@ -60,10 +54,6 @@
           <groupId>org.apache.httpcomponents.core5</groupId>
           <artifactId>httpcore5-h2</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.apache.iceberg</groupId>
-          <artifactId>iceberg-bundled-guava</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>org.checkerframework</groupId>
           <artifactId>checker-qual</artifactId>
@@ -74,6 +64,22 @@
         </exclusion>
       </exclusions>
     </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.avro</groupId>
+          <artifactId>avro</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-ozone/iceberg/src/main/java/org/apache/hadoop/ozone/iceberg/RewriteTablePathOzoneAction.java

@@ -1,41 +1,55 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
+
 package org.apache.hadoop.ozone.iceberg;
 
+import java.util.HashSet;
+import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
-
 import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.PartitionStatisticsFile;
 import org.apache.iceberg.RewriteTablePathUtil;
+import org.apache.iceberg.RewriteTablePathUtil.RewriteResult;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.StaticTableOperations;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.TableMetadata;
 import org.apache.iceberg.TableMetadata.MetadataLogEntry;
+import org.apache.iceberg.TableMetadataParser;
 import org.apache.iceberg.actions.ImmutableRewriteTablePath;
 import org.apache.iceberg.actions.RewriteTablePath;
+import org.apache.iceberg.util.Pair;
 
+/**
+ * An implementation of {@link RewriteTablePath} for Apache Ozone backed Iceberg tables.
+ *
+ * <p>This action rewrites table's metadata and position delete file paths by replacing a source
+ * prefix with a target prefix. It processes table versions, snapshots, manifests and position delete files.</p>
+ *
+ * <p>The rewrite can be scoped between optional start and end metadata versions,
+ * and all rewritten files are staged in a temporary directory.</p>
+ */
 public class RewriteTablePathOzoneAction implements RewriteTablePath {
 
-  private static final String RESULT_LOCATION = "file-list";
-
   private String sourcePrefix;
   private String targetPrefix;
   private String startVersionName;
@@ -44,7 +58,6 @@ public class RewriteTablePathOzoneAction implements RewriteTablePath {
   private int parallelism;
 
   private final Table table;
-  private ExecutorService executorService;
 
   public RewriteTablePathOzoneAction(Table table) {
     this.table = table;
@@ -58,38 +71,39 @@ public RewriteTablePathOzoneAction(Table table, int parallelism) {
 
   @Override
   public RewriteTablePath rewriteLocationPrefix(String sPrefix, String tPrefix) {
-    checkNonNullNonEmpty(sPrefix, "Source prefix");
-    checkNonNullNonEmpty(tPrefix, "Target prefix");
+    RewriteTablePathOzoneUtils.checkNonNullNonEmpty(sPrefix, "Source prefix");
+    RewriteTablePathOzoneUtils.checkNonNullNonEmpty(tPrefix, "Target prefix");
     this.sourcePrefix = sPrefix;
     this.targetPrefix = tPrefix;
     return this;
   }
 
   @Override
   public RewriteTablePath startVersion(String sVersion) {
-    checkNonNullNonEmpty(sVersion, "Start version");
+    RewriteTablePathOzoneUtils.checkNonNullNonEmpty(sVersion, "Start version");
     this.startVersionName = sVersion;
     return this;
   }
 
   @Override
   public RewriteTablePath endVersion(String eVersion) {
-    checkNonNullNonEmpty(eVersion, "End version");
+    RewriteTablePathOzoneUtils.checkNonNullNonEmpty(eVersion, "End version");
     this.endVersionName = eVersion;
     return this;
   }
 
   @Override
   public RewriteTablePath stagingLocation(String stagingLocation) {
-    checkNonNullNonEmpty(stagingLocation, "Staging location");
+    RewriteTablePathOzoneUtils.checkNonNullNonEmpty(stagingLocation, "Staging location");
     this.stagingDir = stagingLocation;
     return this;
   }
 
   @Override
   public Result execute() {
     validateInputs();
-    this.executorService = Executors.newFixedThreadPool(parallelism);
+    // TODO: should use for parallel manifest and position delete file rewriting.
+    ExecutorService executorService = Executors.newFixedThreadPool(parallelism);
     try {
       return doExecute();
     } finally {
@@ -121,12 +135,13 @@ private void validateInputs() {
               "Source prefix cannot be the same as target prefix (%s)", sourcePrefix));
     }
 
-    validateAndSetEndVersion();
-    validateAndSetStartVersion();
+    TableMetadata tableMetadata = ((HasTableOperations) table).operations().current();
+    validateAndSetEndVersion(tableMetadata);
+    validateAndSetStartVersion(tableMetadata);
 
     if (stagingDir == null) {
       stagingDir =
-          getMetadataLocation(table)
+              RewriteTablePathOzoneUtils.getMetadataLocation(table)
               + "copy-table-staging-"
               + UUID.randomUUID()
               + RewriteTablePathUtil.FILE_SEPARATOR;
@@ -135,9 +150,7 @@ private void validateInputs() {
     }
   }
 
-  private void validateAndSetEndVersion() {
-    TableMetadata tableMetadata = ((HasTableOperations) table).operations().current();
-
+  private void validateAndSetEndVersion(TableMetadata tableMetadata) {
     if (endVersionName == null) {
       Objects.requireNonNull(
           tableMetadata.metadataFileLocation(), "Metadata file location should not be null");
@@ -147,9 +160,7 @@ private void validateAndSetEndVersion() {
     }
   }
 
-  private void validateAndSetStartVersion() {
-    TableMetadata tableMetadata = ((HasTableOperations) table).operations().current();
-
+  private void validateAndSetStartVersion(TableMetadata tableMetadata) {
     if (startVersionName != null) {
       this.startVersionName = validateVersion(tableMetadata, startVersionName);
     }
@@ -159,11 +170,12 @@ private String validateVersion(TableMetadata tableMetadata, String versionFileNa
     String versionFile = null;
     if (versionInFilePath(tableMetadata.metadataFileLocation(), versionFileName)) {
       versionFile = tableMetadata.metadataFileLocation();
-    }
-
-    for (MetadataLogEntry log : tableMetadata.previousFiles()) {
-      if (versionInFilePath(log.file(), versionFileName)) {
-        versionFile = log.file();
+    } else {
+      for (MetadataLogEntry log : tableMetadata.previousFiles()) {
+        if (versionInFilePath(log.file(), versionFileName)) {
+          versionFile = log.file();
+          break;
+        }
       }
     }
 
@@ -172,7 +184,7 @@ private String validateVersion(TableMetadata tableMetadata, String versionFileNa
           String.format(
               "Cannot find provided version file %s in metadata log.", versionFileName));
     }
-    if (!fileExist(versionFile)) {
+    if (!RewriteTablePathOzoneUtils.fileExist(versionFile, table.io())) {
       throw new IllegalArgumentException(
           String.format("Version file %s does not exist.", versionFile));
     }
@@ -184,36 +196,59 @@ private boolean versionInFilePath(String path, String version) {
   }
 
   private String rebuildMetadata() {
-    //TODO need to implement rewrite of metadata files , manifest list , manifest files and position delete files.
-    return null;
-  }
+    //TODO need to implement rewrite of manifest list , manifest files and position delete files.
+    TableMetadata endMetadata = new StaticTableOperations(endVersionName, table.io()).current();
 
-  private boolean fileExist(String path) {
-    if (path == null || path.trim().isEmpty()) {
-      return false;
+    List<PartitionStatisticsFile> partitionStats = endMetadata.partitionStatisticsFiles();
+    if (partitionStats != null && !partitionStats.isEmpty()) {
+      throw new IllegalArgumentException("Partition statistics files are not supported yet.");
     }
-    return table.io().newInputFile(path).exists();
+
+    RewriteResult<Snapshot> rewriteVersionResult = rewriteVersionFiles(endMetadata);
+
+    Set<Pair<String, String>> copyPlan = new HashSet<>();
+    copyPlan.addAll(rewriteVersionResult.copyPlan());
+
+    return RewriteTablePathOzoneUtils.saveFileList(copyPlan, stagingDir, table.io());
   }
 
-  private String getMetadataLocation(Table tbl) {
-    String currentMetadataPath =
-        ((HasTableOperations) tbl).operations().current().metadataFileLocation();
-    int lastIndex = currentMetadataPath.lastIndexOf(RewriteTablePathUtil.FILE_SEPARATOR);
-    String metadataDir = "";
-    if (lastIndex != -1) {
-      metadataDir = currentMetadataPath.substring(0, lastIndex + 1);
-    }
+  private RewriteResult<Snapshot> rewriteVersionFiles(TableMetadata endMetadata) {
+    RewriteResult<Snapshot> result = new RewriteResult<>();
+    result.toRewrite().addAll(endMetadata.snapshots());
+    result.copyPlan().addAll(rewriteVersionFile(endMetadata, endVersionName));
+
+    List<MetadataLogEntry> versions = endMetadata.previousFiles();
+    for (int i = versions.size() - 1; i >= 0; i--) {
+      String versionFilePath = versions.get(i).file();
+      if (versionFilePath.equals(startVersionName)) {
+        break;
+      }
+
+      if (!RewriteTablePathOzoneUtils.fileExist(versionFilePath, table.io())) {
+        throw new IllegalArgumentException(String.format("Version file %s doesn't exist", versionFilePath));
+      }
+
+      TableMetadata tableMetadata = new StaticTableOperations(versionFilePath, table.io()).current();
 
-    if (metadataDir.isEmpty()) {
-      throw new IllegalArgumentException("Failed to get the metadata file root directory");
+      result.toRewrite().addAll(tableMetadata.snapshots());
+      result.copyPlan().addAll(rewriteVersionFile(tableMetadata, versionFilePath));
     }
-    return metadataDir;
+
+    return result;
   }
 
-  private static void checkNonNullNonEmpty(String value, String name) {
-    Objects.requireNonNull(value, () -> name + " is null");
-    if (value.trim().isEmpty()) {
-      throw new IllegalArgumentException(name + " is empty");
-    }
+  private Set<Pair<String, String>> rewriteVersionFile(TableMetadata metadata, String versionFilePath) {
+    Set<Pair<String, String>> result = new HashSet<>();
+    String stagingPath = RewriteTablePathUtil.stagingPath(versionFilePath, sourcePrefix, stagingDir);
+    
+    System.out.println("Processing version file " + versionFilePath);
+    TableMetadata newTableMetadata = RewriteTablePathUtil.replacePaths(metadata, sourcePrefix, targetPrefix);
+    TableMetadataParser.overwrite(newTableMetadata, table.io().newOutputFile(stagingPath));
+    
+    result.add(Pair.of(stagingPath, RewriteTablePathUtil.newPath(versionFilePath, sourcePrefix, targetPrefix)));
+    result.addAll(RewriteTablePathOzoneUtils.statsFileCopyPlan(
+            metadata.statisticsFiles(), newTableMetadata.statisticsFiles()));
+
+    return result;
   }
 }