solving conflicts

Author: Selim Soufargi

.gitignore

@@ -46,4 +46,7 @@ my_config.yaml
 my_config_catalog.yaml
 
 # REST generated models
-spec/generated
+spec/generated
+
+# SDKMan
+.sdkmanrc

1

@@ -0,0 +1,20 @@
+handle lightweight metadata in the conversionSource
+
+# Please enter the commit message for your changes. Lines starting
+# with '#' will be ignored, and an empty message aborts the commit.
+#
+# interactive rebase in progress; onto f991e31
+# Last commands done (39 commands done):
+#    pick 4d0f245 bug fixes, TODO integrate functions into ParquetConversionSource
+#    pick 219656e handle lightweight metadata in the conversionSource
+# Next commands to do (62 remaining commands):
+#    pick ff809d7 fix CI
+#    pick e06368f fix CI, refactoring and isIncrementalSyncSafeFrom implementation
+# You are currently rebasing branch 'parquet_incr_sync' on 'f991e31'.
+#
+# Changes to be committed:
+#	modified:   xtable-core/src/main/java/org/apache/xtable/parquet/ParquetConversionSource.java
+#	modified:   xtable-core/src/main/java/org/apache/xtable/parquet/ParquetDataManager.java
+#	modified:   xtable-core/src/main/java/org/apache/xtable/parquet/ParquetFileConfig.java
+#	modified:   xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetDataManager.java
+#

pom.xml

@@ -63,7 +63,7 @@
         <project.build.outputTimestamp>2025-01-01T00:00:00Z</project.build.outputTimestamp>
         <maven.compiler.target>8</maven.compiler.target>
         <maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
-        <avro.version>1.11.4</avro.version>
+        <avro.version>1.12.0</avro.version>
         <log4j.version>2.22.0</log4j.version>
         <junit.version>5.11.4</junit.version>
         <junit.platform.runner.version>1.11.4</junit.platform.runner.version>
@@ -87,11 +87,12 @@
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.4.2</spark.version>
         <spark.version.prefix>3.4</spark.version.prefix>
-        <iceberg.version>1.4.2</iceberg.version>
+        <iceberg.version>1.9.2</iceberg.version>
+        <iceberg.hive.runtime.version>1.7.2</iceberg.hive.runtime.version>
         <delta.version>2.4.0</delta.version>
-        <paimon.version>1.2.0</paimon.version>
+        <paimon.version>1.3.1</paimon.version>
         <jackson.version>2.18.2</jackson.version>
-        <spotless.version>2.43.0</spotless.version>
+        <spotless.version>2.46.1</spotless.version>
         <apache.rat.version>0.16.1</apache.rat.version>
         <google.java.format.version>1.10.0</google.java.format.version>
         <delta.standalone.version>3.3.0</delta.standalone.version>
@@ -315,7 +316,7 @@
             <dependency>
                 <groupId>org.apache.iceberg</groupId>
                 <artifactId>iceberg-hive-runtime</artifactId>
-                <version>${iceberg.version}</version>
+                <version>${iceberg.hive.runtime.version}</version>
             </dependency>
 
             <!-- Delta -->
@@ -374,6 +375,12 @@
                 <version>${spark.version}</version>
                 <scope>provided</scope>
             </dependency>
+            <dependency>
+                <groupId>org.apache.spark</groupId>
+                <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
+                <version>${spark.version}</version>
+                <scope>provided</scope>
+            </dependency>
 
             <dependency>
                 <groupId>commons-cli</groupId>
@@ -870,6 +877,7 @@
                             <order>java,javax,lombok,org,org.apache.hudi,org.apache.iceberg,org.apache.spark.sql.delta,scala,com,io.delta,org.apache.xtable</order>
                         </importOrder>
                         <removeUnusedImports/>
+                        <removeWildcardImports/>
                         <excludes>
                             <exclude>${target.dir.pattern}</exclude>
                         </excludes>

xtable-api/src/main/java/org/apache/xtable/model/storage/FilesDiff.java

@@ -18,7 +18,11 @@
 
 package org.apache.xtable.model.storage;
 
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 

xtable-api/src/test/java/org/apache/xtable/model/catalog/TestThreePartHierarchicalTableIdentifier.java

@@ -18,7 +18,7 @@
 
 package org.apache.xtable.model.catalog;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;

xtable-api/src/test/java/org/apache/xtable/spi/sync/TestCatalogSync.java

@@ -18,7 +18,7 @@
 
 package org.apache.xtable.spi.sync;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.times;

xtable-aws/src/main/resources/META-INF/LICENSE-bundled

@@ -217,7 +217,7 @@ com.google.guava:guava:32.1.3-jre
 com.lmax:disruptor:3.4.2
 io.delta:delta-core_2.12:2.4.0
 io.delta:delta-storage:2.4.0
-org.apache.avro:avro:1.11.4
+org.apache.avro:avro:1.12.0
 org.apache.hbase:hbase-client:2.4.9
 org.apache.hbase:hbase-common:2.4.9
 org.apache.hbase:hbase-server:2.4.9
@@ -226,8 +226,8 @@ org.apache.httpcomponents:fluent-hc:4.4.1
 org.apache.hudi:hudi-common:0.14.0
 org.apache.hudi:hudi-java-client:0.14.0
 org.apache.hudi:hudi-timeline-service:0.14.0
-org.apache.iceberg:iceberg-api:1.4.2
-org.apache.iceberg:iceberg-core:1.4.2
+org.apache.iceberg:iceberg-api:1.9.2
+org.apache.iceberg:iceberg-core:1.9.2
 org.apache.logging.log4j:log4j-1.2-api:2.22.0
 org.apache.logging.log4j:log4j-api:2.22.0
 org.javassist:javassist:3.25.0-GA

xtable-core/src/main/java/org/apache/xtable/delta/DeltaConversionSource.java

@@ -85,16 +85,14 @@ public InternalTable getTable(Long version) {
 
   @Override
   public InternalTable getCurrentTable() {
-    DeltaLog deltaLog = DeltaLog.forTable(sparkSession, basePath);
     Snapshot snapshot = deltaLog.snapshot();
-    return getTable(snapshot.version());
+    return tableExtractor.table(snapshot, tableName);
   }
 
   @Override
   public InternalSnapshot getCurrentSnapshot() {
-    DeltaLog deltaLog = DeltaLog.forTable(sparkSession, basePath);
     Snapshot snapshot = deltaLog.snapshot();
-    InternalTable table = getTable(snapshot.version());
+    InternalTable table = tableExtractor.table(snapshot, tableName);
     return InternalSnapshot.builder()
         .table(table)
         .partitionedDataFiles(getInternalDataFiles(snapshot, table.getReadSchema()))
@@ -104,9 +102,9 @@ public InternalSnapshot getCurrentSnapshot() {
 
   @Override
   public TableChange getTableChangeForCommit(Long versionNumber) {
-    InternalTable tableAtVersion = tableExtractor.table(deltaLog, tableName, versionNumber);
     List<Action> actionsForVersion = getChangesState().getActionsForVersion(versionNumber);
     Snapshot snapshotAtVersion = deltaLog.getSnapshotAt(versionNumber, Option.empty());
+    InternalTable tableAtVersion = tableExtractor.table(snapshotAtVersion, tableName);
     FileFormat fileFormat =
         actionsConverter.convertToFileFormat(snapshotAtVersion.metadata().format().provider());
 

xtable-core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java

@@ -20,7 +20,10 @@
 
 import static org.apache.xtable.delta.ScalaUtils.convertJavaMapToScala;
 
-import java.util.*;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 

xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java

@@ -101,7 +101,8 @@ private InternalSchema toInternalSchema(
         type = InternalType.DOUBLE;
         break;
       case "binary":
-        if (originalMetadata.contains(InternalSchema.XTABLE_LOGICAL_TYPE)
+        if (originalMetadata != null
+            && originalMetadata.contains(InternalSchema.XTABLE_LOGICAL_TYPE)
             && "uuid".equals(originalMetadata.getString(InternalSchema.XTABLE_LOGICAL_TYPE))) {
           type = InternalType.UUID;
         } else {