From f26e71cfb7c14536da2a97443a2f888fe81aadf1 Mon Sep 17 00:00:00 2001 From: Christof Schablinski Date: Fri, 19 Jun 2026 11:40:33 +0200 Subject: [PATCH 01/73] #15824 Include sources and Javadoc of subprojects of spark-runtime-4.0_2.13 into its source and javadoc-jar --- deploy.gradle | 12 +++++++++--- spark/v3.5/build.gradle | 39 +++++++++++++++++++++++++++++++++++++++ spark/v4.0/build.gradle | 39 +++++++++++++++++++++++++++++++++++++++ spark/v4.1/build.gradle | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) diff --git a/deploy.gradle b/deploy.gradle index 65836bf1b3f1..fc380daa71e6 100644 --- a/deploy.gradle +++ b/deploy.gradle @@ -47,15 +47,17 @@ subprojects { from sourceSets.test.output } + def sourceArtifactTask = tasks.findByName('shadowSourceJar') ?: sourceJar + artifacts { - archives sourceJar + archives sourceArtifactTask archives javadocJar archives testJar testArtifacts testJar } // add LICENSE and NOTICE - [jar, sourceJar, javadocJar, testJar].each { task -> + [jar, sourceArtifactTask, javadocJar, testJar].each { task -> task.dependsOn rootProject.tasks.buildInfo task.from("${rootDir}/build") { include 'iceberg-build.properties' @@ -81,8 +83,12 @@ subprojects { } else { project.shadow.component(it) } + if (tasks.matching({task -> task.name == 'shadowSourceJar'}).isEmpty()) { + artifact sourceJar + } else { + artifact shadowSourceJar + } - artifact sourceJar artifact javadocJar artifact testJar diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle index 68bdb1c21a98..964b8af9a52b 100644 --- a/spark/v3.5/build.gradle +++ b/spark/v3.5/build.gradle @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle index 3707e01e4865..207d119108ab 100644 --- a/spark/v4.0/build.gradle +++ b/spark/v4.0/build.gradle @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v4.1/build.gradle b/spark/v4.1/build.gradle index e6455fa34f88..dc649d9ec118 100644 --- a/spark/v4.1/build.gradle +++ b/spark/v4.1/build.gradle @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" } From 40175bf63abe132387367b08122d1a79cc13916c Mon Sep 17 00:00:00 2001 From: Fezinvirtual <39562460+Fezinvirtual@users.noreply.github.com> Date: Fri, 19 Jun 2026 20:53:18 +0800 Subject: [PATCH 02/73] Build: Remove duplicate jackson.databind declaration (#16879) The jackson.databind dependency appears twice in the dependencies closure. This change removes the duplicated line. It does not introduce any functional or behavioral changes to the Kafka Connect connector. --- kafka-connect/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/kafka-connect/build.gradle b/kafka-connect/build.gradle index 43eb245d93a3..5dec94dd5702 100644 --- a/kafka-connect/build.gradle +++ b/kafka-connect/build.gradle @@ -179,7 +179,6 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') { integrationImplementation platform(libs.jackson.bom) integrationImplementation libs.jackson.core integrationImplementation libs.jackson.databind - integrationImplementation libs.jackson.databind integrationImplementation libs.kafka.clients integrationImplementation libs.kafka.connect.api integrationImplementation libs.kafka.connect.json From 722a0735c0e7f0b8bb3bbe0245c6604ca9b773a9 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Fri, 19 Jun 2026 17:24:02 +0200 Subject: [PATCH 03/73] Flink: Backport: Add equality delete conversion DV resolution and writing (#16858) (#16869) --- flink/v1.20/build.gradle | 1 + .../operator/EqualityConvertDVWriter.java | 354 +++++++++++++ .../operator/OperatorTestBase.java | 12 + .../operator/TestEqualityConvertDVWriter.java | 472 ++++++++++++++++++ flink/v2.0/build.gradle | 1 + .../operator/EqualityConvertDVWriter.java | 354 +++++++++++++ .../operator/OperatorTestBase.java | 12 + .../operator/TestEqualityConvertDVWriter.java | 472 ++++++++++++++++++ 8 files changed, 1678 insertions(+) create mode 100644 flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java create mode 100644 flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java create mode 100644 flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java create mode 100644 flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java diff --git a/flink/v1.20/build.gradle b/flink/v1.20/build.gradle index 41f2489c8038..069cf2ee623b 100644 --- a/flink/v1.20/build.gradle +++ b/flink/v1.20/build.gradle @@ -68,6 +68,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") { } implementation libs.datasketches + implementation libs.roaringbitmap testImplementation libs.flink120.connector.test.utils testImplementation libs.flink120.core diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java new file mode 100644 index 000000000000..486b933ee6d7 --- /dev/null +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.BaseDeleteLoader; +import org.apache.iceberg.data.DeleteLoader; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.PositionDeleteIndex; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ManifestEvaluator; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.io.DeleteWriteResult; +import org.apache.iceberg.io.OutputFileFactory; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; +import org.apache.iceberg.util.StructLikeWrapper; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Keyed parallel resolver that buffers {@link DVPosition}s per data-file path, then writes Puffin + * DV files directly via {@link BaseDVFileWriter}. Plan metadata arrives broadcast on input 2, so + * every parallel task sees the cycle's metadata and can validate against the main snapshot. + * + *

Each buffered {@link DVPosition} carries the data file's {@code specId} + encoded partition, + * so writing DVs needs no data-manifest scan. Existing DVs are folded into the rewrite (V3 allows + * one DV per data file): delete manifests are pruned by partition summary to the cycle's affected + * partitions, then filtered to entries referencing the affected data files. No cross-cycle state is + * kept; reads are bounded by the pruned manifest set, not the table's full DV history. + * + *

Buffered positions are transient per-task. On failure recovery, upstream replay rebuilds them. + */ +@Internal +public class EqualityConvertDVWriter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertDVWriter.class); + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + + private transient Table table; + private transient OutputFileFactory fileFactory; + private transient DeleteLoader deleteLoader; + private transient Map positionsByFile; + private transient EqualityConvertPlan planResult; + private transient boolean hasUpstreamError; + private transient int manifestsRead; + + public EqualityConvertDVWriter( + String tableName, String taskName, TableLoader tableLoader, String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + int subtaskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + fileFactory = + OutputFileFactory.builderFor(table, subtaskIndex, 0L).format(FileFormat.PUFFIN).build(); + deleteLoader = new BaseDeleteLoader(deleteFile -> table.io().newInputFile(deleteFile)); + positionsByFile = Maps.newHashMap(); + } + + @Override + public void processElement1(StreamRecord record) { + DVPosition pos = record.getValue(); + if (pos.isAbort()) { + hasUpstreamError = true; + } + + if (!hasUpstreamError) { + positionsByFile + .computeIfAbsent( + pos.dataFilePath(), k -> new FilePositions(pos.specId(), pos.partition())) + .positions + .addLong(pos.position()); + } + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { + if (hasUpstreamError) { + output.collect(new StreamRecord<>(DVWriteResult.ABORT)); + } else { + try { + resolveAndWrite(); + } catch (Exception e) { + LOG.error("Error writing DVs for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + output.collect(new StreamRecord<>(DVWriteResult.ABORT)); + } + } + + positionsByFile.clear(); + hasUpstreamError = false; + planResult = null; + } + + super.processWatermark(mark); + } + + private void resolveAndWrite() throws IOException { + if (positionsByFile.isEmpty()) { + return; + } + + table.refresh(); + + Snapshot mainSnapshot = table.snapshot(targetBranch); + + // Fail fast if the main branch changed since planning, to avoid writing DV files that the + // committer would reject via validateFromSnapshot. The next cycle will reindex. + if (mainSnapshot != null + && planResult.mainSnapshotId() != null + && mainSnapshot.snapshotId() != planResult.mainSnapshotId()) { + throw new IllegalStateException( + "Main branch snapshot changed since planning: expected " + + planResult.mainSnapshotId() + + " but found: " + + mainSnapshot.snapshotId()); + } + + Map dvs = collectExistingDVs(mainSnapshot, positionsByFile.keySet()); + + // Fold staging DVs into the rewrite so the writer emits one DV per data file (V3 rule). Flink + // writes a staging DV only for a newly added data file, so it never collides with a distinct + // existing DV: on a separate target branch collectExistingDVs has not seen it yet; on a shared + // branch it IS that existing DV, so the put is idempotent. + for (DeleteFile sd : planResult.stagingDVFiles()) { + if (ContentFileUtil.isDV(sd) && sd.referencedDataFile() != null) { + dvs.put(sd.referencedDataFile(), sd); + } + } + + BaseDVFileWriter dvWriter = + new BaseDVFileWriter(fileFactory, path -> loadPreviousDV(path, dvs)); + try (dvWriter) { + for (Map.Entry entry : positionsByFile.entrySet()) { + String dataFilePath = entry.getKey(); + FilePositions filePositions = entry.getValue(); + PartitionSpec spec = table.specs().get(filePositions.specId); + StructLike partition = filePositions.partition(spec.partitionType()); + + filePositions.positions.forEach( + (long pos) -> dvWriter.delete(dataFilePath, pos, spec, partition)); + } + } + + DeleteWriteResult result = dvWriter.result(); + LOG.info( + "Wrote {} DV files (rewriting {}) for {} data files in table {} task {}.", + result.deleteFiles().size(), + result.rewrittenDeleteFiles().size(), + positionsByFile.size(), + tableName, + taskName); + + output.collect( + new StreamRecord<>( + new DVWriteResult( + Lists.newArrayList(result.deleteFiles()), + Lists.newArrayList(result.rewrittenDeleteFiles())))); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private Map collectExistingDVs( + Snapshot mainSnapshot, Set affectedPaths) { + manifestsRead = 0; + Map dvs = Maps.newHashMap(); + if (mainSnapshot == null) { + return dvs; + } + + // Prune delete manifests whose partition summaries cannot cover the cycle's affected + // partitions. A DV inherits its referenced data file's spec and partition, so partition pruning + // works for DV manifests. + Map evaluators = partitionEvaluators(positionsByFile); + for (ManifestFile manifest : mainSnapshot.deleteManifests(table.io())) { + ManifestEvaluator evaluator = evaluators.get(manifest.partitionSpecId()); + if (evaluator == null || !evaluator.eval(manifest)) { + continue; + } + + readDVEntries(manifest, affectedPaths, dvs); + } + + return dvs; + } + + private Map partitionEvaluators( + Map positions) { + Map templatesBySpec = Maps.newHashMap(); + Map> partitionsBySpec = Maps.newHashMap(); + for (FilePositions filePositions : positions.values()) { + PartitionSpec spec = table.specs().get(filePositions.specId); + StructLikeWrapper template = + templatesBySpec.computeIfAbsent( + filePositions.specId, id -> StructLikeWrapper.forType(spec.partitionType())); + partitionsBySpec + .computeIfAbsent(filePositions.specId, k -> Sets.newHashSet()) + .add(template.copyFor(filePositions.partition(spec.partitionType()))); + } + + Map evaluators = Maps.newHashMap(); + for (Map.Entry> entry : partitionsBySpec.entrySet()) { + PartitionSpec spec = table.specs().get(entry.getKey()); + Expression filter = partitionFilter(spec, entry.getValue()); + evaluators.put(entry.getKey(), ManifestEvaluator.forPartitionFilter(filter, spec, false)); + } + + return evaluators; + } + + private static Expression partitionFilter(PartitionSpec spec, Set partitions) { + List fields = spec.fields(); + Expression anyPartition = Expressions.alwaysFalse(); + for (StructLikeWrapper wrapper : partitions) { + StructLike partition = wrapper.get(); + Expression onePartition = Expressions.alwaysTrue(); + for (int i = 0; i < fields.size(); i++) { + String name = fields.get(i).name(); + Object value = partition.get(i, Object.class); + Expression predicate = + value == null ? Expressions.isNull(name) : Expressions.equal(name, value); + onePartition = Expressions.and(onePartition, predicate); + } + + anyPartition = Expressions.or(anyPartition, onePartition); + } + + return anyPartition; + } + + private void readDVEntries( + ManifestFile manifest, Set filterPaths, Map out) { + manifestsRead++; + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (ContentFileUtil.isDV(deleteFile) + && deleteFile.referencedDataFile() != null + && filterPaths.contains(deleteFile.referencedDataFile())) { + out.put(deleteFile.referencedDataFile(), deleteFile); + } + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to read manifest: " + manifest.path(), e); + } + } + + @VisibleForTesting + int manifestsReadLastCycle() { + return manifestsRead; + } + + @VisibleForTesting + int retainedStateSize() { + return positionsByFile.size(); + } + + private PositionDeleteIndex loadPreviousDV(String dataFilePath, Map dvs) { + DeleteFile existingDV = dvs.get(dataFilePath); + if (existingDV == null) { + return null; + } + + return deleteLoader.loadPositionDeletes(ImmutableList.of(existingDV), dataFilePath); + } + + private static final class FilePositions { + private final int specId; + private final byte[] encodedPartition; + private final Roaring64Bitmap positions = new Roaring64Bitmap(); + private StructLike decodedPartition; + + FilePositions(int specId, byte[] encodedPartition) { + this.specId = specId; + this.encodedPartition = encodedPartition; + } + + StructLike partition(Types.StructType partitionType) { + if (decodedPartition == null) { + decodedPartition = StructLikeSerializer.decodePartition(encodedPartition, partitionType); + } + + return decodedPartition; + } + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java index c21518c6d29c..a45198c6ea9d 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java @@ -188,6 +188,18 @@ protected static Table createTableWithDelete(int formatVersion) { "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true")); } + protected static Table createPartitionedTableWithDelete(int formatVersion) { + return CATALOG_EXTENSION + .catalog() + .createTable( + TestFixtures.TABLE_IDENTIFIER, + SCHEMA_WITH_PRIMARY_KEY, + PartitionSpec.builderFor(SCHEMA_WITH_PRIMARY_KEY).identity("data").build(), + null, + ImmutableMap.of( + "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true")); + } + protected static Table createPartitionedTable(int formatVersion, FileFormat fileFormat) { return CATALOG_EXTENSION .catalog() diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java new file mode 100644 index 000000000000..a0c2f3568baf --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java @@ -0,0 +1,472 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertDVWriter extends OperatorTestBase { + + private static final byte[] EMPTY_PARTITION = new byte[0]; + + @Test + void writesDVFileForSinglePosition() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + } + } + + @Test + void writesSingleDVFileForMultiplePositionsOnSameDataFile() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 2, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(3); + assertThat(output.get(0).rewrittenDvFiles()).isEmpty(); + } + } + + @Test + void emptyPositionsProducesNoOutput() throws Exception { + createTableWithDelete(3); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void noOutputWithoutPlanResult() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void writesDVFilesForMultipleDataFiles() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + List dataFilePaths = getDataFilePaths(table); + assertThat(dataFilePaths).hasSize(2); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(0), 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(1), 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).dvFiles()).hasSize(2); + } + } + + @Test + void routesErrorToErrorStream() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + dropTable(); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(harness.extractOutputValues().get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void mergesStagingDVIntoWrittenDV() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + // Staging DV for the data file at position 0, produced but not committed to the target branch. + DeleteFile stagingDV = writeStagingDV(dataFilePath, 0); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), Lists.newArrayList(stagingDV), 1L, null, 0L, 0L); + + // New position 1 in the same file. Must merge it with the staged position 0. + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(planResult, time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(2); + assertThat(output.get(0).rewrittenDvFiles()).hasSize(1); + assertThat(output.get(0).rewrittenDvFiles().get(0).location()) + .isEqualTo(stagingDV.location()); + } + } + + @Test + void abortsWhenMainSnapshotChangedSincePlanning() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + long plannedSnapshotId = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), Lists.newArrayList(), 1L, plannedSnapshotId, 0L, 0L); + + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(planResult, time)); + + // Advance the target branch after planning. The writer must refuse to write stale DVs. + insert(table, 2, "b"); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void stateResetBetweenBatches() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time1 = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time1)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time1)); + harness.processBothWatermarks(new Watermark(time1)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Second batch with no positions, plan only. + long time2 = time1 + 1; + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time2)); + harness.processBothWatermarks(new Watermark(time2)); + + // Cumulative output: still hasSize(1) since the empty batch produced nothing. + assertThat(harness.extractOutputValues()).hasSize(1); + } + } + + @Test + void abortsOnUpstreamAbortPosition() throws Exception { + createTableWithDelete(3); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(DVPosition.ABORT, time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + } + } + + @Test + void doesNotRetainStateAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + int cycles = 5; + for (int i = 0; i < cycles; i++) { + insert(table, i, "v" + i); + } + + List dataFilePaths = getDataFilePaths(table); + assertThat(dataFilePaths).hasSize(cycles); + + EqualityConvertDVWriter resolver = newResolver(); + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(resolver)) { + harness.open(); + + long time = System.currentTimeMillis(); + for (int i = 0; i < cycles; i++) { + // Each cycle resolves a position in a distinct data file, growing the set of files seen. + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(i), 0, 0, EMPTY_PARTITION, 0L), time + i)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time + i)); + harness.processBothWatermarks(new Watermark(time + i)); + + // No cross-cycle state: the buffer clears every cycle regardless of how many distinct data + // files have been processed. + assertThat(resolver.retainedStateSize()).isZero(); + assertThat(harness.extractOutputValues()).hasSize(i + 1); + } + } + } + + @Test + void prunesDeleteManifestsByPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, Lists.newArrayList(createRecord(1, "a")), "a"); + DataFile fileA = dataFile(table, "a"); + insertPartitioned(table, Lists.newArrayList(createRecord(2, "b")), "b"); + DataFile fileB = dataFile(table, "b"); + + // Commit one DV per partition: two separate delete manifests, summaries {a} and {b}. + commitDV(table, fileA); + commitDV(table, fileB); + assertThat(table.currentSnapshot().deleteManifests(table.io())).hasSize(2); + + // A new cycle resolves another position in partition a's file. Only partition a's delete + // manifest overlaps; partition b's manifest must be pruned. + EqualityConvertDVWriter resolver = newResolver(); + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(resolver)) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(dvPosition(table, fileA, 0), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + // Folded partition a's existing DV: the matching manifest was read. + assertThat(output.get(0).rewrittenDvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles()).hasSize(1); + // Partition b's manifest was skipped: only one manifest read this cycle. + assertThat(resolver.manifestsReadLastCycle()).isEqualTo(1); + } + } + + private void commitDV(Table table, DataFile dataFile) throws Exception { + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(dvPosition(table, dataFile, 0), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + RowDelta rowDelta = table.newRowDelta(); + harness.extractOutputValues().get(0).dvFiles().forEach(rowDelta::addDeletes); + rowDelta.commit(); + table.refresh(); + } + } + + private static DVPosition dvPosition(Table table, DataFile dataFile, long position) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + byte[] partition = + new StructLikeSerializer().encodePartition(dataFile.partition(), spec.partitionType()); + return new DVPosition(dataFile.location(), position, dataFile.specId(), partition, 0L); + } + + private static DataFile dataFile(Table table, String partitionValue) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + if (partitionValue.equals(String.valueOf(file.partition().get(0, Object.class)))) { + return file; + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data file for partition: " + partitionValue); + } + + private DeleteFile writeStagingDV(String dataFilePath, long position) throws Exception { + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, position, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + return harness.extractOutputValues().get(0).dvFiles().get(0); + } + } + + private EqualityConvertDVWriter newResolver() { + return new EqualityConvertDVWriter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), SnapshotRef.MAIN_BRANCH); + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>(newResolver()); + } + + private static EqualityConvertPlan emptyEqualityConvertPlan() { + return new EqualityConvertPlan(Lists.newArrayList(), Lists.newArrayList(), 1L, null, 0L, 0L); + } + + private static String getFirstDataFilePath(Table table) { + return getDataFilePaths(table).get(0); + } + + private static List getDataFilePaths(Table table) { + List paths = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + paths.add(file.location()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return paths; + } +} diff --git a/flink/v2.0/build.gradle b/flink/v2.0/build.gradle index 7bc37b30e5a1..b34eb6d06537 100644 --- a/flink/v2.0/build.gradle +++ b/flink/v2.0/build.gradle @@ -68,6 +68,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") { } implementation libs.datasketches + implementation libs.roaringbitmap testImplementation libs.flink20.connector.test.utils testImplementation libs.flink20.core diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java new file mode 100644 index 000000000000..486b933ee6d7 --- /dev/null +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.BaseDeleteLoader; +import org.apache.iceberg.data.DeleteLoader; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.PositionDeleteIndex; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ManifestEvaluator; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.io.DeleteWriteResult; +import org.apache.iceberg.io.OutputFileFactory; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; +import org.apache.iceberg.util.StructLikeWrapper; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Keyed parallel resolver that buffers {@link DVPosition}s per data-file path, then writes Puffin + * DV files directly via {@link BaseDVFileWriter}. Plan metadata arrives broadcast on input 2, so + * every parallel task sees the cycle's metadata and can validate against the main snapshot. + * + *

Each buffered {@link DVPosition} carries the data file's {@code specId} + encoded partition, + * so writing DVs needs no data-manifest scan. Existing DVs are folded into the rewrite (V3 allows + * one DV per data file): delete manifests are pruned by partition summary to the cycle's affected + * partitions, then filtered to entries referencing the affected data files. No cross-cycle state is + * kept; reads are bounded by the pruned manifest set, not the table's full DV history. + * + *

Buffered positions are transient per-task. On failure recovery, upstream replay rebuilds them. + */ +@Internal +public class EqualityConvertDVWriter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertDVWriter.class); + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + + private transient Table table; + private transient OutputFileFactory fileFactory; + private transient DeleteLoader deleteLoader; + private transient Map positionsByFile; + private transient EqualityConvertPlan planResult; + private transient boolean hasUpstreamError; + private transient int manifestsRead; + + public EqualityConvertDVWriter( + String tableName, String taskName, TableLoader tableLoader, String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + int subtaskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + fileFactory = + OutputFileFactory.builderFor(table, subtaskIndex, 0L).format(FileFormat.PUFFIN).build(); + deleteLoader = new BaseDeleteLoader(deleteFile -> table.io().newInputFile(deleteFile)); + positionsByFile = Maps.newHashMap(); + } + + @Override + public void processElement1(StreamRecord record) { + DVPosition pos = record.getValue(); + if (pos.isAbort()) { + hasUpstreamError = true; + } + + if (!hasUpstreamError) { + positionsByFile + .computeIfAbsent( + pos.dataFilePath(), k -> new FilePositions(pos.specId(), pos.partition())) + .positions + .addLong(pos.position()); + } + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { + if (hasUpstreamError) { + output.collect(new StreamRecord<>(DVWriteResult.ABORT)); + } else { + try { + resolveAndWrite(); + } catch (Exception e) { + LOG.error("Error writing DVs for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + output.collect(new StreamRecord<>(DVWriteResult.ABORT)); + } + } + + positionsByFile.clear(); + hasUpstreamError = false; + planResult = null; + } + + super.processWatermark(mark); + } + + private void resolveAndWrite() throws IOException { + if (positionsByFile.isEmpty()) { + return; + } + + table.refresh(); + + Snapshot mainSnapshot = table.snapshot(targetBranch); + + // Fail fast if the main branch changed since planning, to avoid writing DV files that the + // committer would reject via validateFromSnapshot. The next cycle will reindex. + if (mainSnapshot != null + && planResult.mainSnapshotId() != null + && mainSnapshot.snapshotId() != planResult.mainSnapshotId()) { + throw new IllegalStateException( + "Main branch snapshot changed since planning: expected " + + planResult.mainSnapshotId() + + " but found: " + + mainSnapshot.snapshotId()); + } + + Map dvs = collectExistingDVs(mainSnapshot, positionsByFile.keySet()); + + // Fold staging DVs into the rewrite so the writer emits one DV per data file (V3 rule). Flink + // writes a staging DV only for a newly added data file, so it never collides with a distinct + // existing DV: on a separate target branch collectExistingDVs has not seen it yet; on a shared + // branch it IS that existing DV, so the put is idempotent. + for (DeleteFile sd : planResult.stagingDVFiles()) { + if (ContentFileUtil.isDV(sd) && sd.referencedDataFile() != null) { + dvs.put(sd.referencedDataFile(), sd); + } + } + + BaseDVFileWriter dvWriter = + new BaseDVFileWriter(fileFactory, path -> loadPreviousDV(path, dvs)); + try (dvWriter) { + for (Map.Entry entry : positionsByFile.entrySet()) { + String dataFilePath = entry.getKey(); + FilePositions filePositions = entry.getValue(); + PartitionSpec spec = table.specs().get(filePositions.specId); + StructLike partition = filePositions.partition(spec.partitionType()); + + filePositions.positions.forEach( + (long pos) -> dvWriter.delete(dataFilePath, pos, spec, partition)); + } + } + + DeleteWriteResult result = dvWriter.result(); + LOG.info( + "Wrote {} DV files (rewriting {}) for {} data files in table {} task {}.", + result.deleteFiles().size(), + result.rewrittenDeleteFiles().size(), + positionsByFile.size(), + tableName, + taskName); + + output.collect( + new StreamRecord<>( + new DVWriteResult( + Lists.newArrayList(result.deleteFiles()), + Lists.newArrayList(result.rewrittenDeleteFiles())))); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private Map collectExistingDVs( + Snapshot mainSnapshot, Set affectedPaths) { + manifestsRead = 0; + Map dvs = Maps.newHashMap(); + if (mainSnapshot == null) { + return dvs; + } + + // Prune delete manifests whose partition summaries cannot cover the cycle's affected + // partitions. A DV inherits its referenced data file's spec and partition, so partition pruning + // works for DV manifests. + Map evaluators = partitionEvaluators(positionsByFile); + for (ManifestFile manifest : mainSnapshot.deleteManifests(table.io())) { + ManifestEvaluator evaluator = evaluators.get(manifest.partitionSpecId()); + if (evaluator == null || !evaluator.eval(manifest)) { + continue; + } + + readDVEntries(manifest, affectedPaths, dvs); + } + + return dvs; + } + + private Map partitionEvaluators( + Map positions) { + Map templatesBySpec = Maps.newHashMap(); + Map> partitionsBySpec = Maps.newHashMap(); + for (FilePositions filePositions : positions.values()) { + PartitionSpec spec = table.specs().get(filePositions.specId); + StructLikeWrapper template = + templatesBySpec.computeIfAbsent( + filePositions.specId, id -> StructLikeWrapper.forType(spec.partitionType())); + partitionsBySpec + .computeIfAbsent(filePositions.specId, k -> Sets.newHashSet()) + .add(template.copyFor(filePositions.partition(spec.partitionType()))); + } + + Map evaluators = Maps.newHashMap(); + for (Map.Entry> entry : partitionsBySpec.entrySet()) { + PartitionSpec spec = table.specs().get(entry.getKey()); + Expression filter = partitionFilter(spec, entry.getValue()); + evaluators.put(entry.getKey(), ManifestEvaluator.forPartitionFilter(filter, spec, false)); + } + + return evaluators; + } + + private static Expression partitionFilter(PartitionSpec spec, Set partitions) { + List fields = spec.fields(); + Expression anyPartition = Expressions.alwaysFalse(); + for (StructLikeWrapper wrapper : partitions) { + StructLike partition = wrapper.get(); + Expression onePartition = Expressions.alwaysTrue(); + for (int i = 0; i < fields.size(); i++) { + String name = fields.get(i).name(); + Object value = partition.get(i, Object.class); + Expression predicate = + value == null ? Expressions.isNull(name) : Expressions.equal(name, value); + onePartition = Expressions.and(onePartition, predicate); + } + + anyPartition = Expressions.or(anyPartition, onePartition); + } + + return anyPartition; + } + + private void readDVEntries( + ManifestFile manifest, Set filterPaths, Map out) { + manifestsRead++; + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (ContentFileUtil.isDV(deleteFile) + && deleteFile.referencedDataFile() != null + && filterPaths.contains(deleteFile.referencedDataFile())) { + out.put(deleteFile.referencedDataFile(), deleteFile); + } + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to read manifest: " + manifest.path(), e); + } + } + + @VisibleForTesting + int manifestsReadLastCycle() { + return manifestsRead; + } + + @VisibleForTesting + int retainedStateSize() { + return positionsByFile.size(); + } + + private PositionDeleteIndex loadPreviousDV(String dataFilePath, Map dvs) { + DeleteFile existingDV = dvs.get(dataFilePath); + if (existingDV == null) { + return null; + } + + return deleteLoader.loadPositionDeletes(ImmutableList.of(existingDV), dataFilePath); + } + + private static final class FilePositions { + private final int specId; + private final byte[] encodedPartition; + private final Roaring64Bitmap positions = new Roaring64Bitmap(); + private StructLike decodedPartition; + + FilePositions(int specId, byte[] encodedPartition) { + this.specId = specId; + this.encodedPartition = encodedPartition; + } + + StructLike partition(Types.StructType partitionType) { + if (decodedPartition == null) { + decodedPartition = StructLikeSerializer.decodePartition(encodedPartition, partitionType); + } + + return decodedPartition; + } + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java index b4f6ef95fc0f..a80e5c1bb2c0 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java @@ -188,6 +188,18 @@ protected static Table createTableWithDelete(int formatVersion) { "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true")); } + protected static Table createPartitionedTableWithDelete(int formatVersion) { + return CATALOG_EXTENSION + .catalog() + .createTable( + TestFixtures.TABLE_IDENTIFIER, + SCHEMA_WITH_PRIMARY_KEY, + PartitionSpec.builderFor(SCHEMA_WITH_PRIMARY_KEY).identity("data").build(), + null, + ImmutableMap.of( + "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true")); + } + protected static Table createPartitionedTable(int formatVersion, FileFormat fileFormat) { return CATALOG_EXTENSION .catalog() diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java new file mode 100644 index 000000000000..a0c2f3568baf --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java @@ -0,0 +1,472 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertDVWriter extends OperatorTestBase { + + private static final byte[] EMPTY_PARTITION = new byte[0]; + + @Test + void writesDVFileForSinglePosition() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + } + } + + @Test + void writesSingleDVFileForMultiplePositionsOnSameDataFile() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 2, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(3); + assertThat(output.get(0).rewrittenDvFiles()).isEmpty(); + } + } + + @Test + void emptyPositionsProducesNoOutput() throws Exception { + createTableWithDelete(3); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void noOutputWithoutPlanResult() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void writesDVFilesForMultipleDataFiles() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + List dataFilePaths = getDataFilePaths(table); + assertThat(dataFilePaths).hasSize(2); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(0), 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(1), 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).dvFiles()).hasSize(2); + } + } + + @Test + void routesErrorToErrorStream() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + dropTable(); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(harness.extractOutputValues().get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void mergesStagingDVIntoWrittenDV() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + // Staging DV for the data file at position 0, produced but not committed to the target branch. + DeleteFile stagingDV = writeStagingDV(dataFilePath, 0); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), Lists.newArrayList(stagingDV), 1L, null, 0L, 0L); + + // New position 1 in the same file. Must merge it with the staged position 0. + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(planResult, time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(2); + assertThat(output.get(0).rewrittenDvFiles()).hasSize(1); + assertThat(output.get(0).rewrittenDvFiles().get(0).location()) + .isEqualTo(stagingDV.location()); + } + } + + @Test + void abortsWhenMainSnapshotChangedSincePlanning() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + long plannedSnapshotId = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), Lists.newArrayList(), 1L, plannedSnapshotId, 0L, 0L); + + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(planResult, time)); + + // Advance the target branch after planning. The writer must refuse to write stale DVs. + insert(table, 2, "b"); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void stateResetBetweenBatches() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time1 = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time1)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time1)); + harness.processBothWatermarks(new Watermark(time1)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Second batch with no positions, plan only. + long time2 = time1 + 1; + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time2)); + harness.processBothWatermarks(new Watermark(time2)); + + // Cumulative output: still hasSize(1) since the empty batch produced nothing. + assertThat(harness.extractOutputValues()).hasSize(1); + } + } + + @Test + void abortsOnUpstreamAbortPosition() throws Exception { + createTableWithDelete(3); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(DVPosition.ABORT, time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + } + } + + @Test + void doesNotRetainStateAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + int cycles = 5; + for (int i = 0; i < cycles; i++) { + insert(table, i, "v" + i); + } + + List dataFilePaths = getDataFilePaths(table); + assertThat(dataFilePaths).hasSize(cycles); + + EqualityConvertDVWriter resolver = newResolver(); + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(resolver)) { + harness.open(); + + long time = System.currentTimeMillis(); + for (int i = 0; i < cycles; i++) { + // Each cycle resolves a position in a distinct data file, growing the set of files seen. + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(i), 0, 0, EMPTY_PARTITION, 0L), time + i)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time + i)); + harness.processBothWatermarks(new Watermark(time + i)); + + // No cross-cycle state: the buffer clears every cycle regardless of how many distinct data + // files have been processed. + assertThat(resolver.retainedStateSize()).isZero(); + assertThat(harness.extractOutputValues()).hasSize(i + 1); + } + } + } + + @Test + void prunesDeleteManifestsByPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, Lists.newArrayList(createRecord(1, "a")), "a"); + DataFile fileA = dataFile(table, "a"); + insertPartitioned(table, Lists.newArrayList(createRecord(2, "b")), "b"); + DataFile fileB = dataFile(table, "b"); + + // Commit one DV per partition: two separate delete manifests, summaries {a} and {b}. + commitDV(table, fileA); + commitDV(table, fileB); + assertThat(table.currentSnapshot().deleteManifests(table.io())).hasSize(2); + + // A new cycle resolves another position in partition a's file. Only partition a's delete + // manifest overlaps; partition b's manifest must be pruned. + EqualityConvertDVWriter resolver = newResolver(); + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(resolver)) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(dvPosition(table, fileA, 0), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + // Folded partition a's existing DV: the matching manifest was read. + assertThat(output.get(0).rewrittenDvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles()).hasSize(1); + // Partition b's manifest was skipped: only one manifest read this cycle. + assertThat(resolver.manifestsReadLastCycle()).isEqualTo(1); + } + } + + private void commitDV(Table table, DataFile dataFile) throws Exception { + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(dvPosition(table, dataFile, 0), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + RowDelta rowDelta = table.newRowDelta(); + harness.extractOutputValues().get(0).dvFiles().forEach(rowDelta::addDeletes); + rowDelta.commit(); + table.refresh(); + } + } + + private static DVPosition dvPosition(Table table, DataFile dataFile, long position) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + byte[] partition = + new StructLikeSerializer().encodePartition(dataFile.partition(), spec.partitionType()); + return new DVPosition(dataFile.location(), position, dataFile.specId(), partition, 0L); + } + + private static DataFile dataFile(Table table, String partitionValue) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + if (partitionValue.equals(String.valueOf(file.partition().get(0, Object.class)))) { + return file; + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data file for partition: " + partitionValue); + } + + private DeleteFile writeStagingDV(String dataFilePath, long position) throws Exception { + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, position, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + return harness.extractOutputValues().get(0).dvFiles().get(0); + } + } + + private EqualityConvertDVWriter newResolver() { + return new EqualityConvertDVWriter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), SnapshotRef.MAIN_BRANCH); + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>(newResolver()); + } + + private static EqualityConvertPlan emptyEqualityConvertPlan() { + return new EqualityConvertPlan(Lists.newArrayList(), Lists.newArrayList(), 1L, null, 0L, 0L); + } + + private static String getFirstDataFilePath(Table table) { + return getDataFilePaths(table).get(0); + } + + private static List getDataFilePaths(Table table) { + List paths = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + paths.add(file.location()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return paths; + } +} From 41c8ee43b46017843b304241d250737fd10837af Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 19 Jun 2026 13:44:02 -0400 Subject: [PATCH 04/73] Docs: Clarify geography type serialization (#16799) --- format/spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/format/spec.md b/format/spec.md index b88f2af1edcc..be28a6c5f000 100644 --- a/format/spec.md +++ b/format/spec.md @@ -1685,7 +1685,7 @@ Types are serialized according to this table: |**`map`**|`JSON object: {`
  `"type": "map",`
  `"key-id": ,`
  `"key": ,`
  `"value-id": ,`
  `"value-required": `
  `"value": `
`}`|`{`
  `"type": "map",`
  `"key-id": 4,`
  `"key": "string",`
  `"value-id": 5,`
  `"value-required": false,`
  `"value": "double"`
`}`| | **`variant`**| `JSON string: "variant"`|`"variant"`| | **`geometry(C)`** |`JSON string: "geometry()"`|`"geometry(srid:4326)"`| -| **`geography(C, A)`** |`JSON string: "geography(,)"`|`"geography(srid:4326,spherical)"`| +| **`geography(C, A)`** |`JSON string: "geography(, )"`|`"geography(srid:4326, spherical)"`| Note that default values are serialized using the JSON single-value serialization in [Appendix D](#appendix-d-single-value-serialization). From 8d1f973ba9717c51070a0b7c2cec76af7660576a Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Sat, 20 Jun 2026 07:45:17 +0200 Subject: [PATCH 05/73] Flink: Add equality delete conversion committer (#16874) * Flink: Add equality delete conversion committer Adds EqualityConvertCommitter, a two input operator for the equality delete conversion task. It buffers the DVWriteResults from the parallel EqualityConvertDVWriter instances and the EqualityConvertPlan from the planner, then commits once both the plan and its done-timestamp watermark have arrived. The committer adds both new staging data files and the writer's merged DVs, removes the superseded DVs, and carries over the remaining staging deletes. It validates against the planner's main snapshot, so external changes on the target branch fail the commit. The commit summary records the processed staging snapshot id, which the planner reads on restart to skip already-committed snapshots, to ensure idempotency. The committer is responsible for sending Trigger records downstream to the TaskResultAggregator of the surrounding table maintenance framework. It emits one after every cycle, including no-op and error, so the task always completes. On an upstream abort or a commit failure it deletes the DVs written this cycle so the failure doesn't leak Puffin files. * fixup! Rename stale DV resolver/merger references to DV writer * fixup! Reword inaccurate watermark-absorption javadoc on the committer * fixup! Add shared-branch DV-merge regression test for the committer --- .../operator/EqualityConvertCommitter.java | 363 ++++++++++++ .../TestEqualityConvertCommitter.java | 524 ++++++++++++++++++ 2 files changed, 887 insertions(+) create mode 100644 flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java create mode 100644 flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java new file mode 100644 index 000000000000..c83b06a55abf --- /dev/null +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.flink.annotation.Internal; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Commits data files and DVs to the target branch. Receives {@link DVWriteResult}s from parallel + * {@link EqualityConvertDVWriter} instances (input 1) and an {@link EqualityConvertPlan} from the + * planner (input 2). Assembles the final file lists and commits using a {@link RowDelta} operation + * once the plan result and done-timestamp watermark have both arrived. + * + *

The commit is gated on the plan's done-timestamp watermark. + * + *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link + * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for + * the Aggregator. + * + *

No-op vs error: a no-op cycle (empty plan result from {@code + * EqualityConvertPlanner.emitNoOpResult}) returns early in {@code commitIfNeeded} without writing + * anything; the Trigger emit in {@code processWatermark} still happens, so the maintenance task + * completes cleanly. Errors are reported via {@link TaskResultAggregator#ERROR_STREAM} side output; + * the Aggregator collects them and surfaces failure on its own watermark. + * + *

The committer is intentionally stateless: {@code bufferedResults} and {@code planResult} are + * not checkpointed. The maintenance framework ensures mutual exclusive tasks, and on restart the + * planner re-derives its position from {@link #COMMITTED_STAGING_SNAPSHOT_PROPERTY} on main, so the + * committer never receives a plan for an already-committed staging snapshot. + */ +@Internal +public class EqualityConvertCommitter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); + + static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = "equality-convert-staging-snapshot"; + + private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; + private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + + private transient Table table; + private transient List bufferedResults; + private transient EqualityConvertPlan planResult; + + private transient Counter errorCounter; + private transient Counter addedDataFileNumCounter; + private transient Counter addedDataFileSizeCounter; + private transient Counter addedDvNumCounter; + private transient Counter commitDurationMsCounter; + + public EqualityConvertCommitter( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + this.table = tableLoader.loadTable(); + this.bufferedResults = Lists.newArrayList(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + this.addedDataFileNumCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC); + this.addedDataFileSizeCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC); + this.addedDvNumCounter = taskMetricGroup.counter(ADDED_DV_NUM_METRIC); + this.commitDurationMsCounter = taskMetricGroup.counter(COMMIT_DURATION_MS_METRIC); + } + + @Override + public void processElement1(StreamRecord record) { + bufferedResults.add(record.getValue()); + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", + tableName, + taskName, + e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + } + + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + } + + // Always forward watermarks to prevent stalling downstream. + super.processWatermark(mark); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private void commitIfNeeded() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + LOG.warn( + "Skipping commit for table {} task {}: a DV writer reported an error.", + tableName, + taskName); + deleteUncommittedDVs(); + return; + } + } + + // No-op cycle: the planner emitted an empty plan result (see + // EqualityConvertPlanner.emitNoOpResult) because the next staging snapshot was filtered by + // shouldSkip or there was nothing new on staging. processWatermark still forwards a Trigger to + // TaskResultAggregator, so the maintenance task completes cleanly. + if (planResult.noOp()) { + return; + } + + table.refresh(); + + List allDvFiles = Lists.newArrayList(); + List allRewrittenDvFiles = Lists.newArrayList(); + for (DVWriteResult result : bufferedResults) { + allDvFiles.addAll(result.dvFiles()); + allRewrittenDvFiles.addAll(result.rewrittenDvFiles()); + } + + RowDelta rowDelta = buildRowDelta(planResult.dataFiles(), allDvFiles, allRewrittenDvFiles); + + long startNano = System.nanoTime(); + try { + commit(rowDelta); + } catch (CommitStateUnknownException e) { + // Commit outcome unknown: the DVs may already be referenced by a committed snapshot. Leave + // them for Remove Orphan Files rather than risk deleting live data. + throw e; + } catch (Exception e) { + // Commit definitively failed: the DVs this cycle wrote are unreferenced. Delete them so a + // failed cycle does not leak Puffin files. Rewritten DVs stay (still live on the target). + deleteUncommittedDVs(); + throw e; + } + + long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano); + commitDurationMsCounter.inc(durationMs); + + LOG.info( + "Committed {} data files and {} DV files to branch '{}' for table {} in {} ms. " + + "Processed staging snapshot {}.", + planResult.dataFiles().size(), + allDvFiles.size(), + targetBranch, + tableName, + durationMs, + planResult.stagingSnapshotId()); + + // Only count files actually added by this commit. When sameBranch, the writer already + // committed the data files to target and buildRowDelta does not re-add them. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : planResult.dataFiles()) { + addedDataFileNumCounter.inc(); + addedDataFileSizeCounter.inc(dataFile.fileSizeInBytes()); + } + } + + addedDvNumCounter.inc(allDvFiles.size()); + } + + @VisibleForTesting + void commit(RowDelta rowDelta) { + rowDelta.commit(); + } + + /** + * Deletes the DVs this cycle wrote but did not commit (abort or definite commit failure). Only + * the newly written DVs are removed; rewritten DVs remain referenced on the target branch. Best + * effort: a delete failure is logged, not propagated, so it never masks the original error. + */ + private void deleteUncommittedDVs() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + continue; + } + + for (DeleteFile dvFile : result.dvFiles()) { + try { + table.io().deleteFile(dvFile.location()); + } catch (RuntimeException e) { + LOG.warn( + "Failed to delete uncommitted DV {} for table {} task {}", + dvFile.location(), + tableName, + taskName, + e); + } + } + } + } + + private RowDelta buildRowDelta( + List dataFiles, List allDvFiles, List allRewrittenDvFiles) { + RowDelta rowDelta = table.newRowDelta(); + + // Fail the commit on external target-branch activity since the planner's snapshot. The next + // trigger detects the change and reindexes. + if (planResult.mainSnapshotId() != null) { + rowDelta.validateFromSnapshot(planResult.mainSnapshotId()); + } + + rowDelta.validateNoConflictingDataFiles(); + rowDelta.validateNoConflictingDeleteFiles(); + + Set referencedDataFiles = Sets.newHashSet(); + for (DeleteFile dvFile : allDvFiles) { + if (dvFile.referencedDataFile() != null) { + referencedDataFiles.add(dvFile.referencedDataFile()); + } + } + + if (!referencedDataFiles.isEmpty()) { + rowDelta.validateDataFilesExist(referencedDataFiles); + } + + // When stagingBranch == targetBranch, the writer already committed data files and DVs to the + // target branch. Re-adding them here would produce duplicate manifest entries. Skip those + // paths; only the new DVs from the writer need to be added. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : dataFiles) { + rowDelta.addRows(dataFile); + } + } + + for (DeleteFile dvFile : allDvFiles) { + rowDelta.addDeletes(dvFile); + } + + if (!stagingOnTargetBranch) { + addStagingDeletes(rowDelta, referencedDataFiles, planResult.stagingDVFiles()); + } + + removeRewrittenDVs( + rowDelta, allRewrittenDvFiles, planResult.stagingDVFiles(), stagingOnTargetBranch); + + rowDelta.toBranch(targetBranch); + rowDelta.set( + COMMITTED_STAGING_SNAPSHOT_PROPERTY, String.valueOf(planResult.stagingSnapshotId())); + return rowDelta; + } + + /** Adds staging delete files, skipping DVs that overlap with conversion DVs (V3 rule). */ + private static void addStagingDeletes( + RowDelta rowDelta, Set dvCoveredDataFiles, List stagingDVFiles) { + for (DeleteFile stagingDelete : stagingDVFiles) { + // V3 allows one DV per data file. When a staging snapshot contains both a writer-committed + // DV and an eq-delete that resolves to additional positions in the same data file, the + // writer folds the staging DV into a new merged DV (via + // EqualityConvertDVWriter.collectExistingDVs). Skip the superseded staging DV; adding both + // would commit two DVs for the same data file. + if (ContentFileUtil.isDV(stagingDelete) + && stagingDelete.referencedDataFile() != null + && dvCoveredDataFiles.contains(stagingDelete.referencedDataFile())) { + continue; + } + + rowDelta.addDeletes(stagingDelete); + } + } + + /** + * Removes rewritten DVs. On a separate target branch, staging DVs are not yet on target, so they + * are skipped: removeDeletes would fail. On a shared branch they are already on target and must + * be removed like any other rewritten DV; the merged DV that supersedes them would otherwise be a + * second DV for the same data file (V3 allows one). + */ + private static void removeRewrittenDVs( + RowDelta rowDelta, + List allRewrittenDvFiles, + List stagingDVFiles, + boolean stagingOnTargetBranch) { + Set stagingDeleteLocations = Sets.newHashSet(); + for (DeleteFile sd : stagingDVFiles) { + stagingDeleteLocations.add(sd.location()); + } + + for (DeleteFile rewrittenDv : allRewrittenDvFiles) { + if (stagingOnTargetBranch || !stagingDeleteLocations.contains(rewrittenDv.location())) { + rowDelta.removeDeletes(rewrittenDv); + } + } + } +} diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java new file mode 100644 index 000000000000..f74570c878db --- /dev/null +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.List; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertCommitter extends OperatorTestBase { + + @Test + void commitsDataFilesToMainBranch() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 42L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + snapshotIdBefore, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(snapshotIdBefore); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isEqualTo(String.valueOf(stagingSnapshotId)); + } + } + + @Test + void skipsCommitForEmptyCycle() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = EqualityConvertPlan.noOp(null, doneTs - 1, doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot()).isNull(); + } + } + + @Test + void abortsCommitWhenDVWriterFailed() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // Receive an abort signal from the DVWriter followed by the planning planResult. + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // The cycle must complete (Trigger emitted) but nothing must be committed to the table. + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + } + } + + @Test + void failsCommitWhenExternalCommitLandsAfterPlan() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 555L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit. + insert(table, 2, "b"); + table.refresh(); + long mainSnapshotIdAfterExternal = table.currentSnapshot().snapshotId(); + assertThat(mainSnapshotIdAfterExternal).isNotEqualTo(mainSnapshotIdAtPlan); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // Trigger still fires (so the aggregator records the cycle), but the commit + // must have been rejected by validateNoConflictingDataFiles. + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // Target head is still the external commit, not our cycle's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotIdAfterExternal); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isNull(); + } + } + + @Test + void failsReplayOfCommittedPlanFromOlderState() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 909L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + // Cycle 1 commits, advancing the target past mainSnapshotIdAtPlan and writing the marker. + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + + table.refresh(); + long committedSnapshotId = table.currentSnapshot().snapshotId(); + assertThat(committedSnapshotId).isNotEqualTo(mainSnapshotIdAtPlan); + + // Restart from older state: the identical plan with its stale mainSnapshotId is replayed. + // The committer is stateless, so validateFromSnapshot is the only guard against a duplicate + // commit. It must reject the replay. + long doneTs2 = doneTs + 1; + EqualityConvertPlan replay = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs2 - 1, + doneTs2); + + harness.processElement2(new StreamRecord<>(replay, doneTs2 - 1)); + harness.processBothWatermarks(new Watermark(doneTs2)); + + // Trigger still fires (so the aggregator records the cycle), but the commit was rejected. + assertThat(harness.extractOutputValues()).hasSize(2); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // No duplicate commit: target head is still cycle 1's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(committedSnapshotId); + } + } + + @Test + void deletesWrittenDvsWhenCommitFails() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 777L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit, so the commit is rejected. + insert(table, 2, "b"); + table.refresh(); + + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(ValidationException.class); + + // The DV written this cycle is unreferenced after the failed commit and must be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + } + } + + @Test + void deletesSiblingDvsOnAbortButKeepsRewrittenDvs() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + DeleteFile rewrittenDv = writePosDeleteFile(table, stagingDataFile.location(), 1L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // One writer task wrote a DV (rewriting an existing one); a sibling task aborted. + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList(rewrittenDv)), + doneTs)); + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + + // The freshly written DV is removed; the rewritten DV is still live on target and stays. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + } + } + + @Test + void retainsWrittenDvsWhenCommitStateUnknown() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + EqualityConvertCommitter committer = + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH) { + @Override + void commit(RowDelta rowDelta) { + throw new CommitStateUnknownException( + new RuntimeException("simulated unknown commit state")); + } + }; + + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(committer)) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 888L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(CommitStateUnknownException.class); + + // Commit outcome unknown: the DV may be live, so it must not be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + } + } + + @Test + void removesRewrittenStagingDvOnSharedBranch() throws Exception { + // Shared branch: stagingBranch == targetBranch == main, so the writer's DVs are already on + // target. A staging DV folded into a merged conversion DV must be removed, else the data file + // would carry two DVs (V3 allows one per data file). + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile dataFile = getFirstDataFile(table); + + // Staging DV already committed on the shared target branch; the writer rewrites it by folding + // it into a merged DV for the same data file. + DeleteFile stagingDv = writeDV(table, dataFile.location(), 0L); + table.newRowDelta().addDeletes(stagingDv).commit(); + table.refresh(); + long mainSnapshotId = table.currentSnapshot().snapshotId(); + + DeleteFile mergedDv = writeDV(table, dataFile.location(), 0L); + + try (TwoInputStreamOperatorTestHarness harness = + sharedBranchHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), + Lists.newArrayList(stagingDv), + 123L, + mainSnapshotId, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(mergedDv), Lists.newArrayList(stagingDv)), + doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Exactly one DV references the data file: the merged DV, with the rewritten staging DV + // removed rather than left as a second DV on the same data file. + table.refresh(); + List dvs = deletesForDataFile(table, dataFile.location()); + assertThat(dvs).hasSize(1); + assertThat(dvs.get(0).location()).isEqualTo(mergedDv.location()); + } + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH)); + } + + private TwoInputStreamOperatorTestHarness + sharedBranchHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + SnapshotRef.MAIN_BRANCH, + SnapshotRef.MAIN_BRANCH)); + } + + private static List deletesForDataFile(Table table, String dataFilePath) { + List deletes = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (dataFilePath.equals(file.referencedDataFile())) { + deletes.add(file.copy()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return deletes; + } + + private static DeleteFile writeDV(Table table, String dataFilePath, long... positions) + throws IOException { + List> deletes = Lists.newArrayList(); + GenericRecord nested = GenericRecord.create(table.schema()); + for (long pos : positions) { + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, pos, nested); + deletes.add(delete); + } + + return FileHelpers.writePosDeleteFile(table, null, null, deletes, 3); + } + + private static DataFile getFirstDataFile(Table table) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + return file.copy(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data files found"); + } +} From 2a2d5c0a6230a1df08bbca8dc5c93d53122081fd Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Sat, 20 Jun 2026 20:19:25 +0200 Subject: [PATCH 06/73] Flink: Backport: Add equality delete conversion committer (#16874) (#16888) --- .../operator/EqualityConvertCommitter.java | 363 ++++++++++++ .../TestEqualityConvertCommitter.java | 524 ++++++++++++++++++ .../operator/EqualityConvertCommitter.java | 363 ++++++++++++ .../TestEqualityConvertCommitter.java | 524 ++++++++++++++++++ 4 files changed, 1774 insertions(+) create mode 100644 flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java create mode 100644 flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java create mode 100644 flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java create mode 100644 flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java new file mode 100644 index 000000000000..c83b06a55abf --- /dev/null +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.flink.annotation.Internal; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Commits data files and DVs to the target branch. Receives {@link DVWriteResult}s from parallel + * {@link EqualityConvertDVWriter} instances (input 1) and an {@link EqualityConvertPlan} from the + * planner (input 2). Assembles the final file lists and commits using a {@link RowDelta} operation + * once the plan result and done-timestamp watermark have both arrived. + * + *

The commit is gated on the plan's done-timestamp watermark. + * + *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link + * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for + * the Aggregator. + * + *

No-op vs error: a no-op cycle (empty plan result from {@code + * EqualityConvertPlanner.emitNoOpResult}) returns early in {@code commitIfNeeded} without writing + * anything; the Trigger emit in {@code processWatermark} still happens, so the maintenance task + * completes cleanly. Errors are reported via {@link TaskResultAggregator#ERROR_STREAM} side output; + * the Aggregator collects them and surfaces failure on its own watermark. + * + *

The committer is intentionally stateless: {@code bufferedResults} and {@code planResult} are + * not checkpointed. The maintenance framework ensures mutual exclusive tasks, and on restart the + * planner re-derives its position from {@link #COMMITTED_STAGING_SNAPSHOT_PROPERTY} on main, so the + * committer never receives a plan for an already-committed staging snapshot. + */ +@Internal +public class EqualityConvertCommitter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); + + static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = "equality-convert-staging-snapshot"; + + private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; + private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + + private transient Table table; + private transient List bufferedResults; + private transient EqualityConvertPlan planResult; + + private transient Counter errorCounter; + private transient Counter addedDataFileNumCounter; + private transient Counter addedDataFileSizeCounter; + private transient Counter addedDvNumCounter; + private transient Counter commitDurationMsCounter; + + public EqualityConvertCommitter( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + this.table = tableLoader.loadTable(); + this.bufferedResults = Lists.newArrayList(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + this.addedDataFileNumCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC); + this.addedDataFileSizeCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC); + this.addedDvNumCounter = taskMetricGroup.counter(ADDED_DV_NUM_METRIC); + this.commitDurationMsCounter = taskMetricGroup.counter(COMMIT_DURATION_MS_METRIC); + } + + @Override + public void processElement1(StreamRecord record) { + bufferedResults.add(record.getValue()); + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", + tableName, + taskName, + e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + } + + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + } + + // Always forward watermarks to prevent stalling downstream. + super.processWatermark(mark); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private void commitIfNeeded() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + LOG.warn( + "Skipping commit for table {} task {}: a DV writer reported an error.", + tableName, + taskName); + deleteUncommittedDVs(); + return; + } + } + + // No-op cycle: the planner emitted an empty plan result (see + // EqualityConvertPlanner.emitNoOpResult) because the next staging snapshot was filtered by + // shouldSkip or there was nothing new on staging. processWatermark still forwards a Trigger to + // TaskResultAggregator, so the maintenance task completes cleanly. + if (planResult.noOp()) { + return; + } + + table.refresh(); + + List allDvFiles = Lists.newArrayList(); + List allRewrittenDvFiles = Lists.newArrayList(); + for (DVWriteResult result : bufferedResults) { + allDvFiles.addAll(result.dvFiles()); + allRewrittenDvFiles.addAll(result.rewrittenDvFiles()); + } + + RowDelta rowDelta = buildRowDelta(planResult.dataFiles(), allDvFiles, allRewrittenDvFiles); + + long startNano = System.nanoTime(); + try { + commit(rowDelta); + } catch (CommitStateUnknownException e) { + // Commit outcome unknown: the DVs may already be referenced by a committed snapshot. Leave + // them for Remove Orphan Files rather than risk deleting live data. + throw e; + } catch (Exception e) { + // Commit definitively failed: the DVs this cycle wrote are unreferenced. Delete them so a + // failed cycle does not leak Puffin files. Rewritten DVs stay (still live on the target). + deleteUncommittedDVs(); + throw e; + } + + long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano); + commitDurationMsCounter.inc(durationMs); + + LOG.info( + "Committed {} data files and {} DV files to branch '{}' for table {} in {} ms. " + + "Processed staging snapshot {}.", + planResult.dataFiles().size(), + allDvFiles.size(), + targetBranch, + tableName, + durationMs, + planResult.stagingSnapshotId()); + + // Only count files actually added by this commit. When sameBranch, the writer already + // committed the data files to target and buildRowDelta does not re-add them. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : planResult.dataFiles()) { + addedDataFileNumCounter.inc(); + addedDataFileSizeCounter.inc(dataFile.fileSizeInBytes()); + } + } + + addedDvNumCounter.inc(allDvFiles.size()); + } + + @VisibleForTesting + void commit(RowDelta rowDelta) { + rowDelta.commit(); + } + + /** + * Deletes the DVs this cycle wrote but did not commit (abort or definite commit failure). Only + * the newly written DVs are removed; rewritten DVs remain referenced on the target branch. Best + * effort: a delete failure is logged, not propagated, so it never masks the original error. + */ + private void deleteUncommittedDVs() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + continue; + } + + for (DeleteFile dvFile : result.dvFiles()) { + try { + table.io().deleteFile(dvFile.location()); + } catch (RuntimeException e) { + LOG.warn( + "Failed to delete uncommitted DV {} for table {} task {}", + dvFile.location(), + tableName, + taskName, + e); + } + } + } + } + + private RowDelta buildRowDelta( + List dataFiles, List allDvFiles, List allRewrittenDvFiles) { + RowDelta rowDelta = table.newRowDelta(); + + // Fail the commit on external target-branch activity since the planner's snapshot. The next + // trigger detects the change and reindexes. + if (planResult.mainSnapshotId() != null) { + rowDelta.validateFromSnapshot(planResult.mainSnapshotId()); + } + + rowDelta.validateNoConflictingDataFiles(); + rowDelta.validateNoConflictingDeleteFiles(); + + Set referencedDataFiles = Sets.newHashSet(); + for (DeleteFile dvFile : allDvFiles) { + if (dvFile.referencedDataFile() != null) { + referencedDataFiles.add(dvFile.referencedDataFile()); + } + } + + if (!referencedDataFiles.isEmpty()) { + rowDelta.validateDataFilesExist(referencedDataFiles); + } + + // When stagingBranch == targetBranch, the writer already committed data files and DVs to the + // target branch. Re-adding them here would produce duplicate manifest entries. Skip those + // paths; only the new DVs from the writer need to be added. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : dataFiles) { + rowDelta.addRows(dataFile); + } + } + + for (DeleteFile dvFile : allDvFiles) { + rowDelta.addDeletes(dvFile); + } + + if (!stagingOnTargetBranch) { + addStagingDeletes(rowDelta, referencedDataFiles, planResult.stagingDVFiles()); + } + + removeRewrittenDVs( + rowDelta, allRewrittenDvFiles, planResult.stagingDVFiles(), stagingOnTargetBranch); + + rowDelta.toBranch(targetBranch); + rowDelta.set( + COMMITTED_STAGING_SNAPSHOT_PROPERTY, String.valueOf(planResult.stagingSnapshotId())); + return rowDelta; + } + + /** Adds staging delete files, skipping DVs that overlap with conversion DVs (V3 rule). */ + private static void addStagingDeletes( + RowDelta rowDelta, Set dvCoveredDataFiles, List stagingDVFiles) { + for (DeleteFile stagingDelete : stagingDVFiles) { + // V3 allows one DV per data file. When a staging snapshot contains both a writer-committed + // DV and an eq-delete that resolves to additional positions in the same data file, the + // writer folds the staging DV into a new merged DV (via + // EqualityConvertDVWriter.collectExistingDVs). Skip the superseded staging DV; adding both + // would commit two DVs for the same data file. + if (ContentFileUtil.isDV(stagingDelete) + && stagingDelete.referencedDataFile() != null + && dvCoveredDataFiles.contains(stagingDelete.referencedDataFile())) { + continue; + } + + rowDelta.addDeletes(stagingDelete); + } + } + + /** + * Removes rewritten DVs. On a separate target branch, staging DVs are not yet on target, so they + * are skipped: removeDeletes would fail. On a shared branch they are already on target and must + * be removed like any other rewritten DV; the merged DV that supersedes them would otherwise be a + * second DV for the same data file (V3 allows one). + */ + private static void removeRewrittenDVs( + RowDelta rowDelta, + List allRewrittenDvFiles, + List stagingDVFiles, + boolean stagingOnTargetBranch) { + Set stagingDeleteLocations = Sets.newHashSet(); + for (DeleteFile sd : stagingDVFiles) { + stagingDeleteLocations.add(sd.location()); + } + + for (DeleteFile rewrittenDv : allRewrittenDvFiles) { + if (stagingOnTargetBranch || !stagingDeleteLocations.contains(rewrittenDv.location())) { + rowDelta.removeDeletes(rewrittenDv); + } + } + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java new file mode 100644 index 000000000000..f74570c878db --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.List; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertCommitter extends OperatorTestBase { + + @Test + void commitsDataFilesToMainBranch() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 42L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + snapshotIdBefore, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(snapshotIdBefore); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isEqualTo(String.valueOf(stagingSnapshotId)); + } + } + + @Test + void skipsCommitForEmptyCycle() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = EqualityConvertPlan.noOp(null, doneTs - 1, doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot()).isNull(); + } + } + + @Test + void abortsCommitWhenDVWriterFailed() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // Receive an abort signal from the DVWriter followed by the planning planResult. + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // The cycle must complete (Trigger emitted) but nothing must be committed to the table. + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + } + } + + @Test + void failsCommitWhenExternalCommitLandsAfterPlan() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 555L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit. + insert(table, 2, "b"); + table.refresh(); + long mainSnapshotIdAfterExternal = table.currentSnapshot().snapshotId(); + assertThat(mainSnapshotIdAfterExternal).isNotEqualTo(mainSnapshotIdAtPlan); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // Trigger still fires (so the aggregator records the cycle), but the commit + // must have been rejected by validateNoConflictingDataFiles. + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // Target head is still the external commit, not our cycle's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotIdAfterExternal); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isNull(); + } + } + + @Test + void failsReplayOfCommittedPlanFromOlderState() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 909L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + // Cycle 1 commits, advancing the target past mainSnapshotIdAtPlan and writing the marker. + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + + table.refresh(); + long committedSnapshotId = table.currentSnapshot().snapshotId(); + assertThat(committedSnapshotId).isNotEqualTo(mainSnapshotIdAtPlan); + + // Restart from older state: the identical plan with its stale mainSnapshotId is replayed. + // The committer is stateless, so validateFromSnapshot is the only guard against a duplicate + // commit. It must reject the replay. + long doneTs2 = doneTs + 1; + EqualityConvertPlan replay = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs2 - 1, + doneTs2); + + harness.processElement2(new StreamRecord<>(replay, doneTs2 - 1)); + harness.processBothWatermarks(new Watermark(doneTs2)); + + // Trigger still fires (so the aggregator records the cycle), but the commit was rejected. + assertThat(harness.extractOutputValues()).hasSize(2); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // No duplicate commit: target head is still cycle 1's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(committedSnapshotId); + } + } + + @Test + void deletesWrittenDvsWhenCommitFails() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 777L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit, so the commit is rejected. + insert(table, 2, "b"); + table.refresh(); + + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(ValidationException.class); + + // The DV written this cycle is unreferenced after the failed commit and must be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + } + } + + @Test + void deletesSiblingDvsOnAbortButKeepsRewrittenDvs() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + DeleteFile rewrittenDv = writePosDeleteFile(table, stagingDataFile.location(), 1L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // One writer task wrote a DV (rewriting an existing one); a sibling task aborted. + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList(rewrittenDv)), + doneTs)); + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + + // The freshly written DV is removed; the rewritten DV is still live on target and stays. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + } + } + + @Test + void retainsWrittenDvsWhenCommitStateUnknown() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + EqualityConvertCommitter committer = + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH) { + @Override + void commit(RowDelta rowDelta) { + throw new CommitStateUnknownException( + new RuntimeException("simulated unknown commit state")); + } + }; + + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(committer)) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 888L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(CommitStateUnknownException.class); + + // Commit outcome unknown: the DV may be live, so it must not be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + } + } + + @Test + void removesRewrittenStagingDvOnSharedBranch() throws Exception { + // Shared branch: stagingBranch == targetBranch == main, so the writer's DVs are already on + // target. A staging DV folded into a merged conversion DV must be removed, else the data file + // would carry two DVs (V3 allows one per data file). + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile dataFile = getFirstDataFile(table); + + // Staging DV already committed on the shared target branch; the writer rewrites it by folding + // it into a merged DV for the same data file. + DeleteFile stagingDv = writeDV(table, dataFile.location(), 0L); + table.newRowDelta().addDeletes(stagingDv).commit(); + table.refresh(); + long mainSnapshotId = table.currentSnapshot().snapshotId(); + + DeleteFile mergedDv = writeDV(table, dataFile.location(), 0L); + + try (TwoInputStreamOperatorTestHarness harness = + sharedBranchHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), + Lists.newArrayList(stagingDv), + 123L, + mainSnapshotId, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(mergedDv), Lists.newArrayList(stagingDv)), + doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Exactly one DV references the data file: the merged DV, with the rewritten staging DV + // removed rather than left as a second DV on the same data file. + table.refresh(); + List dvs = deletesForDataFile(table, dataFile.location()); + assertThat(dvs).hasSize(1); + assertThat(dvs.get(0).location()).isEqualTo(mergedDv.location()); + } + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH)); + } + + private TwoInputStreamOperatorTestHarness + sharedBranchHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + SnapshotRef.MAIN_BRANCH, + SnapshotRef.MAIN_BRANCH)); + } + + private static List deletesForDataFile(Table table, String dataFilePath) { + List deletes = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (dataFilePath.equals(file.referencedDataFile())) { + deletes.add(file.copy()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return deletes; + } + + private static DeleteFile writeDV(Table table, String dataFilePath, long... positions) + throws IOException { + List> deletes = Lists.newArrayList(); + GenericRecord nested = GenericRecord.create(table.schema()); + for (long pos : positions) { + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, pos, nested); + deletes.add(delete); + } + + return FileHelpers.writePosDeleteFile(table, null, null, deletes, 3); + } + + private static DataFile getFirstDataFile(Table table) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + return file.copy(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data files found"); + } +} diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java new file mode 100644 index 000000000000..c83b06a55abf --- /dev/null +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.flink.annotation.Internal; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Commits data files and DVs to the target branch. Receives {@link DVWriteResult}s from parallel + * {@link EqualityConvertDVWriter} instances (input 1) and an {@link EqualityConvertPlan} from the + * planner (input 2). Assembles the final file lists and commits using a {@link RowDelta} operation + * once the plan result and done-timestamp watermark have both arrived. + * + *

The commit is gated on the plan's done-timestamp watermark. + * + *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link + * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for + * the Aggregator. + * + *

No-op vs error: a no-op cycle (empty plan result from {@code + * EqualityConvertPlanner.emitNoOpResult}) returns early in {@code commitIfNeeded} without writing + * anything; the Trigger emit in {@code processWatermark} still happens, so the maintenance task + * completes cleanly. Errors are reported via {@link TaskResultAggregator#ERROR_STREAM} side output; + * the Aggregator collects them and surfaces failure on its own watermark. + * + *

The committer is intentionally stateless: {@code bufferedResults} and {@code planResult} are + * not checkpointed. The maintenance framework ensures mutual exclusive tasks, and on restart the + * planner re-derives its position from {@link #COMMITTED_STAGING_SNAPSHOT_PROPERTY} on main, so the + * committer never receives a plan for an already-committed staging snapshot. + */ +@Internal +public class EqualityConvertCommitter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); + + static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = "equality-convert-staging-snapshot"; + + private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; + private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + + private transient Table table; + private transient List bufferedResults; + private transient EqualityConvertPlan planResult; + + private transient Counter errorCounter; + private transient Counter addedDataFileNumCounter; + private transient Counter addedDataFileSizeCounter; + private transient Counter addedDvNumCounter; + private transient Counter commitDurationMsCounter; + + public EqualityConvertCommitter( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + this.table = tableLoader.loadTable(); + this.bufferedResults = Lists.newArrayList(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + this.addedDataFileNumCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC); + this.addedDataFileSizeCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC); + this.addedDvNumCounter = taskMetricGroup.counter(ADDED_DV_NUM_METRIC); + this.commitDurationMsCounter = taskMetricGroup.counter(COMMIT_DURATION_MS_METRIC); + } + + @Override + public void processElement1(StreamRecord record) { + bufferedResults.add(record.getValue()); + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", + tableName, + taskName, + e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + } + + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + } + + // Always forward watermarks to prevent stalling downstream. + super.processWatermark(mark); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private void commitIfNeeded() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + LOG.warn( + "Skipping commit for table {} task {}: a DV writer reported an error.", + tableName, + taskName); + deleteUncommittedDVs(); + return; + } + } + + // No-op cycle: the planner emitted an empty plan result (see + // EqualityConvertPlanner.emitNoOpResult) because the next staging snapshot was filtered by + // shouldSkip or there was nothing new on staging. processWatermark still forwards a Trigger to + // TaskResultAggregator, so the maintenance task completes cleanly. + if (planResult.noOp()) { + return; + } + + table.refresh(); + + List allDvFiles = Lists.newArrayList(); + List allRewrittenDvFiles = Lists.newArrayList(); + for (DVWriteResult result : bufferedResults) { + allDvFiles.addAll(result.dvFiles()); + allRewrittenDvFiles.addAll(result.rewrittenDvFiles()); + } + + RowDelta rowDelta = buildRowDelta(planResult.dataFiles(), allDvFiles, allRewrittenDvFiles); + + long startNano = System.nanoTime(); + try { + commit(rowDelta); + } catch (CommitStateUnknownException e) { + // Commit outcome unknown: the DVs may already be referenced by a committed snapshot. Leave + // them for Remove Orphan Files rather than risk deleting live data. + throw e; + } catch (Exception e) { + // Commit definitively failed: the DVs this cycle wrote are unreferenced. Delete them so a + // failed cycle does not leak Puffin files. Rewritten DVs stay (still live on the target). + deleteUncommittedDVs(); + throw e; + } + + long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano); + commitDurationMsCounter.inc(durationMs); + + LOG.info( + "Committed {} data files and {} DV files to branch '{}' for table {} in {} ms. " + + "Processed staging snapshot {}.", + planResult.dataFiles().size(), + allDvFiles.size(), + targetBranch, + tableName, + durationMs, + planResult.stagingSnapshotId()); + + // Only count files actually added by this commit. When sameBranch, the writer already + // committed the data files to target and buildRowDelta does not re-add them. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : planResult.dataFiles()) { + addedDataFileNumCounter.inc(); + addedDataFileSizeCounter.inc(dataFile.fileSizeInBytes()); + } + } + + addedDvNumCounter.inc(allDvFiles.size()); + } + + @VisibleForTesting + void commit(RowDelta rowDelta) { + rowDelta.commit(); + } + + /** + * Deletes the DVs this cycle wrote but did not commit (abort or definite commit failure). Only + * the newly written DVs are removed; rewritten DVs remain referenced on the target branch. Best + * effort: a delete failure is logged, not propagated, so it never masks the original error. + */ + private void deleteUncommittedDVs() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + continue; + } + + for (DeleteFile dvFile : result.dvFiles()) { + try { + table.io().deleteFile(dvFile.location()); + } catch (RuntimeException e) { + LOG.warn( + "Failed to delete uncommitted DV {} for table {} task {}", + dvFile.location(), + tableName, + taskName, + e); + } + } + } + } + + private RowDelta buildRowDelta( + List dataFiles, List allDvFiles, List allRewrittenDvFiles) { + RowDelta rowDelta = table.newRowDelta(); + + // Fail the commit on external target-branch activity since the planner's snapshot. The next + // trigger detects the change and reindexes. + if (planResult.mainSnapshotId() != null) { + rowDelta.validateFromSnapshot(planResult.mainSnapshotId()); + } + + rowDelta.validateNoConflictingDataFiles(); + rowDelta.validateNoConflictingDeleteFiles(); + + Set referencedDataFiles = Sets.newHashSet(); + for (DeleteFile dvFile : allDvFiles) { + if (dvFile.referencedDataFile() != null) { + referencedDataFiles.add(dvFile.referencedDataFile()); + } + } + + if (!referencedDataFiles.isEmpty()) { + rowDelta.validateDataFilesExist(referencedDataFiles); + } + + // When stagingBranch == targetBranch, the writer already committed data files and DVs to the + // target branch. Re-adding them here would produce duplicate manifest entries. Skip those + // paths; only the new DVs from the writer need to be added. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : dataFiles) { + rowDelta.addRows(dataFile); + } + } + + for (DeleteFile dvFile : allDvFiles) { + rowDelta.addDeletes(dvFile); + } + + if (!stagingOnTargetBranch) { + addStagingDeletes(rowDelta, referencedDataFiles, planResult.stagingDVFiles()); + } + + removeRewrittenDVs( + rowDelta, allRewrittenDvFiles, planResult.stagingDVFiles(), stagingOnTargetBranch); + + rowDelta.toBranch(targetBranch); + rowDelta.set( + COMMITTED_STAGING_SNAPSHOT_PROPERTY, String.valueOf(planResult.stagingSnapshotId())); + return rowDelta; + } + + /** Adds staging delete files, skipping DVs that overlap with conversion DVs (V3 rule). */ + private static void addStagingDeletes( + RowDelta rowDelta, Set dvCoveredDataFiles, List stagingDVFiles) { + for (DeleteFile stagingDelete : stagingDVFiles) { + // V3 allows one DV per data file. When a staging snapshot contains both a writer-committed + // DV and an eq-delete that resolves to additional positions in the same data file, the + // writer folds the staging DV into a new merged DV (via + // EqualityConvertDVWriter.collectExistingDVs). Skip the superseded staging DV; adding both + // would commit two DVs for the same data file. + if (ContentFileUtil.isDV(stagingDelete) + && stagingDelete.referencedDataFile() != null + && dvCoveredDataFiles.contains(stagingDelete.referencedDataFile())) { + continue; + } + + rowDelta.addDeletes(stagingDelete); + } + } + + /** + * Removes rewritten DVs. On a separate target branch, staging DVs are not yet on target, so they + * are skipped: removeDeletes would fail. On a shared branch they are already on target and must + * be removed like any other rewritten DV; the merged DV that supersedes them would otherwise be a + * second DV for the same data file (V3 allows one). + */ + private static void removeRewrittenDVs( + RowDelta rowDelta, + List allRewrittenDvFiles, + List stagingDVFiles, + boolean stagingOnTargetBranch) { + Set stagingDeleteLocations = Sets.newHashSet(); + for (DeleteFile sd : stagingDVFiles) { + stagingDeleteLocations.add(sd.location()); + } + + for (DeleteFile rewrittenDv : allRewrittenDvFiles) { + if (stagingOnTargetBranch || !stagingDeleteLocations.contains(rewrittenDv.location())) { + rowDelta.removeDeletes(rewrittenDv); + } + } + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java new file mode 100644 index 000000000000..f74570c878db --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.List; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertCommitter extends OperatorTestBase { + + @Test + void commitsDataFilesToMainBranch() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 42L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + snapshotIdBefore, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(snapshotIdBefore); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isEqualTo(String.valueOf(stagingSnapshotId)); + } + } + + @Test + void skipsCommitForEmptyCycle() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = EqualityConvertPlan.noOp(null, doneTs - 1, doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot()).isNull(); + } + } + + @Test + void abortsCommitWhenDVWriterFailed() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // Receive an abort signal from the DVWriter followed by the planning planResult. + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // The cycle must complete (Trigger emitted) but nothing must be committed to the table. + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + } + } + + @Test + void failsCommitWhenExternalCommitLandsAfterPlan() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 555L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit. + insert(table, 2, "b"); + table.refresh(); + long mainSnapshotIdAfterExternal = table.currentSnapshot().snapshotId(); + assertThat(mainSnapshotIdAfterExternal).isNotEqualTo(mainSnapshotIdAtPlan); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // Trigger still fires (so the aggregator records the cycle), but the commit + // must have been rejected by validateNoConflictingDataFiles. + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // Target head is still the external commit, not our cycle's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotIdAfterExternal); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isNull(); + } + } + + @Test + void failsReplayOfCommittedPlanFromOlderState() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 909L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + // Cycle 1 commits, advancing the target past mainSnapshotIdAtPlan and writing the marker. + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + + table.refresh(); + long committedSnapshotId = table.currentSnapshot().snapshotId(); + assertThat(committedSnapshotId).isNotEqualTo(mainSnapshotIdAtPlan); + + // Restart from older state: the identical plan with its stale mainSnapshotId is replayed. + // The committer is stateless, so validateFromSnapshot is the only guard against a duplicate + // commit. It must reject the replay. + long doneTs2 = doneTs + 1; + EqualityConvertPlan replay = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs2 - 1, + doneTs2); + + harness.processElement2(new StreamRecord<>(replay, doneTs2 - 1)); + harness.processBothWatermarks(new Watermark(doneTs2)); + + // Trigger still fires (so the aggregator records the cycle), but the commit was rejected. + assertThat(harness.extractOutputValues()).hasSize(2); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // No duplicate commit: target head is still cycle 1's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(committedSnapshotId); + } + } + + @Test + void deletesWrittenDvsWhenCommitFails() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 777L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit, so the commit is rejected. + insert(table, 2, "b"); + table.refresh(); + + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(ValidationException.class); + + // The DV written this cycle is unreferenced after the failed commit and must be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + } + } + + @Test + void deletesSiblingDvsOnAbortButKeepsRewrittenDvs() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + DeleteFile rewrittenDv = writePosDeleteFile(table, stagingDataFile.location(), 1L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // One writer task wrote a DV (rewriting an existing one); a sibling task aborted. + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList(rewrittenDv)), + doneTs)); + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + + // The freshly written DV is removed; the rewritten DV is still live on target and stays. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + } + } + + @Test + void retainsWrittenDvsWhenCommitStateUnknown() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + EqualityConvertCommitter committer = + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH) { + @Override + void commit(RowDelta rowDelta) { + throw new CommitStateUnknownException( + new RuntimeException("simulated unknown commit state")); + } + }; + + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(committer)) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 888L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(CommitStateUnknownException.class); + + // Commit outcome unknown: the DV may be live, so it must not be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + } + } + + @Test + void removesRewrittenStagingDvOnSharedBranch() throws Exception { + // Shared branch: stagingBranch == targetBranch == main, so the writer's DVs are already on + // target. A staging DV folded into a merged conversion DV must be removed, else the data file + // would carry two DVs (V3 allows one per data file). + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile dataFile = getFirstDataFile(table); + + // Staging DV already committed on the shared target branch; the writer rewrites it by folding + // it into a merged DV for the same data file. + DeleteFile stagingDv = writeDV(table, dataFile.location(), 0L); + table.newRowDelta().addDeletes(stagingDv).commit(); + table.refresh(); + long mainSnapshotId = table.currentSnapshot().snapshotId(); + + DeleteFile mergedDv = writeDV(table, dataFile.location(), 0L); + + try (TwoInputStreamOperatorTestHarness harness = + sharedBranchHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), + Lists.newArrayList(stagingDv), + 123L, + mainSnapshotId, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(mergedDv), Lists.newArrayList(stagingDv)), + doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Exactly one DV references the data file: the merged DV, with the rewritten staging DV + // removed rather than left as a second DV on the same data file. + table.refresh(); + List dvs = deletesForDataFile(table, dataFile.location()); + assertThat(dvs).hasSize(1); + assertThat(dvs.get(0).location()).isEqualTo(mergedDv.location()); + } + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH)); + } + + private TwoInputStreamOperatorTestHarness + sharedBranchHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + SnapshotRef.MAIN_BRANCH, + SnapshotRef.MAIN_BRANCH)); + } + + private static List deletesForDataFile(Table table, String dataFilePath) { + List deletes = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (dataFilePath.equals(file.referencedDataFile())) { + deletes.add(file.copy()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return deletes; + } + + private static DeleteFile writeDV(Table table, String dataFilePath, long... positions) + throws IOException { + List> deletes = Lists.newArrayList(); + GenericRecord nested = GenericRecord.create(table.schema()); + for (long pos : positions) { + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, pos, nested); + deletes.add(delete); + } + + return FileHelpers.writePosDeleteFile(table, null, null, deletes, 3); + } + + private static DataFile getFirstDataFile(Table table) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + return file.copy(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data files found"); + } +} From 9bf1b25159c8ae03fea2bda67c22b13d7e4e6c27 Mon Sep 17 00:00:00 2001 From: Eunbin Son <58901024+thswlsqls@users.noreply.github.com> Date: Sun, 21 Jun 2026 05:15:02 +0900 Subject: [PATCH 07/73] Aliyun: Pass known file length through OSSFileIO.newInputFile (#16870) Override newInputFile(String, long) so OSSFileIO uses the known length instead of falling back to the default that drops it. Without the override, getLength() triggers a HEAD request (getSimplifiedObjectMeta) even when the caller already knows the size. S3FileIO, GCSFileIO, and ADLSFileIO already override this method. Generated-by: Claude Code (claude-opus-4-8) --- .../apache/iceberg/aliyun/oss/OSSFileIO.java | 5 ++++ .../iceberg/aliyun/oss/TestOSSFileIO.java | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java b/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java index be85b93a75f5..3096cfe4e986 100644 --- a/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java +++ b/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java @@ -76,6 +76,11 @@ public InputFile newInputFile(String path) { return new OSSInputFile(client(), new OSSURI(path), aliyunProperties, metrics); } + @Override + public InputFile newInputFile(String path, long length) { + return new OSSInputFile(client(), new OSSURI(path), aliyunProperties, length, metrics); + } + @Override public OutputFile newOutputFile(String path) { return new OSSOutputFile(client(), new OSSURI(path), aliyunProperties, metrics); diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java index dda4e75c2d00..eff7daf8b369 100644 --- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java +++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java @@ -19,6 +19,10 @@ package org.apache.iceberg.aliyun.oss; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.AdditionalAnswers.delegatesTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClient; @@ -99,6 +103,29 @@ public void testInputFile() throws IOException { assertThat(inFileContent(in, dataSize)).as("Should have expected content").isEqualTo(data); } + @Test + public void testNewInputFileWithLength() throws IOException { + String location = randomLocation(); + int dataSize = 1024 * 10; + byte[] data = randomData(dataSize); + OutputFile out = fileIO().newOutputFile(location); + writeOSSData(out, data); + + OSSURI uri = new OSSURI(location); + OSS ossMock = mock(OSS.class, delegatesTo(ossClient().get())); + try (FileIO io = new OSSFileIO(() -> ossMock)) { + InputFile in = io.newInputFile(location, dataSize); + assertThat(in.getLength()).as("Should return the known length").isEqualTo(dataSize); + verify(ossMock, times(0)).getSimplifiedObjectMeta(uri.bucket(), uri.key()); + + InputFile inWithoutLength = io.newInputFile(location); + assertThat(inWithoutLength.getLength()) + .as("Should return the actual length") + .isEqualTo(dataSize); + verify(ossMock, times(1)).getSimplifiedObjectMeta(uri.bucket(), uri.key()); + } + } + @Test public void testDeleteFile() throws IOException { String location = randomLocation(); From 56b1e19707913609a4596f9eb8cd03b7519e3ad1 Mon Sep 17 00:00:00 2001 From: Neelesh Salian Date: Sat, 20 Jun 2026 13:27:26 -0700 Subject: [PATCH 08/73] Parquet: Fix variant metrics crash when value column has no stats (#16585) * Parquet: Fix variant metrics crash when value column has no stats * Fixup after 16327 --- .../iceberg/parquet/ParquetMetrics.java | 5 ++ .../iceberg/parquet/TestVariantMetrics.java | 47 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java index af6566e747b2..57bfc4295f12 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java @@ -477,6 +477,11 @@ public Iterable value( return typedResult; } + if (Iterables.isEmpty(valueResult)) { + // missing value stats invalidate typed bounds + return ImmutableList.of(); + } + ParquetVariantUtil.VariantMetrics valueMetrics = Iterables.getOnlyElement(valueResult); if (typedResult != null && valueMetrics.valueCount() == valueMetrics.nullCount()) { // all the variant-encoded values are null, so the typed stats can be used diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java index b451558a39e8..b7b63d6f6b80 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java @@ -26,7 +26,9 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; @@ -34,6 +36,7 @@ import org.apache.iceberg.inmemory.InMemoryOutputFile; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -48,6 +51,10 @@ import org.apache.iceberg.variants.VariantTestUtil; import org.apache.iceberg.variants.VariantValue; import org.apache.iceberg.variants.Variants; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.hadoop.ParquetFileWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.schema.MessageType; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.FieldSource; @@ -479,6 +486,46 @@ public void testShreddedObjectFieldTypeMismatch() throws IOException { .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get())); } + @Test + public void testShreddedValueColumnWithEmptyStats() throws IOException { + // typed bounds are dropped when value-column stats are missing on a shredded variant + OutputFile out = new InMemoryOutputFile(); + GenericRecord record = GenericRecord.create(SCHEMA); + + VariantShreddingFunction shredding = + (id, name) -> ParquetVariantUtil.toParquetSchema(Variants.of((byte) 0)); + MessageType parquetSchema = ParquetSchemaUtil.convert(SCHEMA, "table", shredding); + ParquetProperties props = ParquetProperties.builder().withStatisticsEnabled(false).build(); + + // Parquet.write() cannot disable stats on variant sub-columns (no field IDs) + ParquetWriter writer = + new ParquetWriter<>( + new Configuration(), + out, + SCHEMA, + parquetSchema, + 1024, + ImmutableMap.of(), + (s, m) -> InternalWriter.create(s.asStruct(), m), + CompressionCodecName.SNAPPY, + props, + MetricsConfig.getDefault(), + ParquetFileWriter.Mode.CREATE, + null, + false); + + try (writer) { + record.setField("id", 1L); + record.setField("var", Variant.of(EMPTY, Variants.of((byte) 5))); + writer.add(record); + } + + Metrics metrics = writer.metrics(); + assertThat(metrics.recordCount()).isEqualTo(1L); + assertThat(metrics.lowerBounds()).doesNotContainKey(2); + assertThat(metrics.upperBounds()).doesNotContainKey(2); + } + private Metrics writeParquet(VariantShreddingFunction shredding, Variant... variants) throws IOException { OutputFile out = new InMemoryOutputFile(); From 55e025f0cc32df2658105f1714dad334cb1ad215 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:05:30 -0700 Subject: [PATCH 09/73] Build: Bump com.google.cloud.gcs.analytics:gcs-analytics-core (#16904) Bumps [com.google.cloud.gcs.analytics:gcs-analytics-core](https://github.com/GoogleCloudPlatform/gcs-analytics-core) from 1.3.0 to 1.3.1. - [Release notes](https://github.com/GoogleCloudPlatform/gcs-analytics-core/releases) - [Changelog](https://github.com/GoogleCloudPlatform/gcs-analytics-core/blob/main/CHANGELOG.md) - [Commits](https://github.com/GoogleCloudPlatform/gcs-analytics-core/compare/v1.3.0...v1.3.1) --- updated-dependencies: - dependency-name: com.google.cloud.gcs.analytics:gcs-analytics-core dependency-version: 1.3.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1a31b6f103fd..dde27fbc34f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -52,7 +52,7 @@ flink120 = { strictly = "1.20.1"} flink20 = { strictly = "2.0.0"} flink21 = { strictly = "2.1.0"} google-libraries-bom = "26.83.0" -gcs-analytics-core = "1.3.0" +gcs-analytics-core = "1.3.1" guava = "33.6.0-jre" hadoop3 = "3.4.3" httpcomponents-httpclient5 = "5.6.1" From e4d49808f89c70f003ab91b6677c70f956e3ccf2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:05:54 -0700 Subject: [PATCH 10/73] Build: Bump org.openapitools:openapi-generator-gradle-plugin (#16903) Bumps [org.openapitools:openapi-generator-gradle-plugin](https://github.com/OpenAPITools/openapi-generator) from 7.22.0 to 7.23.0. - [Release notes](https://github.com/OpenAPITools/openapi-generator/releases) - [Changelog](https://github.com/OpenAPITools/openapi-generator/blob/master/docs/release-summary.md) - [Commits](https://github.com/OpenAPITools/openapi-generator/compare/v7.22.0...v7.23.0) --- updated-dependencies: - dependency-name: org.openapitools:openapi-generator-gradle-plugin dependency-version: 7.23.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 90a38f633fa3..01e282ecf9dd 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ buildscript { classpath 'org.revapi:gradle-revapi:1.8.0' classpath 'com.gorylenko.gradle-git-properties:gradle-git-properties:2.5.7' classpath 'com.palantir.gradle.gitversion:gradle-git-version:4.3.0' - classpath 'org.openapitools:openapi-generator-gradle-plugin:7.22.0' + classpath 'org.openapitools:openapi-generator-gradle-plugin:7.23.0' } } From d0a05b0f8228ef2fdcf8e5caf6bf682e5ddad613 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:06:15 -0700 Subject: [PATCH 11/73] Build: Bump io.grpc:grpc-netty-shaded from 1.81.0 to 1.82.0 (#16902) Bumps [io.grpc:grpc-netty-shaded](https://github.com/grpc/grpc-java) from 1.81.0 to 1.82.0. - [Release notes](https://github.com/grpc/grpc-java/releases) - [Commits](https://github.com/grpc/grpc-java/compare/v1.81.0...v1.82.0) --- updated-dependencies: - dependency-name: io.grpc:grpc-netty-shaded dependency-version: 1.82.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- kafka-connect/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-connect/build.gradle b/kafka-connect/build.gradle index 5dec94dd5702..8dcea6f7ce37 100644 --- a/kafka-connect/build.gradle +++ b/kafka-connect/build.gradle @@ -81,7 +81,7 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') { force 'org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.5.0' force 'com.fasterxml.woodstox:woodstox-core:6.7.0' force 'commons-beanutils:commons-beanutils:1.11.0' - force 'io.grpc:grpc-netty-shaded:1.81.0' + force 'io.grpc:grpc-netty-shaded:1.82.0' } } } From 926368f15be78fe16b12f7ab553797a0926aa07e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:06:38 -0700 Subject: [PATCH 12/73] Build: Bump software.amazon.awssdk:bom from 2.46.5 to 2.46.10 (#16901) Bumps software.amazon.awssdk:bom from 2.46.5 to 2.46.10. --- updated-dependencies: - dependency-name: software.amazon.awssdk:bom dependency-version: 2.46.10 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index dde27fbc34f2..b2254e1050f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -33,7 +33,7 @@ arrow = "15.0.2" avro = "1.12.1" assertj-core = "3.27.7" awaitility = "4.3.0" -awssdk-bom = "2.46.5" +awssdk-bom = "2.46.10" azuresdk-bom = "1.3.7" awssdk-s3accessgrants = "2.4.1" bouncycastle = "1.84" From ebcb902f58f4fd00a541e668cbbba416cc1ba030 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:06:59 -0700 Subject: [PATCH 13/73] Build: Bump com.google.errorprone:error_prone_annotations (#16900) Bumps [com.google.errorprone:error_prone_annotations](https://github.com/google/error-prone) from 2.49.0 to 2.50.0. - [Release notes](https://github.com/google/error-prone/releases) - [Commits](https://github.com/google/error-prone/compare/v2.49.0...v2.50.0) --- updated-dependencies: - dependency-name: com.google.errorprone:error_prone_annotations dependency-version: 2.50.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b2254e1050f2..e995c8e1bb91 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -45,7 +45,7 @@ delta-standalone = "3.3.2" delta-spark = "3.3.2" derby = "10.15.2.0" esotericsoftware-kryo = "4.0.3" -errorprone-annotations = "2.49.0" +errorprone-annotations = "2.50.0" failsafe = "3.3.2" findbugs-jsr305 = "3.0.2" flink120 = { strictly = "1.20.1"} From 72c226afe97f598d3277ece6351ef794835a6b80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:07:20 -0700 Subject: [PATCH 14/73] Build: Bump gradle/actions from 5.0.2 to 6.2.0 (#16899) Bumps [gradle/actions](https://github.com/gradle/actions) from 5.0.2 to 6.2.0. - [Release notes](https://github.com/gradle/actions/releases) - [Commits](https://github.com/gradle/actions/compare/0723195856401067f7a2779048b490ace7a47d7c...3f131e8634966bd73d06cc69884922b02e6faf92) --- updated-dependencies: - dependency-name: gradle/actions dependency-version: 6.2.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/api-binary-compatibility.yml | 2 +- .github/workflows/cve-scan.yml | 2 +- .github/workflows/delta-conversion-ci.yml | 4 ++-- .github/workflows/flink-ci.yml | 2 +- .github/workflows/hive-ci.yml | 2 +- .github/workflows/java-ci.yml | 8 ++++---- .github/workflows/jmh-benchmarks.yml | 2 +- .github/workflows/kafka-connect-ci.yml | 2 +- .github/workflows/publish-iceberg-rest-fixture-docker.yml | 2 +- .github/workflows/publish-snapshot.yml | 2 +- .github/workflows/recurring-jmh-benchmarks.yml | 2 +- .github/workflows/spark-ci.yml | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/api-binary-compatibility.yml b/.github/workflows/api-binary-compatibility.yml index 4a4954bad3de..50758c1c5d4c 100644 --- a/.github/workflows/api-binary-compatibility.yml +++ b/.github/workflows/api-binary-compatibility.yml @@ -59,7 +59,7 @@ jobs: with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/cve-scan.yml b/.github/workflows/cve-scan.yml index 78bfb9937d61..22f3401b64de 100644 --- a/.github/workflows/cve-scan.yml +++ b/.github/workflows/cve-scan.yml @@ -174,7 +174,7 @@ jobs: with: distribution: zulu java-version: 21 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index a35b21dcb4a6..7bdd48307fd2 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -89,7 +89,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true @@ -118,7 +118,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index 9fec6df124d6..a06c30c81e84 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -93,7 +93,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index b4fd3cc8027c..2876502b8fed 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -90,7 +90,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index bf4120d2022c..bf0ede1107c1 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -85,7 +85,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true @@ -112,7 +112,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Writes cache on main; read-only otherwise. cache-read-only: ${{ !(github.ref == 'refs/heads/main' && matrix.jvm == 17) }} @@ -132,7 +132,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true @@ -148,7 +148,7 @@ jobs: with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/jmh-benchmarks.yml b/.github/workflows/jmh-benchmarks.yml index f6e561a7eae8..5d2feae989f5 100644 --- a/.github/workflows/jmh-benchmarks.yml +++ b/.github/workflows/jmh-benchmarks.yml @@ -103,7 +103,7 @@ jobs: with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Disabled: dispatched against arbitrary repo/ref inputs; never restore or write to avoid cache poisoning. cache-disabled: true diff --git a/.github/workflows/kafka-connect-ci.yml b/.github/workflows/kafka-connect-ci.yml index 562fd970a111..115f9d0c898e 100644 --- a/.github/workflows/kafka-connect-ci.yml +++ b/.github/workflows/kafka-connect-ci.yml @@ -90,7 +90,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/publish-iceberg-rest-fixture-docker.yml b/.github/workflows/publish-iceberg-rest-fixture-docker.yml index 3275aab4c9d1..71e6c68b07e5 100644 --- a/.github/workflows/publish-iceberg-rest-fixture-docker.yml +++ b/.github/workflows/publish-iceberg-rest-fixture-docker.yml @@ -48,7 +48,7 @@ jobs: with: distribution: zulu java-version: 21 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index cade0d93e0fa..dd9a0f107983 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -43,7 +43,7 @@ jobs: with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/recurring-jmh-benchmarks.yml b/.github/workflows/recurring-jmh-benchmarks.yml index 50b2231df841..ef05e2b671d1 100644 --- a/.github/workflows/recurring-jmh-benchmarks.yml +++ b/.github/workflows/recurring-jmh-benchmarks.yml @@ -58,7 +58,7 @@ jobs: with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index 06230ddc78d6..ce5cae2969f9 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -104,7 +104,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true From afedbb1d1786e48606b54c62c100a922222ac5d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:08:09 -0700 Subject: [PATCH 15/73] Build: Bump nessie from 0.107.9 to 0.108.0 (#16897) Bumps `nessie` from 0.107.9 to 0.108.0. Updates `org.projectnessie.nessie:nessie-client` from 0.107.9 to 0.108.0 - [Release notes](https://github.com/projectnessie/nessie/releases) - [Changelog](https://github.com/projectnessie/nessie/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectnessie/nessie/compare/nessie-0.107.9...nessie-0.108.0) Updates `org.projectnessie.nessie:nessie-jaxrs-testextension` from 0.107.9 to 0.108.0 - [Release notes](https://github.com/projectnessie/nessie/releases) - [Changelog](https://github.com/projectnessie/nessie/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectnessie/nessie/compare/nessie-0.107.9...nessie-0.108.0) Updates `org.projectnessie.nessie:nessie-versioned-storage-inmemory-tests` from 0.107.9 to 0.108.0 - [Release notes](https://github.com/projectnessie/nessie/releases) - [Changelog](https://github.com/projectnessie/nessie/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectnessie/nessie/compare/nessie-0.107.9...nessie-0.108.0) Updates `org.projectnessie.nessie:nessie-versioned-storage-testextension` from 0.107.9 to 0.108.0 - [Release notes](https://github.com/projectnessie/nessie/releases) - [Changelog](https://github.com/projectnessie/nessie/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectnessie/nessie/compare/nessie-0.107.9...nessie-0.108.0) --- updated-dependencies: - dependency-name: org.projectnessie.nessie:nessie-client dependency-version: 0.108.0 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.projectnessie.nessie:nessie-jaxrs-testextension dependency-version: 0.108.0 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.projectnessie.nessie:nessie-versioned-storage-inmemory-tests dependency-version: 0.108.0 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.projectnessie.nessie:nessie-versioned-storage-testextension dependency-version: 0.108.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e995c8e1bb91..662dc6196935 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -77,7 +77,7 @@ lz4Java = "1.11.0" microprofile-openapi-api = "3.1.2" mockito = "4.11.0" mockserver = "5.15.0" -nessie = "0.107.9" +nessie = "0.108.0" netty-buffer = "4.2.15.Final" object-client-bundle = "3.3.2" orc = "1.9.8" From d5c86d6164404ef758f3fc174f8d3afe5bef1697 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:08:34 -0700 Subject: [PATCH 16/73] Build: Bump pymarkdownlnt from 0.9.37 to 0.9.38 (#16895) Bumps [pymarkdownlnt](https://github.com/jackdewinter/pymarkdown) from 0.9.37 to 0.9.38. - [Release notes](https://github.com/jackdewinter/pymarkdown/releases) - [Changelog](https://github.com/jackdewinter/pymarkdown/blob/main/changelog.md) - [Commits](https://github.com/jackdewinter/pymarkdown/compare/v0.9.37...v0.9.38) --- updated-dependencies: - dependency-name: pymarkdownlnt dependency-version: 0.9.38 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- site/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/requirements.txt b/site/requirements.txt index 8b38df3fc236..b33232b8f104 100644 --- a/site/requirements.txt +++ b/site/requirements.txt @@ -23,4 +23,4 @@ mkdocs-monorepo-plugin @ git+https://github.com/bitsondatadev/mkdocs-monorepo-pl mkdocs-redirects==1.2.3 mkdocs-rss-plugin==1.19.0 mkdocs-exclude-search==0.6.6 -pymarkdownlnt==0.9.37 +pymarkdownlnt==0.9.38 From b0977bbfcbbdfec17711acdfe0c14921d50d2f41 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Mon, 22 Jun 2026 02:26:23 +0900 Subject: [PATCH 17/73] Build: Bump datamodel-code-generator from 0.60.0 to 0.63.0 (#16907) * Flink, GCP, Kafka, Spark: Update runtime-deps.txt * Build: Bump datamodel-code-generator from 0.60.0 to 0.63.0 Bumps [datamodel-code-generator](https://github.com/koxudaxi/datamodel-code-generator) from 0.60.0 to 0.63.0. - [Release notes](https://github.com/koxudaxi/datamodel-code-generator/releases) - [Changelog](https://github.com/koxudaxi/datamodel-code-generator/blob/main/CHANGELOG.md) - [Commits](https://github.com/koxudaxi/datamodel-code-generator/compare/0.60.0...0.63.0) --- updated-dependencies: - dependency-name: datamodel-code-generator dependency-version: 0.63.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flink/v1.20/flink-runtime/runtime-deps.txt | 4 +- flink/v2.0/flink-runtime/runtime-deps.txt | 4 +- flink/v2.1/flink-runtime/runtime-deps.txt | 4 +- gcp-bundle/runtime-deps.txt | 56 +++++++++---------- .../kafka-connect-runtime/runtime-deps.txt | 2 +- open-api/requirements.txt | 2 +- open-api/rest-catalog-open-api.py | 7 ++- spark/v3.5/spark-runtime/runtime-deps.txt | 4 +- spark/v4.0/spark-runtime/runtime-deps.txt | 4 +- spark/v4.1/spark-runtime/runtime-deps.txt | 4 +- 10 files changed, 47 insertions(+), 44 deletions(-) diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt index 8e493d31c510..a295230cef19 100644 --- a/flink/v1.20/flink-runtime/runtime-deps.txt +++ b/flink/v1.20/flink-runtime/runtime-deps.txt @@ -25,7 +25,7 @@ org.apache.parquet:parquet-variant:1.17 org.checkerframework:checker-qual:3.19 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/flink/v2.0/flink-runtime/runtime-deps.txt b/flink/v2.0/flink-runtime/runtime-deps.txt index 8e493d31c510..a295230cef19 100644 --- a/flink/v2.0/flink-runtime/runtime-deps.txt +++ b/flink/v2.0/flink-runtime/runtime-deps.txt @@ -25,7 +25,7 @@ org.apache.parquet:parquet-variant:1.17 org.checkerframework:checker-qual:3.19 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt index 8e493d31c510..a295230cef19 100644 --- a/flink/v2.1/flink-runtime/runtime-deps.txt +++ b/flink/v2.1/flink-runtime/runtime-deps.txt @@ -25,7 +25,7 @@ org.apache.parquet:parquet-variant:1.17 org.checkerframework:checker-qual:3.19 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/gcp-bundle/runtime-deps.txt b/gcp-bundle/runtime-deps.txt index 0ff407294f5d..5577e8669e4f 100644 --- a/gcp-bundle/runtime-deps.txt +++ b/gcp-bundle/runtime-deps.txt @@ -4,14 +4,14 @@ com.fasterxml.jackson.core:jackson-databind:2.18 com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.18 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18 com.fasterxml.woodstox:woodstox-core:7.0 -com.github.ben-manes.caffeine:caffeine:3.1 +com.github.ben-manes.caffeine:caffeine:3.2 com.google.android:annotations:4.1 com.google.api-client:google-api-client:2.7 -com.google.api.grpc:gapic-google-cloud-storage-v2:2.68 +com.google.api.grpc:gapic-google-cloud-storage-v2:2.69 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.28 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.200 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.200 -com.google.api.grpc:grpc-google-cloud-storage-v2:2.68 +com.google.api.grpc:grpc-google-cloud-storage-v2:2.69 com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.28 com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.28 com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.200 @@ -19,17 +19,17 @@ com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.200 com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.28 com.google.api.grpc:proto-google-cloud-kms-v1:0.186 com.google.api.grpc:proto-google-cloud-monitoring-v3:3.93 -com.google.api.grpc:proto-google-cloud-storage-v2:2.68 -com.google.api.grpc:proto-google-common-protos:2.71 -com.google.api.grpc:proto-google-iam-v1:1.66 -com.google.api:api-common:2.63 -com.google.api:gax-grpc:2.80 -com.google.api:gax-httpjson:2.80 -com.google.api:gax:2.80 +com.google.api.grpc:proto-google-cloud-storage-v2:2.69 +com.google.api.grpc:proto-google-common-protos:2.72 +com.google.api.grpc:proto-google-iam-v1:1.67 +com.google.api:api-common:2.64 +com.google.api:gax-grpc:2.81 +com.google.api:gax-httpjson:2.81 +com.google.api:gax:2.81 com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0 com.google.apis:google-api-services-storage:v1-rev20260204-2.0 -com.google.auth:google-auth-library-credentials:1.47 -com.google.auth:google-auth-library-oauth2-http:1.47 +com.google.auth:google-auth-library-credentials:1.48 +com.google.auth:google-auth-library-oauth2-http:1.48 com.google.auto.value:auto-value-annotations:1.11 com.google.cloud.gcs.analytics:client:1.3 com.google.cloud.gcs.analytics:common:1.3 @@ -39,17 +39,17 @@ com.google.cloud.opentelemetry:exporter-metrics:0.33 com.google.cloud.opentelemetry:shared-resourcemapping:0.33 com.google.cloud:google-cloud-bigquery:2.66 com.google.cloud:google-cloud-bigquerystorage:3.28 -com.google.cloud:google-cloud-core-grpc:2.70 -com.google.cloud:google-cloud-core-http:2.70 -com.google.cloud:google-cloud-core:2.70 +com.google.cloud:google-cloud-core-grpc:2.71 +com.google.cloud:google-cloud-core-http:2.71 +com.google.cloud:google-cloud-core:2.71 com.google.cloud:google-cloud-kms:2.95 com.google.cloud:google-cloud-monitoring:3.93 -com.google.cloud:google-cloud-storage:2.68 +com.google.cloud:google-cloud-storage:2.69 com.google.code.gson:gson:2.13 -com.google.errorprone:error_prone_annotations:2.48 +com.google.errorprone:error_prone_annotations:2.49 com.google.flatbuffers:flatbuffers-java:24.3 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:2.1 com.google.http-client:google-http-client-appengine:2.1 @@ -85,16 +85,16 @@ io.opencensus:opencensus-api:0.31 io.opencensus:opencensus-contrib-http-util:0.31 io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37 io.opentelemetry.semconv:opentelemetry-semconv:1.29 -io.opentelemetry:opentelemetry-api:1.57 -io.opentelemetry:opentelemetry-common:1.57 -io.opentelemetry:opentelemetry-context:1.57 -io.opentelemetry:opentelemetry-exporter-logging:1.52 -io.opentelemetry:opentelemetry-sdk-common:1.57 -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.57 -io.opentelemetry:opentelemetry-sdk-logs:1.57 -io.opentelemetry:opentelemetry-sdk-metrics:1.57 -io.opentelemetry:opentelemetry-sdk-trace:1.57 -io.opentelemetry:opentelemetry-sdk:1.57 +io.opentelemetry:opentelemetry-api:1.63 +io.opentelemetry:opentelemetry-common:1.63 +io.opentelemetry:opentelemetry-context:1.63 +io.opentelemetry:opentelemetry-exporter-logging:1.63 +io.opentelemetry:opentelemetry-sdk-common:1.63 +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.62 +io.opentelemetry:opentelemetry-sdk-logs:1.63 +io.opentelemetry:opentelemetry-sdk-metrics:1.63 +io.opentelemetry:opentelemetry-sdk-trace:1.63 +io.opentelemetry:opentelemetry-sdk:1.63 io.perfmark:perfmark-api:0.27 javax.annotation:javax.annotation-api:1.3 org.apache.arrow:arrow-format:17.0 diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt index 48d11bdf6f0f..2aef692c4f6f 100644 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -90,7 +90,7 @@ io.grpc:grpc-core:1.81 io.grpc:grpc-googleapis:1.81 io.grpc:grpc-grpclb:1.81 io.grpc:grpc-inprocess:1.81 -io.grpc:grpc-netty-shaded:1.81 +io.grpc:grpc-netty-shaded:1.82 io.grpc:grpc-opentelemetry:1.81 io.grpc:grpc-protobuf-lite:1.81 io.grpc:grpc-protobuf:1.81 diff --git a/open-api/requirements.txt b/open-api/requirements.txt index 6cab5500c6ef..35dc0d4b4d57 100644 --- a/open-api/requirements.txt +++ b/open-api/requirements.txt @@ -16,5 +16,5 @@ # under the License. openapi-spec-validator==0.9.0 -datamodel-code-generator==0.60.0 +datamodel-code-generator==0.63.0 yamllint==1.38.0 diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 4e05d10e9cdd..7b6c4f73fc9e 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Literal +from typing import Dict, Literal from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -269,10 +269,13 @@ class Summary(BaseModel): model_config = ConfigDict( extra='allow', ) - __pydantic_extra__: dict[str, str] operation: Literal['append', 'replace', 'overwrite', 'delete'] +Summary.__annotations__['__pydantic_extra__'] = Dict[str, str] +Summary.model_rebuild(force=True) + + class Snapshot(BaseModel): snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt index 1fcb7e284d8c..18c17ad30177 100644 --- a/spark/v3.5/spark-runtime/runtime-deps.txt +++ b/spark/v3.5/spark-runtime/runtime-deps.txt @@ -34,7 +34,7 @@ org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt index 1fcb7e284d8c..18c17ad30177 100644 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -34,7 +34,7 @@ org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt index 1fcb7e284d8c..18c17ad30177 100644 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -34,7 +34,7 @@ org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 From 0b30919372df34afb632f037df88c05cdba0b134 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Mon, 22 Jun 2026 02:09:59 +0800 Subject: [PATCH 18/73] Docs: Clarify test method naming guidance (#16866) --- AGENTS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index e6c771d4b482..ada467efa4ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -139,7 +139,8 @@ The `api/` module has the strongest stability guarantees — breaking changes ar - Test classes and methods should be package private unless required by inheritance. - Compute expected values, don't hardcode. Tests belong in the module that owns the code. - Write the most direct test for the bug. Parameterized tests for type variations. -- JUnit 5 + AssertJ: `@Test` (no `test` prefix), `assertThat`, `assertThatThrownBy`. +- JUnit 5 + AssertJ: `@Test`, `assertThat`, `assertThatThrownBy`. +- Avoid using `test` prefixes for newly added tests. - `waitUntilAfter` for time-dependent tests. Separate tests over combined. ### REST / OpenAPI Spec From fa072cc6b7ca3ead3e4ef7e09bd01d00c2988c88 Mon Sep 17 00:00:00 2001 From: Eunbin Son <58901024+thswlsqls@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:48:20 +0900 Subject: [PATCH 19/73] Spark 4.1: Throw on unsupported complex types in SparkValueConverter (#16924) SparkValueConverter.convertToSpark returned a new UnsupportedOperationException for STRUCT, LIST, and MAP instead of throwing it, so the exception object would be passed downstream as a value. Throw it to match the method's own default branch. --- .../iceberg/spark/SparkValueConverter.java | 2 +- .../spark/TestSparkValueConverter.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java index 28b717ac090e..af405bb04d9d 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java @@ -129,7 +129,7 @@ public static Object convertToSpark(Type type, Object object) { case STRUCT: case LIST: case MAP: - return new UnsupportedOperationException("Complex types currently not supported"); + throw new UnsupportedOperationException("Complex types currently not supported"); case DATE: return DateTimeUtils.daysToLocalDate((int) object); case TIMESTAMP: diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java index c7a2e6c18fca..b96493511cf8 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java @@ -19,6 +19,7 @@ package org.apache.iceberg.spark; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; @@ -83,6 +84,26 @@ public void testSparkNullPrimitiveConvert() { assertCorrectNullConversion(schema); } + @Test + public void testConvertToSparkComplexTypesThrow() { + Types.StructType struct = + Types.StructType.of(Types.NestedField.required(1, "lat", Types.FloatType.get())); + Types.ListType list = Types.ListType.ofOptional(1, Types.StringType.get()); + Types.MapType map = + Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.StringType.get()); + + for (Types.NestedField field : + new Types.NestedField[] { + Types.NestedField.required(0, "s", struct), + Types.NestedField.required(0, "l", list), + Types.NestedField.required(0, "m", map) + }) { + assertThatThrownBy(() -> SparkValueConverter.convertToSpark(field.type(), "unused")) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Complex types currently not supported"); + } + } + private void assertCorrectNullConversion(Schema schema) { Row sparkRow = RowFactory.create(1, null); Record record = GenericRecord.create(schema); From fb6bb97e393b8af692973e3a1268cccf6ae2c5c2 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Mon, 22 Jun 2026 22:23:27 +0900 Subject: [PATCH 20/73] Spark 3.5, 4.0: Throw on unsupported complex types in SparkValueConverter (#16925) --- .../iceberg/spark/SparkValueConverter.java | 2 +- .../iceberg/spark/TestSparkValueConverter.java | 18 ++++++++++++++++++ .../iceberg/spark/SparkValueConverter.java | 2 +- .../iceberg/spark/TestSparkValueConverter.java | 18 ++++++++++++++++++ .../iceberg/spark/TestSparkValueConverter.java | 11 ++++------- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java index 28b717ac090e..af405bb04d9d 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java @@ -129,7 +129,7 @@ public static Object convertToSpark(Type type, Object object) { case STRUCT: case LIST: case MAP: - return new UnsupportedOperationException("Complex types currently not supported"); + throw new UnsupportedOperationException("Complex types currently not supported"); case DATE: return DateTimeUtils.daysToLocalDate((int) object); case TIMESTAMP: diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java index c7a2e6c18fca..3ffc303759db 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java @@ -19,10 +19,13 @@ package org.apache.iceberg.spark; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.List; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -83,6 +86,21 @@ public void testSparkNullPrimitiveConvert() { assertCorrectNullConversion(schema); } + @Test + public void testConvertToSparkComplexTypesThrow() { + Types.StructType struct = + Types.StructType.of(Types.NestedField.required(1, "lat", Types.FloatType.get())); + Types.ListType list = Types.ListType.ofOptional(1, Types.StringType.get()); + Types.MapType map = + Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.StringType.get()); + + for (Type type : List.of(struct, list, map)) { + assertThatThrownBy(() -> SparkValueConverter.convertToSpark(type, "unused")) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Complex types currently not supported"); + } + } + private void assertCorrectNullConversion(Schema schema) { Row sparkRow = RowFactory.create(1, null); Record record = GenericRecord.create(schema); diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java index 28b717ac090e..af405bb04d9d 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java @@ -129,7 +129,7 @@ public static Object convertToSpark(Type type, Object object) { case STRUCT: case LIST: case MAP: - return new UnsupportedOperationException("Complex types currently not supported"); + throw new UnsupportedOperationException("Complex types currently not supported"); case DATE: return DateTimeUtils.daysToLocalDate((int) object); case TIMESTAMP: diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java index c7a2e6c18fca..3ffc303759db 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java @@ -19,10 +19,13 @@ package org.apache.iceberg.spark; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.List; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -83,6 +86,21 @@ public void testSparkNullPrimitiveConvert() { assertCorrectNullConversion(schema); } + @Test + public void testConvertToSparkComplexTypesThrow() { + Types.StructType struct = + Types.StructType.of(Types.NestedField.required(1, "lat", Types.FloatType.get())); + Types.ListType list = Types.ListType.ofOptional(1, Types.StringType.get()); + Types.MapType map = + Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.StringType.get()); + + for (Type type : List.of(struct, list, map)) { + assertThatThrownBy(() -> SparkValueConverter.convertToSpark(type, "unused")) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Complex types currently not supported"); + } + } + private void assertCorrectNullConversion(Schema schema) { Row sparkRow = RowFactory.create(1, null); Record record = GenericRecord.create(schema); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java index b96493511cf8..3ffc303759db 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java @@ -21,9 +21,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.List; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -92,13 +94,8 @@ public void testConvertToSparkComplexTypesThrow() { Types.MapType map = Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.StringType.get()); - for (Types.NestedField field : - new Types.NestedField[] { - Types.NestedField.required(0, "s", struct), - Types.NestedField.required(0, "l", list), - Types.NestedField.required(0, "m", map) - }) { - assertThatThrownBy(() -> SparkValueConverter.convertToSpark(field.type(), "unused")) + for (Type type : List.of(struct, list, map)) { + assertThatThrownBy(() -> SparkValueConverter.convertToSpark(type, "unused")) .isInstanceOf(UnsupportedOperationException.class) .hasMessageContaining("Complex types currently not supported"); } From 902ed1b80d3cc24733fc72544578c58ca0ffe9c6 Mon Sep 17 00:00:00 2001 From: Andrei Tserakhau Date: Mon, 22 Jun 2026 17:23:51 +0200 Subject: [PATCH 21/73] Revert "Spark: Spark tests cache rewrite input (#16740)" (#16912) This reverts commit e5151f35a09e05fcbc846fb6f1f76627b8313740. --- .../actions/TestRewriteDataFilesAction.java | 127 +----------------- .../actions/TestRewriteDataFilesAction.java | 127 +----------------- .../actions/TestRewriteDataFilesAction.java | 127 +----------------- 3 files changed, 12 insertions(+), 369 deletions(-) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index 130078a344a5..d74d8a29f994 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -41,23 +41,18 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Path; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.AppendFiles; import org.apache.iceberg.BaseTable; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; @@ -107,12 +102,10 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.spark.FileRewriteCoordinator; @@ -136,7 +129,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.DataTypes; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; @@ -150,18 +142,6 @@ public class TestRewriteDataFilesAction extends TestBase { @TempDir private File tableDir; private static final int SCALE = 400000; - // Row group size used by createTable(); part of the unpartitioned cache key so a future - // override of this property can't silently hand back cached files of a different shape. - private static final int INPUT_PARQUET_ROW_GROUP_SIZE_BYTES = 20 * 1024; - - // Cache of pre-written input data files keyed by table shape (schema/spec/props are - // fixed per key), so identical large inputs are materialized via Spark only once per JVM - // fork and reused by every test that asks for the same shape. The Spark write of SCALE - // rows dominates these tests; the rewrite under test still runs per test on a fresh table. - @TempDir private static Path inputCacheDir; - private static final Map> INPUT_FILE_CACHE = Maps.newConcurrentMap(); - private static final Map INPUT_CACHE_LOCKS = Maps.newConcurrentMap(); - private static final AtomicInteger INPUT_CACHE_SEQ = new AtomicInteger(); private static final HadoopTables TABLES = new HadoopTables(new Configuration()); private static final Schema SCHEMA = @@ -189,16 +169,6 @@ public static void setupSpark() { spark.conf().set(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false"); } - @AfterAll - public static void clearInputFileCache() { - // inputCacheDir is a static @TempDir that JUnit recreates if the class runs twice in one JVM - // (IDE re-run, forkCount=0). Clear the cache so a second run can't return DataFiles pointing - // into the deleted first-run directory. - INPUT_FILE_CACHE.clear(); - INPUT_CACHE_LOCKS.clear(); - INPUT_CACHE_SEQ.set(0); - } - @BeforeEach public void setupTableLocation() { this.tableLocation = tableDir.toURI().toString(); @@ -2210,9 +2180,6 @@ protected void shouldHaveSnapshots(Table table, int expectedSnapshots) { } protected void shouldHaveNoOrphans(Table table) { - // Cached input files live under the static inputCacheDir, outside table.location(), so - // deleteOrphanFiles (which only scans the table prefix) never sees them by design. Orphan - // coverage therefore does not extend to the shared cached inputs. assertThat( actions() .deleteOrphanFiles(table) @@ -2325,9 +2292,7 @@ protected Table createTable() { Table table = TABLES.create(SCHEMA, spec, options, tableLocation); table .updateProperties() - .set( - TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, - Integer.toString(INPUT_PARQUET_ROW_GROUP_SIZE_BYTES)) + .set(TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, Integer.toString(20 * 1024)) .commit(); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); return table; @@ -2340,21 +2305,8 @@ protected Table createTable() { * @return the created table */ protected Table createTable(int files) { - String key = - String.format( - "unpartitioned|fv=%d|rowGroup=%d|files=%d|rows=%d", - formatVersion, INPUT_PARQUET_ROW_GROUP_SIZE_BYTES, files, SCALE); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = createTable(); - writeRecords(files, SCALE); - golden.refresh(); - return golden; - }); Table table = createTable(); - appendInputFiles(table, inputFiles); + writeRecords(files, SCALE); table.refresh(); return table; } @@ -2362,85 +2314,14 @@ protected Table createTable(int files) { protected Table createTablePartitioned( int partitions, int files, int numRecords, Map options) { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").truncate("c2", 2).build(); - String key = - String.format( - "partitioned|fv=%d|spec=%s|opts=%s|files=%d|rows=%d|partitions=%d", - formatVersion, spec, new TreeMap<>(options), files, numRecords, partitions); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = TABLES.create(SCHEMA, spec, options, tableLocation); - assertThat(golden.currentSnapshot()).as("Table must be empty").isNull(); - writeRecords(files, numRecords, partitions); - golden.refresh(); - return golden; - }); Table table = TABLES.create(SCHEMA, spec, options, tableLocation); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); - appendInputFiles(table, inputFiles); + + writeRecords(files, numRecords, partitions); table.refresh(); return table; } - /** - * Returns the input data files for a given table shape, materializing them with Spark exactly - * once per JVM fork and reusing them afterwards. On a cache miss the {@code goldenBuilder} writes - * the data into a stable cache location (kept alive for the whole class via a static {@link - * TempDir}); on a hit the cached {@link DataFile}s are returned and re-appended to a fresh table - * by {@link #appendInputFiles}. The data is deterministic (fixed RNG seed) so reuse is - * byte-identical to regenerating it. - */ - private List cachedInputFiles(String key, Supplier goldenBuilder) { - List cached = INPUT_FILE_CACHE.get(key); - if (cached != null) { - return cached; - } - // Serialize builds per key: concurrent callers requesting the same table shape block on the - // first build and then reuse its result, instead of materializing identical input twice. The - // heavy Spark write happens outside any map lock, so distinct shapes can still build in - // parallel. - Object lock = INPUT_CACHE_LOCKS.computeIfAbsent(key, ignored -> new Object()); - synchronized (lock) { - List existing = INPUT_FILE_CACHE.get(key); - if (existing != null) { - return existing; - } - String savedLocation = this.tableLocation; - try { - this.tableLocation = - inputCacheDir.resolve("input-" + INPUT_CACHE_SEQ.incrementAndGet()).toUri().toString(); - Table golden = goldenBuilder.get(); - // includeColumnStats() is required: a plain scan drops lower/upper bounds and - // value counts, and re-appending stat-less files breaks tests that read bounds. - // planFiles() returns a CloseableIterable holding manifest readers open, so close it via - // try-with-resources; otherwise every cache miss leaks file descriptors and can leave - // manifest files locked on Windows. - List built; - try (CloseableIterable tasks = - golden.newScan().includeColumnStats().planFiles()) { - built = - Streams.stream(tasks) - .map(FileScanTask::file) - .map(DataFile::copy) - .collect(ImmutableList.toImmutableList()); - } catch (IOException e) { - throw new UncheckedIOException("Failed to plan cached input files", e); - } - INPUT_FILE_CACHE.put(key, built); - return built; - } finally { - this.tableLocation = savedLocation; - } - } - } - - private static void appendInputFiles(Table table, List inputFiles) { - AppendFiles append = table.newAppend(); - inputFiles.forEach(append::appendFile); - append.commit(); - } - protected Table createTablePartitioned(int partitions, int files) { return createTablePartitioned( partitions, diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index 06d45436955e..38ddefd26a45 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -41,24 +41,19 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Path; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.AppendFiles; import org.apache.iceberg.BaseTable; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; @@ -108,12 +103,10 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.spark.FileRewriteCoordinator; @@ -137,7 +130,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.DataTypes; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; @@ -151,18 +143,6 @@ public class TestRewriteDataFilesAction extends TestBase { @TempDir private File tableDir; private static final int SCALE = 400000; - // Row group size used by createTable(); part of the unpartitioned cache key so a future - // override of this property can't silently hand back cached files of a different shape. - private static final int INPUT_PARQUET_ROW_GROUP_SIZE_BYTES = 20 * 1024; - - // Cache of pre-written input data files keyed by table shape (schema/spec/props are - // fixed per key), so identical large inputs are materialized via Spark only once per JVM - // fork and reused by every test that asks for the same shape. The Spark write of SCALE - // rows dominates these tests; the rewrite under test still runs per test on a fresh table. - @TempDir private static Path inputCacheDir; - private static final Map> INPUT_FILE_CACHE = Maps.newConcurrentMap(); - private static final Map INPUT_CACHE_LOCKS = Maps.newConcurrentMap(); - private static final AtomicInteger INPUT_CACHE_SEQ = new AtomicInteger(); private static final HadoopTables TABLES = new HadoopTables(new Configuration()); private static final Schema SCHEMA = @@ -190,16 +170,6 @@ public static void setupSpark() { spark.conf().set(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false"); } - @AfterAll - public static void clearInputFileCache() { - // inputCacheDir is a static @TempDir that JUnit recreates if the class runs twice in one JVM - // (IDE re-run, forkCount=0). Clear the cache so a second run can't return DataFiles pointing - // into the deleted first-run directory. - INPUT_FILE_CACHE.clear(); - INPUT_CACHE_LOCKS.clear(); - INPUT_CACHE_SEQ.set(0); - } - @BeforeEach public void setupTableLocation() { this.tableLocation = tableDir.toURI().toString(); @@ -2260,9 +2230,6 @@ protected void shouldHaveSnapshots(Table table, int expectedSnapshots) { } protected void shouldHaveNoOrphans(Table table) { - // Cached input files live under the static inputCacheDir, outside table.location(), so - // deleteOrphanFiles (which only scans the table prefix) never sees them by design. Orphan - // coverage therefore does not extend to the shared cached inputs. assertThat( actions() .deleteOrphanFiles(table) @@ -2375,9 +2342,7 @@ protected Table createTable() { Table table = TABLES.create(SCHEMA, spec, options, tableLocation); table .updateProperties() - .set( - TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, - Integer.toString(INPUT_PARQUET_ROW_GROUP_SIZE_BYTES)) + .set(TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, Integer.toString(20 * 1024)) .commit(); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); return table; @@ -2390,21 +2355,8 @@ protected Table createTable() { * @return the created table */ protected Table createTable(int files) { - String key = - String.format( - "unpartitioned|fv=%d|rowGroup=%d|files=%d|rows=%d", - formatVersion, INPUT_PARQUET_ROW_GROUP_SIZE_BYTES, files, SCALE); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = createTable(); - writeRecords(files, SCALE); - golden.refresh(); - return golden; - }); Table table = createTable(); - appendInputFiles(table, inputFiles); + writeRecords(files, SCALE); table.refresh(); return table; } @@ -2412,85 +2364,14 @@ protected Table createTable(int files) { protected Table createTablePartitioned( int partitions, int files, int numRecords, Map options) { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").truncate("c2", 2).build(); - String key = - String.format( - "partitioned|fv=%d|spec=%s|opts=%s|files=%d|rows=%d|partitions=%d", - formatVersion, spec, new TreeMap<>(options), files, numRecords, partitions); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = TABLES.create(SCHEMA, spec, options, tableLocation); - assertThat(golden.currentSnapshot()).as("Table must be empty").isNull(); - writeRecords(files, numRecords, partitions); - golden.refresh(); - return golden; - }); Table table = TABLES.create(SCHEMA, spec, options, tableLocation); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); - appendInputFiles(table, inputFiles); + + writeRecords(files, numRecords, partitions); table.refresh(); return table; } - /** - * Returns the input data files for a given table shape, materializing them with Spark exactly - * once per JVM fork and reusing them afterwards. On a cache miss the {@code goldenBuilder} writes - * the data into a stable cache location (kept alive for the whole class via a static {@link - * TempDir}); on a hit the cached {@link DataFile}s are returned and re-appended to a fresh table - * by {@link #appendInputFiles}. The data is deterministic (fixed RNG seed) so reuse is - * byte-identical to regenerating it. - */ - private List cachedInputFiles(String key, Supplier
goldenBuilder) { - List cached = INPUT_FILE_CACHE.get(key); - if (cached != null) { - return cached; - } - // Serialize builds per key: concurrent callers requesting the same table shape block on the - // first build and then reuse its result, instead of materializing identical input twice. The - // heavy Spark write happens outside any map lock, so distinct shapes can still build in - // parallel. - Object lock = INPUT_CACHE_LOCKS.computeIfAbsent(key, ignored -> new Object()); - synchronized (lock) { - List existing = INPUT_FILE_CACHE.get(key); - if (existing != null) { - return existing; - } - String savedLocation = this.tableLocation; - try { - this.tableLocation = - inputCacheDir.resolve("input-" + INPUT_CACHE_SEQ.incrementAndGet()).toUri().toString(); - Table golden = goldenBuilder.get(); - // includeColumnStats() is required: a plain scan drops lower/upper bounds and - // value counts, and re-appending stat-less files breaks tests that read bounds. - // planFiles() returns a CloseableIterable holding manifest readers open, so close it via - // try-with-resources; otherwise every cache miss leaks file descriptors and can leave - // manifest files locked on Windows. - List built; - try (CloseableIterable tasks = - golden.newScan().includeColumnStats().planFiles()) { - built = - Streams.stream(tasks) - .map(FileScanTask::file) - .map(DataFile::copy) - .collect(ImmutableList.toImmutableList()); - } catch (IOException e) { - throw new UncheckedIOException("Failed to plan cached input files", e); - } - INPUT_FILE_CACHE.put(key, built); - return built; - } finally { - this.tableLocation = savedLocation; - } - } - } - - private static void appendInputFiles(Table table, List inputFiles) { - AppendFiles append = table.newAppend(); - inputFiles.forEach(append::appendFile); - append.commit(); - } - protected Table createTablePartitioned(int partitions, int files) { return createTablePartitioned( partitions, diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index df59a0ceefae..110e43ede1f9 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -41,24 +41,19 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Path; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.AppendFiles; import org.apache.iceberg.BaseTable; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; @@ -108,12 +103,10 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.spark.FileRewriteCoordinator; @@ -137,7 +130,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.DataTypes; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; @@ -151,18 +143,6 @@ public class TestRewriteDataFilesAction extends TestBase { @TempDir private File tableDir; private static final int SCALE = 400000; - // Row group size used by createTable(); part of the unpartitioned cache key so a future - // override of this property can't silently hand back cached files of a different shape. - private static final int INPUT_PARQUET_ROW_GROUP_SIZE_BYTES = 20 * 1024; - - // Cache of pre-written input data files keyed by table shape (schema/spec/props are - // fixed per key), so identical large inputs are materialized via Spark only once per JVM - // fork and reused by every test that asks for the same shape. The Spark write of SCALE - // rows dominates these tests; the rewrite under test still runs per test on a fresh table. - @TempDir private static Path inputCacheDir; - private static final Map> INPUT_FILE_CACHE = Maps.newConcurrentMap(); - private static final Map INPUT_CACHE_LOCKS = Maps.newConcurrentMap(); - private static final AtomicInteger INPUT_CACHE_SEQ = new AtomicInteger(); private static final HadoopTables TABLES = new HadoopTables(new Configuration()); private static final Schema SCHEMA = @@ -190,16 +170,6 @@ public static void setupSpark() { spark.conf().set(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false"); } - @AfterAll - public static void clearInputFileCache() { - // inputCacheDir is a static @TempDir that JUnit recreates if the class runs twice in one JVM - // (IDE re-run, forkCount=0). Clear the cache so a second run can't return DataFiles pointing - // into the deleted first-run directory. - INPUT_FILE_CACHE.clear(); - INPUT_CACHE_LOCKS.clear(); - INPUT_CACHE_SEQ.set(0); - } - @BeforeEach public void setupTableLocation() { this.tableLocation = tableDir.toURI().toString(); @@ -2260,9 +2230,6 @@ protected void shouldHaveSnapshots(Table table, int expectedSnapshots) { } protected void shouldHaveNoOrphans(Table table) { - // Cached input files live under the static inputCacheDir, outside table.location(), so - // deleteOrphanFiles (which only scans the table prefix) never sees them by design. Orphan - // coverage therefore does not extend to the shared cached inputs. assertThat( actions() .deleteOrphanFiles(table) @@ -2375,9 +2342,7 @@ protected Table createTable() { Table table = TABLES.create(SCHEMA, spec, options, tableLocation); table .updateProperties() - .set( - TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, - Integer.toString(INPUT_PARQUET_ROW_GROUP_SIZE_BYTES)) + .set(TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, Integer.toString(20 * 1024)) .commit(); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); return table; @@ -2390,21 +2355,8 @@ protected Table createTable() { * @return the created table */ protected Table createTable(int files) { - String key = - String.format( - "unpartitioned|fv=%d|rowGroup=%d|files=%d|rows=%d", - formatVersion, INPUT_PARQUET_ROW_GROUP_SIZE_BYTES, files, SCALE); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = createTable(); - writeRecords(files, SCALE); - golden.refresh(); - return golden; - }); Table table = createTable(); - appendInputFiles(table, inputFiles); + writeRecords(files, SCALE); table.refresh(); return table; } @@ -2412,85 +2364,14 @@ protected Table createTable(int files) { protected Table createTablePartitioned( int partitions, int files, int numRecords, Map options) { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").truncate("c2", 2).build(); - String key = - String.format( - "partitioned|fv=%d|spec=%s|opts=%s|files=%d|rows=%d|partitions=%d", - formatVersion, spec, new TreeMap<>(options), files, numRecords, partitions); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = TABLES.create(SCHEMA, spec, options, tableLocation); - assertThat(golden.currentSnapshot()).as("Table must be empty").isNull(); - writeRecords(files, numRecords, partitions); - golden.refresh(); - return golden; - }); Table table = TABLES.create(SCHEMA, spec, options, tableLocation); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); - appendInputFiles(table, inputFiles); + + writeRecords(files, numRecords, partitions); table.refresh(); return table; } - /** - * Returns the input data files for a given table shape, materializing them with Spark exactly - * once per JVM fork and reusing them afterwards. On a cache miss the {@code goldenBuilder} writes - * the data into a stable cache location (kept alive for the whole class via a static {@link - * TempDir}); on a hit the cached {@link DataFile}s are returned and re-appended to a fresh table - * by {@link #appendInputFiles}. The data is deterministic (fixed RNG seed) so reuse is - * byte-identical to regenerating it. - */ - private List cachedInputFiles(String key, Supplier
goldenBuilder) { - List cached = INPUT_FILE_CACHE.get(key); - if (cached != null) { - return cached; - } - // Serialize builds per key: concurrent callers requesting the same table shape block on the - // first build and then reuse its result, instead of materializing identical input twice. The - // heavy Spark write happens outside any map lock, so distinct shapes can still build in - // parallel. - Object lock = INPUT_CACHE_LOCKS.computeIfAbsent(key, ignored -> new Object()); - synchronized (lock) { - List existing = INPUT_FILE_CACHE.get(key); - if (existing != null) { - return existing; - } - String savedLocation = this.tableLocation; - try { - this.tableLocation = - inputCacheDir.resolve("input-" + INPUT_CACHE_SEQ.incrementAndGet()).toUri().toString(); - Table golden = goldenBuilder.get(); - // includeColumnStats() is required: a plain scan drops lower/upper bounds and - // value counts, and re-appending stat-less files breaks tests that read bounds. - // planFiles() returns a CloseableIterable holding manifest readers open, so close it via - // try-with-resources; otherwise every cache miss leaks file descriptors and can leave - // manifest files locked on Windows. - List built; - try (CloseableIterable tasks = - golden.newScan().includeColumnStats().planFiles()) { - built = - Streams.stream(tasks) - .map(FileScanTask::file) - .map(DataFile::copy) - .collect(ImmutableList.toImmutableList()); - } catch (IOException e) { - throw new UncheckedIOException("Failed to plan cached input files", e); - } - INPUT_FILE_CACHE.put(key, built); - return built; - } finally { - this.tableLocation = savedLocation; - } - } - } - - private static void appendInputFiles(Table table, List inputFiles) { - AppendFiles append = table.newAppend(); - inputFiles.forEach(append::appendFile); - append.commit(); - } - protected Table createTablePartitioned(int partitions, int files) { return createTablePartitioned( partitions, From 02631cdaecd90ef840da12017c60981469a34b8a Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Tue, 23 Jun 2026 04:53:22 +0800 Subject: [PATCH 22/73] CI: Check ASF action allowlist on every PR (#16926) Remove path filters from the ASF allowlist workflow so it runs for all pull requests and pushes to main. This surfaces upstream approved-pattern drift as a visible check failure even when a pull request does not edit workflow files. Fixes #16914 Generated-by: Codex Co-authored-by: Codex --- .github/workflows/asf-allowlist-check.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml index 71311bca7235..ea5920511029 100644 --- a/.github/workflows/asf-allowlist-check.yml +++ b/.github/workflows/asf-allowlist-check.yml @@ -25,13 +25,9 @@ name: "ASF Allowlist Check" on: pull_request: - paths: - - ".github/**" push: branches: - main - paths: - - ".github/**" permissions: contents: read From ac874e80bd5f411b60578d51fcb9861e2f6d45ce Mon Sep 17 00:00:00 2001 From: Thomas Thornton Date: Mon, 22 Jun 2026 15:13:29 -0700 Subject: [PATCH 23/73] Kafka Connect: evolve table schema when record schema is updated but value is null (#16826) * kafka-connect: evolve table schema when record schema is updated but value is null Signed-off-by: Thomas Thornton * kafka connect: fix nested schema evolution when parent evolves, add map key evolution Signed-off-by: Thomas Thornton * Fix style Signed-off-by: Thomas Thornton * kafka connect: improve docs and testing for evolve schema when value is null Signed-off-by: Thomas Thornton * kafka connect: defer nested null value evolution when parent evolves, drop map key recursion Signed-off-by: Thomas Thornton --------- Signed-off-by: Thomas Thornton --- .../iceberg/connect/data/RecordConverter.java | 95 +++- .../connect/data/TestRecordConverter.java | 420 ++++++++++++++++++ 2 files changed, 514 insertions(+), 1 deletion(-) diff --git a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java index a152af2fd02a..41e70d67555a 100644 --- a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java +++ b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java @@ -81,9 +81,13 @@ import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.ConnectException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class RecordConverter { + private static final Logger LOG = LoggerFactory.getLogger(RecordConverter.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); private static final DateTimeFormatter OFFSET_TIMESTAMP_FORMAT = @@ -255,11 +259,19 @@ private GenericRecord convertToStruct( hasSchemaUpdates = true; } } + Object recordFieldValue = struct.get(recordField); + if (recordFieldValue == null && schemaUpdateConsumer != null && !hasSchemaUpdates) { + evolveSchemaFromConnectSchema( + recordField.schema(), + tableField.type(), + tableField.fieldId(), + schemaUpdateConsumer); + } if (!hasSchemaUpdates) { result.setField( tableField.name(), convertValue( - struct.get(recordField), + recordFieldValue, tableField.type(), tableField.fieldId(), schemaUpdateConsumer)); @@ -269,6 +281,87 @@ private GenericRecord convertToStruct( return result; } + /** + * Recursively traverses the Connect schema and emits all evolution events (addColumn, updateType, + * and makeOptional) at every nested level. + * + *

Unlike {@link #convertToStruct(Struct, StructType, int, SchemaUpdate.Consumer)} which skips + * a field's children once an update is detected (deferring nested discovery to re-conversion), + * this method always recurses through the full schema. + */ + private void evolveSchemaFromConnectSchema( + org.apache.kafka.connect.data.Schema recordSchema, + Type tableType, + int tableFieldId, + SchemaUpdate.Consumer schemaUpdateConsumer) { + if (recordSchema == null) { + return; + } + switch (recordSchema.type()) { + case STRUCT: + if (tableType.isStructType()) { + StructType structType = tableType.asStructType(); + for (Field field : recordSchema.fields()) { + NestedField nestedField = lookupStructField(field.name(), structType, tableFieldId); + if (nestedField == null) { + String parentFieldName = + tableFieldId < 0 ? null : tableSchema.findColumnName(tableFieldId); + Type type = SchemaUtils.toIcebergType(field.schema(), config); + schemaUpdateConsumer.addColumn(parentFieldName, field.name(), type); + } else { + PrimitiveType evolveDataType = + SchemaUtils.needsDataTypeUpdate(nestedField.type(), field.schema()); + if (evolveDataType != null) { + String fieldName = tableSchema.findColumnName(nestedField.fieldId()); + schemaUpdateConsumer.updateType(fieldName, evolveDataType); + } + if (nestedField.isRequired() && field.schema().isOptional()) { + String fieldName = tableSchema.findColumnName(nestedField.fieldId()); + schemaUpdateConsumer.makeOptional(fieldName); + } + evolveSchemaFromConnectSchema( + field.schema(), nestedField.type(), nestedField.fieldId(), schemaUpdateConsumer); + } + } + } else { + logMismatchedType(recordSchema.type(), tableType); + } + break; + case ARRAY: + if (tableType.isListType()) { + ListType listType = tableType.asListType(); + evolveSchemaFromConnectSchema( + recordSchema.valueSchema(), + listType.elementType(), + listType.elementId(), + schemaUpdateConsumer); + } else { + logMismatchedType(recordSchema.type(), tableType); + } + break; + case MAP: + if (tableType.isMapType()) { + MapType mapType = tableType.asMapType(); + evolveSchemaFromConnectSchema( + recordSchema.valueSchema(), + mapType.valueType(), + mapType.valueId(), + schemaUpdateConsumer); + } else { + logMismatchedType(recordSchema.type(), tableType); + } + break; + default: + break; + } + } + + private void logMismatchedType( + org.apache.kafka.connect.data.Schema.Type recordSchemaType, Type tableType) { + LOG.warn( + "Record schema of type {} does not match table of type {}", recordSchemaType, tableType); + } + private NestedField lookupStructField(String fieldName, StructType schema, int structFieldId) { if (nameMapping == null) { return config.schemaCaseInsensitive() diff --git a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java index f2abb9d17da7..1030e5839ba1 100644 --- a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java +++ b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java @@ -874,6 +874,426 @@ private void assertTypesAddedFromStruct(Function fn) { assertThat(fn.apply("ma")).isInstanceOf(MapType.class); } + @Test + public void testNestedSchemaEvolutionStructWithNullValue() { + org.apache.iceberg.Schema nestedStructSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, "nested", StructType.of(NestedField.required(3, "a", IntegerType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(nestedStructSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema connectNestedSchema = + SchemaBuilder.struct() + .optional() + .field("a", Schema.INT32_SCHEMA) + .field("b", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", connectNestedSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(1); + AddColumn addCol = addCols.iterator().next(); + assertThat(addCol.parentName()).isEqualTo("nested"); + assertThat(addCol.name()).isEqualTo("b"); + assertThat(addCol.type()).isInstanceOf(StringType.class); + } + + @Test + public void testNoSchemaEvolutionStructWithNullValue() { + org.apache.iceberg.Schema nestedStructSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, "nested", StructType.of(NestedField.required(3, "a", IntegerType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(nestedStructSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema connectNestedSchema = + SchemaBuilder.struct().optional().field("a", Schema.INT32_SCHEMA).build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", connectNestedSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + assertThat(consumer.addColumns()).isEmpty(); + assertThat(consumer.makeOptionals()).isEmpty(); + assertThat(consumer.updateTypes()).isEmpty(); + assertThat(consumer.empty()).isTrue(); + } + + @Test + @SuppressWarnings("unchecked") + public void testNestedSchemaEvolutionListOfStructsWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required( + 1, + "items", + ListType.ofRequired( + 2, + StructType.of( + NestedField.required(3, "product_id", IntegerType.get()), + NestedField.optional( + 4, + "details", + StructType.of(NestedField.required(5, "name", StringType.get()))))))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema detailsSchema = + SchemaBuilder.struct() + .optional() + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("price", Schema.OPTIONAL_FLOAT64_SCHEMA) + .build(); + Schema itemSchema = + SchemaBuilder.struct() + .field("product_id", Schema.INT32_SCHEMA) + .field("details", detailsSchema) + .build(); + Schema connectSchema = + SchemaBuilder.struct().field("items", SchemaBuilder.array(itemSchema).build()).build(); + + Struct item = new Struct(itemSchema).put("product_id", 101).put("details", null); + Struct data = new Struct(connectSchema).put("items", ImmutableList.of(item)); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + List items = (List) result.getField("items"); + assertThat(items).hasSize(1); + assertThat(items.get(0).getField("product_id")).isEqualTo(101); + assertThat(items.get(0).getField("details")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(1); + AddColumn addCol = addCols.iterator().next(); + assertThat(addCol.parentName()).isEqualTo("items.element.details"); + assertThat(addCol.name()).isEqualTo("price"); + assertThat(addCol.type()).isInstanceOf(DoubleType.class); + } + + @Test + @SuppressWarnings("unchecked") + public void testNestedSchemaEvolutionMapOfStructsWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.optional( + 1, + "metadata", + MapType.ofRequired( + 2, + 3, + StringType.get(), + StructType.of( + NestedField.required(4, "key", StringType.get()), + NestedField.optional( + 5, + "info", + StructType.of(NestedField.required(6, "name", StringType.get()))))))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema infoSchema = + SchemaBuilder.struct() + .optional() + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("value", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema mapValueSchema = + SchemaBuilder.struct().field("key", Schema.STRING_SCHEMA).field("info", infoSchema).build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("metadata", SchemaBuilder.map(Schema.STRING_SCHEMA, mapValueSchema).build()) + .build(); + + Struct mapValue = new Struct(mapValueSchema).put("key", "source_system").put("info", null); + Struct data = new Struct(connectSchema).put("metadata", ImmutableMap.of("source", mapValue)); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + Map metadata = (Map) result.getField("metadata"); + assertThat(metadata).containsKey("source"); + assertThat(metadata.get("source").getField("key")).isEqualTo("source_system"); + assertThat(metadata.get("source").getField("info")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(1); + AddColumn addCol = addCols.iterator().next(); + assertThat(addCol.parentName()).isEqualTo("metadata.value.info"); + assertThat(addCol.name()).isEqualTo("value"); + assertThat(addCol.type()).isInstanceOf(StringType.class); + } + + @Test + public void testNestedSchemaEvolutionStructInStructWithNullParent() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, + "customer", + StructType.of( + NestedField.required(3, "customer_id", IntegerType.get()), + NestedField.optional( + 4, + "details", + StructType.of(NestedField.required(5, "name", StringType.get())))))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema detailsSchema = + SchemaBuilder.struct() + .optional() + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("email", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema customerSchema = + SchemaBuilder.struct() + .optional() + .field("customer_id", Schema.INT32_SCHEMA) + .field("details", detailsSchema) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("customer", customerSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("customer", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("customer")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(1); + AddColumn addCol = addCols.iterator().next(); + assertThat(addCol.parentName()).isEqualTo("customer.details"); + assertThat(addCol.name()).isEqualTo("email"); + assertThat(addCol.type()).isInstanceOf(StringType.class); + } + + @Test + public void testNestedSchemaEvolutionTypePromotionWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, + "nested", + StructType.of( + NestedField.required(3, "a", IntegerType.get()), + NestedField.required(4, "b", FloatType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema connectNestedSchema = + SchemaBuilder.struct() + .optional() + .field("a", Schema.INT64_SCHEMA) + .field("b", Schema.FLOAT64_SCHEMA) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", connectNestedSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + Collection updates = consumer.updateTypes(); + assertThat(updates).hasSize(2); + Map updateMap = Maps.newHashMap(); + updates.forEach(update -> updateMap.put(update.name(), update)); + assertThat(updateMap.get("nested.a").type()).isInstanceOf(LongType.class); + assertThat(updateMap.get("nested.b").type()).isInstanceOf(DoubleType.class); + } + + @Test + public void testNestedSchemaEvolutionMakeOptionalWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, + "nested", + StructType.of( + NestedField.required(3, "a", IntegerType.get()), + NestedField.required(4, "b", StringType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema connectNestedSchema = + SchemaBuilder.struct() + .optional() + .field("a", Schema.OPTIONAL_INT32_SCHEMA) + .field("b", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", connectNestedSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + Collection makeOptionals = consumer.makeOptionals(); + assertThat(makeOptionals).hasSize(2); + Map optionalMap = Maps.newHashMap(); + makeOptionals.forEach(mo -> optionalMap.put(mo.name(), mo)); + assertThat(optionalMap).containsKey("nested.a"); + assertThat(optionalMap).containsKey("nested.b"); + } + + @Test + public void testSchemaEvolutionForFieldAndNestedFieldsAcrossTwoRecords() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.required( + 2, "nested", StructType.of(NestedField.required(3, "x", IntegerType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema nestedSchema = + SchemaBuilder.struct() + .optional() + .field("x", Schema.INT32_SCHEMA) + .field("y", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", nestedSchema) + .build(); + + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + Collection makeOptionals = consumer.makeOptionals(); + assertThat(makeOptionals).hasSize(1); + assertThat(makeOptionals.iterator().next().name()).isEqualTo("nested"); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(0); + + org.apache.iceberg.Schema updatedSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, "nested", StructType.of(NestedField.required(3, "x", IntegerType.get())))); + when(table.schema()).thenReturn(updatedSchema); + RecordConverter converter2 = new RecordConverter(table, config); + + SchemaUpdate.Consumer consumer2 = new SchemaUpdate.Consumer(); + Record result2 = converter2.convert(data, consumer2); + + assertThat(result2.getField("id")).isEqualTo(1); + assertThat(result2.getField("nested")).isNull(); + + Collection addCols2 = consumer2.addColumns(); + assertThat(addCols2).hasSize(1); + AddColumn addCol2 = addCols2.iterator().next(); + assertThat(addCol2.parentName()).isEqualTo("nested"); + assertThat(addCol2.name()).isEqualTo("y"); + assertThat(addCol2.type()).isInstanceOf(StringType.class); + } + + @Test + public void testNoNestedSchemaEvolutionMapKeyWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, + "data", + MapType.ofRequired( + 3, + 4, + StructType.of(NestedField.required(5, "k1", StringType.get())), + StringType.get()))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema keySchema = + SchemaBuilder.struct() + .field("k1", Schema.STRING_SCHEMA) + .field("k2", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema mapSchema = + SchemaBuilder.map(keySchema, Schema.OPTIONAL_STRING_SCHEMA).optional().build(); + Schema connectSchema = + SchemaBuilder.struct().field("id", Schema.INT32_SCHEMA).field("data", mapSchema).build(); + + Struct data = new Struct(connectSchema).put("id", 1).put("data", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("data")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(0); + } + @Test public void testEvolveTypeDetectionStruct() { org.apache.iceberg.Schema tableSchema = From 7c13104c8c20323c895aeb33fe5ca1f3b127889f Mon Sep 17 00:00:00 2001 From: gaborkaszab Date: Tue, 23 Jun 2026 00:17:15 +0200 Subject: [PATCH 24/73] Core: Introduce builder for TrackedFile (#16769) --- .../apache/iceberg/DeletionVectorStruct.java | 21 + .../apache/iceberg/TrackedFileBuilder.java | 364 ++++++++ .../org/apache/iceberg/TrackedFileStruct.java | 24 +- .../iceberg/TestDeletionVectorStruct.java | 58 ++ .../iceberg/TestTrackedFileAdapters.java | 156 ++-- .../iceberg/TestTrackedFileBuilder.java | 863 ++++++++++++++++++ .../apache/iceberg/TestTrackedFileStruct.java | 61 +- 7 files changed, 1422 insertions(+), 125 deletions(-) create mode 100644 core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java create mode 100644 core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java diff --git a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java index 04d23fa33abe..3f5be0756fad 100644 --- a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java +++ b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java @@ -19,6 +19,7 @@ package org.apache.iceberg; import java.io.Serializable; +import java.util.Objects; import org.apache.iceberg.avro.SupportsIndexProjection; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -128,6 +129,26 @@ static Builder builder() { return new Builder(); } + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (!(other instanceof DeletionVectorStruct)) { + return false; + } + + DeletionVectorStruct that = (DeletionVectorStruct) other; + return Objects.equals(location, that.location) + && offset == that.offset + && sizeInBytes == that.sizeInBytes + && cardinality == that.cardinality; + } + + @Override + public int hashCode() { + return Objects.hash(location, offset, sizeInBytes, cardinality); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java b/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java new file mode 100644 index 000000000000..cfffd0b9c8ca --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +class TrackedFileBuilder { + private final long snapshotId; + private final FileContent contentType; + + // Required fields + private Integer writerFormatVersion = null; + private String location = null; + private FileFormat fileFormat = null; + private Long recordCount = null; + private Long fileSizeInBytes = null; + private PartitionData partitionData = null; + + // optional fields + private Integer specId = null; + private ContentStats contentStats = null; + private Integer sortOrderId = null; + private DeletionVector deletionVector = null; + private ManifestInfo manifestInfo = null; + private ByteBuffer keyMetadata = null; + private List splitOffsets = null; + private List equalityIds = null; + + // tracking-related fields + private Tracking sourceTracking = null; + private boolean dvUpdated = false; + private ByteBuffer deletedPositions = null; + private ByteBuffer replacedPositions = null; + + /** + * Creates a builder for a newly added data file entry. + * + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder data(long newSnapshotId) { + return new TrackedFileBuilder(FileContent.DATA, newSnapshotId); + } + + /** + * Creates a builder for a newly added equality delete file entry. + * + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder equalityDelete(long newSnapshotId) { + return new TrackedFileBuilder(FileContent.EQUALITY_DELETES, newSnapshotId); + } + + /** + * Creates a builder for a newly added data manifest entry. + * + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder dataManifest(long newSnapshotId) { + return new TrackedFileBuilder(FileContent.DATA_MANIFEST, newSnapshotId); + } + + /** + * Creates a builder for a newly added delete manifest entry. + * + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder deleteManifest(long newSnapshotId) { + return new TrackedFileBuilder(FileContent.DELETE_MANIFEST, newSnapshotId); + } + + /** + * Creates a builder for a tracked file derived from {@code source}. + * + * @param source source tracked file to copy fields from + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder from(TrackedFile source, long newSnapshotId) { + Preconditions.checkArgument(source != null, "Invalid source: null"); + return new TrackedFileBuilder(source, newSnapshotId); + } + + /** + * Returns a DELETED tracked file derived from {@code source}. + * + * @param source source tracked file + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFile deleted(TrackedFile source, long newSnapshotId) { + Preconditions.checkArgument(source != null, "Invalid source: null"); + return terminal(source, TrackingBuilder.deleted(source.tracking(), newSnapshotId)); + } + + /** + * Returns a REPLACED tracked file derived from {@code source}. + * + *

Manifest entries cannot transition to REPLACED. + * + * @param source source tracked file + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFile replaced(TrackedFile source, long newSnapshotId) { + Preconditions.checkArgument(source != null, "Invalid source: null"); + Preconditions.checkArgument( + !isLeafManifest(source.contentType()), + "Manifest entries cannot transition to REPLACED, but entry type is: %s", + source.contentType()); + return terminal(source, TrackingBuilder.replaced(source.tracking(), newSnapshotId)); + } + + private static TrackedFile terminal(TrackedFile source, Tracking tracking) { + return new TrackedFileStruct( + tracking, + source.contentType(), + source.writerFormatVersion(), + source.location(), + source.fileFormat(), + (PartitionData) source.partition(), + source.recordCount(), + source.fileSizeInBytes(), + source.specId(), + source.contentStats(), + source.sortOrderId(), + source.deletionVector(), + source.manifestInfo(), + source.keyMetadata(), + source.splitOffsets(), + source.equalityIds()); + } + + private TrackedFileBuilder(FileContent contentType, long snapshotId) { + this.contentType = contentType; + this.snapshotId = snapshotId; + } + + private TrackedFileBuilder(TrackedFile source, long snapshotId) { + this.contentType = source.contentType(); + this.snapshotId = snapshotId; + this.writerFormatVersion = source.writerFormatVersion(); + this.location = source.location(); + this.fileFormat = source.fileFormat(); + this.recordCount = source.recordCount(); + this.fileSizeInBytes = source.fileSizeInBytes(); + this.partitionData = (PartitionData) source.partition(); + this.specId = source.specId(); + this.contentStats = source.contentStats(); + this.sortOrderId = source.sortOrderId(); + this.deletionVector = source.deletionVector(); + this.manifestInfo = source.manifestInfo(); + this.keyMetadata = source.keyMetadata(); + this.splitOffsets = source.splitOffsets(); + this.equalityIds = source.equalityIds(); + this.sourceTracking = source.tracking(); + } + + TrackedFileBuilder writerFormatVersion(int newWriterFormatVersion) { + Preconditions.checkArgument( + newWriterFormatVersion >= 0, + "Invalid writer format version: %s (must be >= 0)", + newWriterFormatVersion); + this.writerFormatVersion = newWriterFormatVersion; + return this; + } + + TrackedFileBuilder location(String newLocation) { + Preconditions.checkArgument(newLocation != null, "Invalid location: null"); + this.location = newLocation; + return this; + } + + TrackedFileBuilder fileFormat(FileFormat newFileFormat) { + Preconditions.checkArgument(newFileFormat != null, "Invalid file format: null"); + this.fileFormat = newFileFormat; + return this; + } + + TrackedFileBuilder recordCount(long newRecordCount) { + Preconditions.checkArgument( + newRecordCount >= 0, "Invalid record count: %s (must be >= 0)", newRecordCount); + this.recordCount = newRecordCount; + return this; + } + + TrackedFileBuilder fileSizeInBytes(long newFileSizeInBytes) { + Preconditions.checkArgument( + newFileSizeInBytes >= 0, + "Invalid file size in bytes: %s (must be >= 0)", + newFileSizeInBytes); + this.fileSizeInBytes = newFileSizeInBytes; + return this; + } + + TrackedFileBuilder specId(int newSpecId) { + Preconditions.checkArgument(newSpecId >= 0, "Invalid spec ID: %s (must be >= 0)", newSpecId); + this.specId = newSpecId; + return this; + } + + TrackedFileBuilder partition(PartitionData newPartitionData) { + Preconditions.checkArgument(newPartitionData != null, "Invalid partition: null"); + this.partitionData = newPartitionData; + return this; + } + + TrackedFileBuilder contentStats(ContentStats newContentStats) { + Preconditions.checkArgument(newContentStats != null, "Invalid content stats: null"); + this.contentStats = newContentStats; + return this; + } + + TrackedFileBuilder sortOrderId(int newSortOrderId) { + Preconditions.checkArgument( + !isLeafManifest(contentType), + "Sort order ID cannot be added to manifest entries, but entry type is: %s", + contentType); + Preconditions.checkArgument( + newSortOrderId >= 0, "Invalid sort order ID: %s (must be >= 0)", newSortOrderId); + this.sortOrderId = newSortOrderId; + return this; + } + + TrackedFileBuilder deletionVector(DeletionVector newDeletionVector) { + Preconditions.checkArgument(newDeletionVector != null, "Invalid deletion vector: null"); + Preconditions.checkArgument( + contentType == FileContent.DATA, + "Deletion vector can only be added to DATA entries, but entry type is: %s", + contentType); + Preconditions.checkArgument( + this.deletionVector == null || !this.deletionVector.equals(newDeletionVector), + "The same deletion vector already added"); + this.deletionVector = newDeletionVector; + this.dvUpdated = true; + return this; + } + + TrackedFileBuilder manifestInfo(ManifestInfo newManifestInfo) { + Preconditions.checkArgument(newManifestInfo != null, "Invalid manifest info: null"); + Preconditions.checkArgument( + isLeafManifest(contentType), + "Manifest info can only be added to manifests, but entry type is: %s", + contentType); + this.manifestInfo = newManifestInfo; + return this; + } + + TrackedFileBuilder keyMetadata(ByteBuffer newKeyMetadata) { + Preconditions.checkArgument(newKeyMetadata != null, "Invalid key metadata: null"); + this.keyMetadata = newKeyMetadata; + return this; + } + + TrackedFileBuilder splitOffsets(List newSplitOffsets) { + Preconditions.checkArgument(newSplitOffsets != null, "Invalid split offsets: null"); + Preconditions.checkArgument( + !isLeafManifest(contentType), + "Split offsets cannot be added to manifest entries, but entry type is: %s", + contentType); + this.splitOffsets = newSplitOffsets; + return this; + } + + TrackedFileBuilder equalityIds(List newEqualityIds) { + Preconditions.checkArgument(newEqualityIds != null, "Invalid equality IDs: null"); + Preconditions.checkArgument( + contentType == FileContent.EQUALITY_DELETES, + "Equality IDs can only be added to EQUALITY_DELETES entries, but entry type is: %s", + contentType); + this.equalityIds = newEqualityIds; + return this; + } + + TrackedFileBuilder deletedPositions(ByteBuffer newDeletedPositions) { + Preconditions.checkArgument(newDeletedPositions != null, "Invalid deleted positions: null"); + Preconditions.checkArgument( + isLeafManifest(contentType), + "Deleted positions can only be added to manifest entries, but entry type is: %s", + contentType); + this.deletedPositions = newDeletedPositions; + return this; + } + + TrackedFileBuilder replacedPositions(ByteBuffer newReplacedPositions) { + Preconditions.checkArgument(newReplacedPositions != null, "Invalid replaced positions: null"); + Preconditions.checkArgument( + isLeafManifest(contentType), + "Replaced positions can only be added to manifest entries, but entry type is: %s", + contentType); + this.replacedPositions = newReplacedPositions; + return this; + } + + private static boolean isLeafManifest(FileContent contentType) { + return contentType == FileContent.DATA_MANIFEST || contentType == FileContent.DELETE_MANIFEST; + } + + TrackedFile build() { + Preconditions.checkArgument( + writerFormatVersion != null, "Missing required field: writer format version"); + Preconditions.checkArgument(location != null, "Missing required field: location"); + Preconditions.checkArgument(fileFormat != null, "Missing required field: file format"); + Preconditions.checkArgument(recordCount != null, "Missing required field: record count"); + Preconditions.checkArgument( + fileSizeInBytes != null, "Missing required field: file size in bytes"); + Preconditions.checkArgument(partitionData != null, "Missing required field: partition data"); + Preconditions.checkArgument( + !isLeafManifest(contentType) || manifestInfo != null, + "Missing required field: manifest info"); + Preconditions.checkArgument( + contentType != FileContent.EQUALITY_DELETES || equalityIds != null, + "Missing required field: equality IDs"); + + TrackingBuilder trackingBuilder = + sourceTracking == null + ? TrackingBuilder.added(snapshotId) + : TrackingBuilder.from(sourceTracking, snapshotId); + + if (dvUpdated) { + trackingBuilder.dvUpdated(); + } + + if (deletedPositions != null) { + trackingBuilder.deletedPositions(deletedPositions); + } + + if (replacedPositions != null) { + trackingBuilder.replacedPositions(replacedPositions); + } + + return new TrackedFileStruct( + trackingBuilder.build(), + contentType, + writerFormatVersion, + location, + fileFormat, + partitionData, + recordCount, + fileSizeInBytes, + specId, + contentStats, + sortOrderId, + deletionVector, + manifestInfo, + keyMetadata, + splitOffsets, + equalityIds); + } +} diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 3c350b89373d..9a44e8045cbb 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -72,13 +72,13 @@ public PartitionData copy() { private int writerFormatVersion = -1; private String location = null; private FileFormat fileFormat = null; + private Tracking tracking = null; private long recordCount = -1L; private long fileSizeInBytes = -1L; - private Integer specId = null; private PartitionData partitionData = EMPTY_PARTITION_DATA; // optional fields - private Tracking tracking = null; + private Integer specId = null; private ContentStats contentStats = null; private Integer sortOrderId = null; private DeletionVector deletionVector = null; @@ -102,7 +102,6 @@ public PartitionData copy() { super(BASE_TYPE.fields().size()); } - /** Constructor that accepts required fields. */ TrackedFileStruct( Tracking tracking, FileContent contentType, @@ -111,7 +110,15 @@ public PartitionData copy() { FileFormat fileFormat, PartitionData partition, long recordCount, - long fileSizeInBytes) { + long fileSizeInBytes, + Integer specId, + ContentStats contentStats, + Integer sortOrderId, + DeletionVector deletionVector, + ManifestInfo manifestInfo, + ByteBuffer keyMetadata, + List splitOffsets, + List equalityIds) { super(BASE_TYPE.fields().size()); this.tracking = tracking; this.contentType = contentType; @@ -123,6 +130,15 @@ public PartitionData copy() { if (partition != null) { this.partitionData = partition; } + + this.specId = specId; + this.contentStats = contentStats; + this.sortOrderId = sortOrderId; + this.deletionVector = deletionVector; + this.manifestInfo = manifestInfo; + this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); + this.splitOffsets = ArrayUtil.toLongArray(splitOffsets); + this.equalityIds = ArrayUtil.toIntArray(equalityIds); } /** Copy constructor. */ diff --git a/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java b/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java index 8242be38e94a..0f08b59e150d 100644 --- a/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java @@ -163,6 +163,64 @@ void testBuilderMissingRequiredFields() { .hasMessage("Missing required value: cardinality"); } + @Test + void testDvEquality() { + DeletionVectorStruct dv = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(256L) + .sizeInBytes(128L) + .cardinality(42L) + .build(); + + DeletionVectorStruct sameDv = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(256L) + .sizeInBytes(128L) + .cardinality(42L) + .build(); + + DeletionVectorStruct dvWithDifferentLocation = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv2.puffin") + .offset(256L) + .sizeInBytes(128L) + .cardinality(42L) + .build(); + + DeletionVectorStruct dvWithDifferentOffset = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(1L) + .sizeInBytes(128L) + .cardinality(42L) + .build(); + + DeletionVectorStruct dvWithDifferentSize = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(256L) + .sizeInBytes(8L) + .cardinality(42L) + .build(); + + DeletionVectorStruct dvWithDifferentCardinality = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(256L) + .sizeInBytes(128L) + .cardinality(2L) + .build(); + + assertThat(dv).isEqualTo(dv); + assertThat(dv).isEqualTo(sameDv); + assertThat(dv).isNotEqualTo(dvWithDifferentLocation); + assertThat(dv).isNotEqualTo(dvWithDifferentOffset); + assertThat(dv).isNotEqualTo(dvWithDifferentSize); + assertThat(dv).isNotEqualTo(dvWithDifferentCardinality); + } + @Test void testBuilderRejectsInvalidValuesAtSetter() { assertThatThrownBy(() -> DeletionVectorStruct.builder().location(null)) diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index e13a342b8d5a..dc8f26dce8a8 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -57,13 +57,9 @@ class TestTrackedFileAdapters { .identity("category") .withSpecId(PARTITIONED_SPEC_ID) .build(); - - // Passed for unpartitioned test files, where there is no partition tuple. - private static final PartitionData NO_PARTITION = null; + private static final PartitionData PARTITION = partition("books"); // Tracking field ordinals, looked up from the schema so the tests do not hard-code offsets. - private static final int STATUS_ORDINAL = ordinalOf(Tracking.schema(), "status"); - private static final int SNAPSHOT_ID_ORDINAL = ordinalOf(Tracking.schema(), "snapshot_id"); private static final int DATA_SEQUENCE_NUMBER_ORDINAL = ordinalOf(Tracking.schema(), "sequence_number"); private static final int FILE_SEQUENCE_NUMBER_ORDINAL = @@ -75,40 +71,34 @@ class TestTrackedFileAdapters { // TrackedFile optional field ordinals, looked up from the schema. private static final Types.StructType TRACKED_FILE_SCHEMA = TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()); + private static final int CONTENT_TYPE_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "content_type"); private static final int SPEC_ID_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "spec_id"); - private static final int CONTENT_STATS_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "content_stats"); - private static final int SORT_ORDER_ID_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "sort_order_id"); private static final int DELETION_VECTOR_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "deletion_vector"); - private static final int KEY_METADATA_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "key_metadata"); - private static final int SPLIT_OFFSETS_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "split_offsets"); - private static final int EQUALITY_IDS_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "equality_ids"); @Test void testDataFileAdapterDelegation() { - PartitionData partition = partition("books"); - - TrackedFileStruct file = - new TrackedFileStruct( - createTracking(), - FileContent.DATA, - WRITER_FORMAT_VERSION, - DATA_FILE_LOCATION, - FileFormat.PARQUET, - partition, - 100L, - 1024L); - file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID); - file.set(CONTENT_STATS_ORDINAL, createContentStats()); - file.set(SORT_ORDER_ID_ORDINAL, 3); - file.set(KEY_METADATA_ORDINAL, ByteBuffer.wrap(new byte[] {1, 2, 3})); - file.set(SPLIT_OFFSETS_ORDINAL, ImmutableList.of(50L, 100L)); + TrackedFile file = + TrackedFileBuilder.data(42L) + .writerFormatVersion(WRITER_FORMAT_VERSION) + .location(DATA_FILE_LOCATION) + .fileFormat(FileFormat.PARQUET) + .partition(PARTITION) + .recordCount(100L) + .fileSizeInBytes(1024L) + .specId(PARTITIONED_SPEC_ID) + .contentStats(createContentStats()) + .sortOrderId(3) + .keyMetadata(ByteBuffer.wrap(new byte[] {1, 2, 3})) + .splitOffsets(ImmutableList.of(50L, 100L)) + .build(); + populateTrackingFields(file); DataFile dataFile = TrackedFileAdapters.asDataFile(file, specsById(PARTITIONED_SPEC)); assertThat(dataFile.pos()).isEqualTo(MANIFEST_POS); assertThat(dataFile.specId()).isEqualTo(PARTITIONED_SPEC_ID); - assertThat(dataFile.partition()).isSameAs(partition); + assertThat(dataFile.partition()).isSameAs(PARTITION); assertThat(dataFile.content()).isEqualTo(FileContent.DATA); assertThat(dataFile.location()).isEqualTo(DATA_FILE_LOCATION); assertThat(dataFile.format()).isEqualTo(FileFormat.PARQUET); @@ -139,7 +129,7 @@ void testDataFileAdapterDelegation() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "DATA") void testDataFileAdapterRejectsNonDataContent(FileContent contentType) { - TrackedFileStruct file = trackedFile(contentType); + TrackedFileStruct file = dummyTrackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -148,31 +138,29 @@ void testDataFileAdapterRejectsNonDataContent(FileContent contentType) { @Test void testEqualityDeleteFileAdapterDelegation() { - PartitionData partition = partition("books"); - - TrackedFileStruct file = - new TrackedFileStruct( - createTracking(), - FileContent.EQUALITY_DELETES, - WRITER_FORMAT_VERSION, - "s3://bucket/eq-delete.avro", - FileFormat.AVRO, - partition, - 50L, - 512L); - file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID); - file.set(CONTENT_STATS_ORDINAL, createContentStats()); - file.set(SORT_ORDER_ID_ORDINAL, 5); - file.set(KEY_METADATA_ORDINAL, ByteBuffer.wrap(new byte[] {4, 5})); - file.set(SPLIT_OFFSETS_ORDINAL, ImmutableList.of(200L)); - file.set(EQUALITY_IDS_ORDINAL, ImmutableList.of(1, 2, 3)); + TrackedFile file = + TrackedFileBuilder.equalityDelete(42L) + .writerFormatVersion(WRITER_FORMAT_VERSION) + .location("s3://bucket/eq-delete.avro") + .fileFormat(FileFormat.AVRO) + .partition(PARTITION) + .recordCount(50L) + .fileSizeInBytes(512L) + .specId(PARTITIONED_SPEC_ID) + .contentStats(createContentStats()) + .sortOrderId(5) + .keyMetadata(ByteBuffer.wrap(new byte[] {4, 5})) + .splitOffsets(ImmutableList.of(200L)) + .equalityIds(ImmutableList.of(1, 2, 3)) + .build(); + populateTrackingFields(file); DeleteFile deleteFile = TrackedFileAdapters.asEqualityDeleteFile(file, specsById(PARTITIONED_SPEC)); assertThat(deleteFile.pos()).isEqualTo(MANIFEST_POS); assertThat(deleteFile.specId()).isEqualTo(PARTITIONED_SPEC_ID); - assertThat(deleteFile.partition()).isSameAs(partition); + assertThat(deleteFile.partition()).isSameAs(PARTITION); assertThat(deleteFile.content()).isEqualTo(FileContent.EQUALITY_DELETES); assertThat(deleteFile.location()).isEqualTo("s3://bucket/eq-delete.avro"); assertThat(deleteFile.format()).isEqualTo(FileFormat.AVRO); @@ -203,7 +191,7 @@ void testEqualityDeleteFileAdapterDelegation() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "EQUALITY_DELETES") void testEqualityDeleteFileAdapterRejectsNonEqualityContent(FileContent contentType) { - TrackedFileStruct file = trackedFile(contentType); + TrackedFileStruct file = dummyTrackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asEqualityDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -220,19 +208,18 @@ void testDVDeleteFileAdapterDelegation() { .cardinality(10L) .build(); - PartitionData partition = partition("books"); - TrackedFileStruct file = - new TrackedFileStruct( - createTracking(), - FileContent.DATA, - WRITER_FORMAT_VERSION, - DATA_FILE_LOCATION, - FileFormat.PARQUET, - partition, - 100L, - 1024L); - file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID); - file.set(DELETION_VECTOR_ORDINAL, dv); + TrackedFile file = + TrackedFileBuilder.data(42L) + .writerFormatVersion(WRITER_FORMAT_VERSION) + .location(DATA_FILE_LOCATION) + .fileFormat(FileFormat.PARQUET) + .partition(PARTITION) + .recordCount(100L) + .fileSizeInBytes(1024L) + .specId(PARTITIONED_SPEC_ID) + .deletionVector(dv) + .build(); + populateTrackingFields(file); DeleteFile dvFile = TrackedFileAdapters.asDVDeleteFile(file, specsById(PARTITIONED_SPEC)); @@ -251,7 +238,7 @@ void testDVDeleteFileAdapterDelegation() { // fields delegated from TrackedFile / Tracking assertThat(dvFile.pos()).isEqualTo(MANIFEST_POS); assertThat(dvFile.specId()).isEqualTo(PARTITIONED_SPEC_ID); - assertThat(dvFile.partition()).isSameAs(partition); + assertThat(dvFile.partition()).isSameAs(PARTITION); assertThat(dvFile.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(dvFile.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); assertThat(dvFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); @@ -273,7 +260,7 @@ void testDVDeleteFileAdapterDelegation() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "DATA") void testDVDeleteFileAdapterRejectsNonDataContent(FileContent contentType) { - TrackedFileStruct file = trackedFile(contentType); + TrackedFileStruct file = dummyTrackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asDVDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -282,7 +269,7 @@ void testDVDeleteFileAdapterRejectsNonDataContent(FileContent contentType) { @Test void testDVDeleteFileAdapterRejectsNullDeletionVector() { - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); assertThatThrownBy(() -> TrackedFileAdapters.asDVDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -291,7 +278,7 @@ void testDVDeleteFileAdapterRejectsNullDeletionVector() { @Test void testNullContentStatsReturnsNullStats() { - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, UNPARTITIONED); @@ -307,19 +294,19 @@ void testNullTrackingReturnsNullTrackingFields() { // Files read before manifest inheritance have no tracking; tracking-derived fields must be // null rather than throwing. assertNullTrackingFields( - TrackedFileAdapters.asDataFile(trackedFile(FileContent.DATA), UNPARTITIONED)); + TrackedFileAdapters.asDataFile(dummyTrackedFile(FileContent.DATA), UNPARTITIONED)); assertNullTrackingFields( TrackedFileAdapters.asEqualityDeleteFile( - trackedFile(FileContent.EQUALITY_DELETES), UNPARTITIONED)); + dummyTrackedFile(FileContent.EQUALITY_DELETES), UNPARTITIONED)); - TrackedFileStruct dvFile = trackedFile(FileContent.DATA); + TrackedFileStruct dvFile = dummyTrackedFile(FileContent.DATA); dvFile.set(DELETION_VECTOR_ORDINAL, deletionVector()); assertNullTrackingFields(TrackedFileAdapters.asDVDeleteFile(dvFile, UNPARTITIONED)); } @Test void testUnpartitionedFilePartitionIsEmpty() { - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, UNPARTITIONED); @@ -330,7 +317,7 @@ void testUnpartitionedFilePartitionIsEmpty() { @Test void testNullSpecIdResolvesToUnpartitionedSpec() { PartitionSpec unpartitioned = PartitionSpec.builderFor(new Schema()).withSpecId(5).build(); - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, specsById(unpartitioned)); @@ -341,7 +328,7 @@ void testNullSpecIdResolvesToUnpartitionedSpec() { void testNullSpecIdThrowsWhenNoUnpartitionedSpec() { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); PartitionSpec partitioned = PartitionSpec.builderFor(schema).identity("id").build(); - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, specsById(partitioned))) .isInstanceOf(IllegalArgumentException.class) @@ -350,7 +337,7 @@ void testNullSpecIdThrowsWhenNoUnpartitionedSpec() { @Test void testUnknownSpecIdThrows() { - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); file.set(SPEC_ID_ORDINAL, 99); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, ImmutableMap.of())) @@ -361,7 +348,7 @@ void testUnknownSpecIdThrows() { @Test void testSpecIdMismatchThrows() { int mismatchedSpecId = PARTITIONED_SPEC_ID + 1; - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID); PartitionSpec mismatched = PartitionSpec.builderFor(PARTITION_SCHEMA) @@ -399,28 +386,19 @@ private static PartitionData partition(String category) { } /** Minimal file with no tracking, used by the rejection and null-tracking tests. */ - private static TrackedFileStruct trackedFile(FileContent contentType) { - return new TrackedFileStruct( - null, - contentType, - WRITER_FORMAT_VERSION, - "s3://bucket/file", - FileFormat.PARQUET, - NO_PARTITION, - 1L, - 1L); + private static TrackedFileStruct dummyTrackedFile(FileContent contentType) { + TrackedFileStruct file = new TrackedFileStruct(); + file.set(CONTENT_TYPE_ORDINAL, contentType.id()); + return file; } - private static TrackingStruct createTracking() { - TrackingStruct tracking = new TrackingStruct(); - tracking.set(STATUS_ORDINAL, EntryStatus.ADDED.id()); - tracking.set(SNAPSHOT_ID_ORDINAL, 42L); + private static void populateTrackingFields(TrackedFile file) { + TrackingStruct tracking = (TrackingStruct) file.tracking(); tracking.set(DATA_SEQUENCE_NUMBER_ORDINAL, DATA_SEQUENCE_NUMBER); tracking.set(FILE_SEQUENCE_NUMBER_ORDINAL, FILE_SEQUENCE_NUMBER); tracking.set(FIRST_ROW_ID_ORDINAL, FIRST_ROW_ID); tracking.setManifestLocation(MANIFEST_LOCATION); tracking.set(MANIFEST_POS_ORDINAL, MANIFEST_POS); - return tracking; } private static DeletionVector deletionVector() { diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java new file mode 100644 index 000000000000..9e96923f2fc0 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java @@ -0,0 +1,863 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.ByteBuffer; +import java.util.stream.Stream; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class TestTrackedFileBuilder { + private static final int WRITER_FORMAT_VERSION_V4 = 4; + private static final Schema TABLE_SCHEMA = + new Schema( + optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get())); + private static final Types.StructType PARTITION_TYPE = + PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build().partitionType(); + private static final PartitionData PARTITION_DATA = new PartitionData(PARTITION_TYPE); + private static final ManifestInfo MANIFEST_INFO = + ManifestInfoStruct.builder() + .addedFilesCount(10) + .existingFilesCount(20) + .deletedFilesCount(3) + .replacedFilesCount(2) + .addedRowsCount(1000L) + .existingRowsCount(2000L) + .deletedRowsCount(300L) + .replacedRowsCount(200L) + .minSequenceNumber(5L) + .build(); + private static final DeletionVector DELETION_VECTOR = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(0L) + .sizeInBytes(128L) + .cardinality(10L) + .build(); + private static final ContentStats CONTENT_STATS = + BaseContentStats.builder() + .withTableSchema(TABLE_SCHEMA) + .withFieldStats( + BaseFieldStats.builder() + .fieldId(1) + .type(Types.IntegerType.get()) + .valueCount(2000L) + .nullValueCount(0L) + .lowerBound(1) + .upperBound(1000) + .build()) + .withFieldStats( + BaseFieldStats.builder() + .fieldId(2) + .type(Types.StringType.get()) + .valueCount(2000L) + .nullValueCount(5L) + .lowerBound("a") + .upperBound("z") + .build()) + .build(); + private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[] {1, 2, 3}); + private static final ImmutableList SPLIT_OFFSETS = ImmutableList.of(0L, 4096L, 8192L); + private static final ByteBuffer DELETED_POSITIONS = ByteBuffer.wrap(new byte[] {10, 11, 12}); + private static final ByteBuffer REPLACED_POSITIONS = ByteBuffer.wrap(new byte[] {20, 21, 22}); + + private static Stream missingRequiredFieldCases() { + return Stream.of( + Arguments.of("writerFormatVersion", "Missing required field: writer format version"), + Arguments.of("location", "Missing required field: location"), + Arguments.of("fileFormat", "Missing required field: file format"), + Arguments.of("recordCount", "Missing required field: record count"), + Arguments.of("fileSizeInBytes", "Missing required field: file size in bytes"), + Arguments.of("partition", "Missing required field: partition data")); + } + + @ParameterizedTest + @MethodSource("missingRequiredFieldCases") + public void missingRequiredFields(String missingField, String expectedMessage) { + TrackedFileBuilder dataBuilder = + builderWithMissingRequiredField(TrackedFileBuilder.data(50L), missingField); + TrackedFileBuilder equalityDeleteBuilder = + builderWithMissingRequiredField(TrackedFileBuilder.equalityDelete(50L), missingField); + TrackedFileBuilder dataManifestBuilder = + builderWithMissingRequiredField(TrackedFileBuilder.dataManifest(50L), missingField); + TrackedFileBuilder deleteManifestBuilder = + builderWithMissingRequiredField(TrackedFileBuilder.deleteManifest(50L), missingField); + + assertThatThrownBy(dataBuilder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedMessage); + assertThatThrownBy(equalityDeleteBuilder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedMessage); + assertThatThrownBy(dataManifestBuilder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedMessage); + assertThatThrownBy(deleteManifestBuilder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedMessage); + } + + private TrackedFileBuilder builderWithMissingRequiredField( + TrackedFileBuilder builder, String missingField) { + if (!"writerFormatVersion".equals(missingField)) { + builder.writerFormatVersion(WRITER_FORMAT_VERSION_V4); + } + if (!"location".equals(missingField)) { + builder.location("s3://bucket/data/file"); + } + if (!"fileFormat".equals(missingField)) { + builder.fileFormat(FileFormat.PARQUET); + } + if (!"recordCount".equals(missingField)) { + builder.recordCount(2000L); + } + if (!"fileSizeInBytes".equals(missingField)) { + builder.fileSizeInBytes(12345L); + } + if (!"partition".equals(missingField)) { + builder.partition(PARTITION_DATA); + } + return builder; + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void missingFieldsForManifests(TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy( + () -> + builder + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/manifest.avro") + .fileFormat(FileFormat.AVRO) + .recordCount(420L) + .fileSizeInBytes(556L) + .partition(PARTITION_DATA) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required field: manifest info"); + } + + @Test + public void missingEqualityIdsForEqualityDeletes() { + assertThatThrownBy( + () -> + TrackedFileBuilder.equalityDelete(50L) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/eq_delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required field: equality IDs"); + } + + private static Stream nonEqualityDeleteBuilders() { + return Stream.of( + Arguments.of(TrackedFileBuilder.data(10L), FileContent.DATA), + Arguments.of(TrackedFileBuilder.dataManifest(10L), FileContent.DATA_MANIFEST), + Arguments.of(TrackedFileBuilder.deleteManifest(10L), FileContent.DELETE_MANIFEST), + Arguments.of(TrackedFileBuilder.from(sourceData(12L), 20L), FileContent.DATA), + Arguments.of( + TrackedFileBuilder.from(sourceDataManifest(21L), 25L), FileContent.DATA_MANIFEST), + Arguments.of( + TrackedFileBuilder.from(sourceDeleteManifest(12L), 20L), FileContent.DELETE_MANIFEST)); + } + + @ParameterizedTest + @MethodSource("nonEqualityDeleteBuilders") + public void invalidEqualityIdsForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.equalityIds(ImmutableList.of(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Equality IDs can only be added to EQUALITY_DELETES entries, but entry type is: " + + contentType); + } + + private static Stream nonDataBuilders() { + return Stream.of( + Arguments.of(TrackedFileBuilder.equalityDelete(10L), FileContent.EQUALITY_DELETES), + Arguments.of(TrackedFileBuilder.dataManifest(10L), FileContent.DATA_MANIFEST), + Arguments.of(TrackedFileBuilder.deleteManifest(10L), FileContent.DELETE_MANIFEST), + Arguments.of( + TrackedFileBuilder.from(sourceEqualityDelete(12L), 20L), FileContent.EQUALITY_DELETES), + Arguments.of( + TrackedFileBuilder.from(sourceDataManifest(21L), 25L), FileContent.DATA_MANIFEST), + Arguments.of( + TrackedFileBuilder.from(sourceDeleteManifest(12L), 20L), FileContent.DELETE_MANIFEST)); + } + + @ParameterizedTest + @MethodSource("nonDataBuilders") + public void invalidDeletionVectorForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.deletionVector(DELETION_VECTOR)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Deletion vector can only be added to DATA entries, but entry type is: " + contentType); + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void invalidSortOrderIdForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.sortOrderId(1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Sort order ID cannot be added to manifest entries, but entry type is: " + contentType); + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void invalidSplitOffsetsForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.splitOffsets(SPLIT_OFFSETS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Split offsets cannot be added to manifest entries, but entry type is: " + contentType); + } + + private static Stream nonManifestBuilders() { + return Stream.of( + Arguments.of(TrackedFileBuilder.data(10L), FileContent.DATA), + Arguments.of(TrackedFileBuilder.equalityDelete(10L), FileContent.EQUALITY_DELETES), + Arguments.of(TrackedFileBuilder.from(sourceData(12L), 20L), FileContent.DATA), + Arguments.of( + TrackedFileBuilder.from(sourceEqualityDelete(12L), 20L), FileContent.EQUALITY_DELETES)); + } + + @ParameterizedTest + @MethodSource("nonManifestBuilders") + public void invalidManifestInfoForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.manifestInfo(MANIFEST_INFO)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Manifest info can only be added to manifests, but entry type is: " + contentType); + } + + @ParameterizedTest + @MethodSource("nonManifestBuilders") + public void invalidDeletedPositionsForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.deletedPositions(DELETED_POSITIONS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Deleted positions can only be added to manifest entries, but entry type is: " + + contentType); + } + + @ParameterizedTest + @MethodSource("nonManifestBuilders") + public void invalidReplacedPositionsForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.replacedPositions(REPLACED_POSITIONS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Replaced positions can only be added to manifest entries, but entry type is: " + + contentType); + } + + @Test + public void invalidNullInputs() { + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).location(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid location: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).fileFormat(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file format: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).partition(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid partition: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).contentStats(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid content stats: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).deletionVector(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid deletion vector: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(30L).manifestInfo(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid manifest info: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).keyMetadata(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid key metadata: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).splitOffsets(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid split offsets: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.equalityDelete(30L).equalityIds(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid equality IDs: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.from(null, 20L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid source: null"); + + assertThatThrownBy( + () -> + TrackedFileBuilder.from( + entryWithInheritedSeqNums(sourceDataManifest(10L), 15L), 20L) + .deletedPositions(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid deleted positions: null"); + + assertThatThrownBy( + () -> + TrackedFileBuilder.from( + entryWithInheritedSeqNums(sourceDeleteManifest(100L), 150L), 200L) + .replacedPositions(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid replaced positions: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.deleted(null, 20L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid source: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.replaced(null, 20L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid source: null"); + } + + @Test + public void invalidNegativeInputs() { + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).writerFormatVersion(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid writer format version: -1 (must be >= 0)"); + + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).recordCount(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid record count: -1 (must be >= 0)"); + + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).fileSizeInBytes(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file size in bytes: -1 (must be >= 0)"); + + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).specId(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid spec ID: -1 (must be >= 0)"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(40L).sortOrderId(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid sort order ID: -1 (must be >= 0)"); + } + + @Test + public void buildDataFileWithRequiredFieldsOnly() { + TrackedFile trackedFile = + TrackedFileBuilder.data(50L) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .build(); + + assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/file.parquet"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(trackedFile.recordCount()).isEqualTo(2000L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + assertThat(trackedFile.tracking().dvSnapshotId()).isNull(); + + assertThat(trackedFile.specId()).isNull(); + assertThat(trackedFile.contentStats()).isNull(); + assertThat(trackedFile.sortOrderId()).isNull(); + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.manifestInfo()).isNull(); + assertThat(trackedFile.keyMetadata()).isNull(); + assertThat(trackedFile.splitOffsets()).isNull(); + assertThat(trackedFile.equalityIds()).isNull(); + } + + @Test + public void buildDataFileWithAllFields() { + TrackedFile trackedFile = + TrackedFileBuilder.data(50L) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .specId(7) + .partition(PARTITION_DATA) + .contentStats(CONTENT_STATS) + .sortOrderId(3) + .deletionVector(DELETION_VECTOR) + .keyMetadata(KEY_METADATA) + .splitOffsets(SPLIT_OFFSETS) + .build(); + + assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/file.parquet"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(trackedFile.recordCount()).isEqualTo(2000L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L); + assertThat(trackedFile.specId()).isEqualTo(7); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS); + assertThat(trackedFile.sortOrderId()).isEqualTo(3); + assertThat(trackedFile.deletionVector()).isSameAs(DELETION_VECTOR); + assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA); + assertThat(trackedFile.splitOffsets()).isEqualTo(SPLIT_OFFSETS); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(50L); + + // Unsupported fields for data files + assertThat(trackedFile.manifestInfo()).isNull(); + assertThat(trackedFile.equalityIds()).isNull(); + } + + @Test + public void buildEqualityDeleteFileWithRequiredFieldsOnly() { + TrackedFile trackedFile = + TrackedFileBuilder.equalityDelete(50L) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/eq_delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .equalityIds(ImmutableList.of(1)) + .build(); + + assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/eq_delete.parquet"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(trackedFile.recordCount()).isEqualTo(2000L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.equalityIds()).containsExactly(1); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + + assertThat(trackedFile.specId()).isNull(); + assertThat(trackedFile.contentStats()).isNull(); + assertThat(trackedFile.sortOrderId()).isNull(); + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.manifestInfo()).isNull(); + assertThat(trackedFile.keyMetadata()).isNull(); + assertThat(trackedFile.splitOffsets()).isNull(); + } + + @Test + public void buildEqualityDeleteFileWithAllFields() { + TrackedFile trackedFile = + TrackedFileBuilder.equalityDelete(50L) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/eq_delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .specId(7) + .partition(PARTITION_DATA) + .contentStats(CONTENT_STATS) + .keyMetadata(KEY_METADATA) + .splitOffsets(SPLIT_OFFSETS) + .sortOrderId(3) + .equalityIds(ImmutableList.of(1, 2)) + .build(); + + assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/eq_delete.parquet"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(trackedFile.recordCount()).isEqualTo(2000L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L); + assertThat(trackedFile.specId()).isEqualTo(7); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS); + assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA); + assertThat(trackedFile.splitOffsets()).isEqualTo(SPLIT_OFFSETS); + assertThat(trackedFile.sortOrderId()).isEqualTo(3); + assertThat(trackedFile.equalityIds()).containsExactly(1, 2); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + + // Unsupported fields for equality delete files + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.manifestInfo()).isNull(); + } + + private static Stream manifestBuilders() { + return Stream.of( + Arguments.of(TrackedFileBuilder.dataManifest(50L), FileContent.DATA_MANIFEST), + Arguments.of(TrackedFileBuilder.deleteManifest(50L), FileContent.DELETE_MANIFEST)); + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void buildManifestWithRequiredFieldsOnly( + TrackedFileBuilder builder, FileContent contentType) { + TrackedFile trackedFile = + builder + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/manifest.avro") + .fileFormat(FileFormat.AVRO) + .recordCount(420L) + .fileSizeInBytes(556L) + .partition(PARTITION_DATA) + .manifestInfo(MANIFEST_INFO) + .build(); + + assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(contentType); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/manifest.avro"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.AVRO); + assertThat(trackedFile.recordCount()).isEqualTo(420L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(556L); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.manifestInfo()).isSameAs(MANIFEST_INFO); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + + assertThat(trackedFile.specId()).isNull(); + assertThat(trackedFile.contentStats()).isNull(); + assertThat(trackedFile.sortOrderId()).isNull(); + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.keyMetadata()).isNull(); + assertThat(trackedFile.splitOffsets()).isNull(); + assertThat(trackedFile.equalityIds()).isNull(); + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void buildManifestWithAllFields(TrackedFileBuilder builder, FileContent contentType) { + TrackedFile trackedFile = + builder + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/manifest.avro") + .fileFormat(FileFormat.AVRO) + .recordCount(420L) + .fileSizeInBytes(556L) + .specId(7) + .partition(PARTITION_DATA) + .contentStats(CONTENT_STATS) + .keyMetadata(KEY_METADATA) + .manifestInfo(MANIFEST_INFO) + .build(); + + assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(contentType); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/manifest.avro"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.AVRO); + assertThat(trackedFile.recordCount()).isEqualTo(420L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(556L); + assertThat(trackedFile.specId()).isEqualTo(7); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS); + assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA); + assertThat(trackedFile.manifestInfo()).isSameAs(MANIFEST_INFO); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + + // Unsupported fields for manifests + assertThat(trackedFile.sortOrderId()).isNull(); + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.splitOffsets()).isNull(); + assertThat(trackedFile.equalityIds()).isNull(); + } + + private static Stream manifestSources() { + return Stream.of( + Arguments.of(sourceDataManifest(10L), FileContent.DATA_MANIFEST), + Arguments.of(sourceDeleteManifest(10L), FileContent.DELETE_MANIFEST)); + } + + @ParameterizedTest + @MethodSource("manifestSources") + public void buildManifestFromSourceWithDeletedPositions( + TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 7L); + + TrackedFile trackedFile = + TrackedFileBuilder.from(source, 20L).deletedPositions(DELETED_POSITIONS).build(); + + assertThat(trackedFile.contentType()).isEqualTo(contentType); + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED); + assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L); + assertThat(trackedFile.tracking().deletedPositions()).isEqualTo(DELETED_POSITIONS); + assertThat(trackedFile.tracking().replacedPositions()).isNull(); + } + + @ParameterizedTest + @MethodSource("manifestSources") + public void buildManifestFromSourceWithReplacedPositions( + TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 7L); + + TrackedFile trackedFile = + TrackedFileBuilder.from(source, 20L).replacedPositions(REPLACED_POSITIONS).build(); + + assertThat(trackedFile.contentType()).isEqualTo(contentType); + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED); + assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L); + assertThat(trackedFile.tracking().deletedPositions()).isNull(); + assertThat(trackedFile.tracking().replacedPositions()).isEqualTo(REPLACED_POSITIONS); + } + + @ParameterizedTest + @MethodSource("manifestSources") + public void buildManifestFromSourceClearsPositions(TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 7L); + + TrackedFile sourceWithPositions = + TrackedFileBuilder.from(source, 15L) + .deletedPositions(DELETED_POSITIONS) + .replacedPositions(REPLACED_POSITIONS) + .build(); + + TrackedFile newEntry = TrackedFileBuilder.from(sourceWithPositions, 20L).build(); + + // Building a new entry from this source should not carry the positions over + assertThat(newEntry.contentType()).isEqualTo(contentType); + assertThat(newEntry.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(newEntry.tracking().deletedPositions()).isNull(); + assertThat(newEntry.tracking().replacedPositions()).isNull(); + assertThat(newEntry.tracking().dvSnapshotId()).isEqualTo(15L); + } + + @Test + public void buildDataFileFromSource() { + TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L); + + TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).build(); + + assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA); + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(source.tracking().snapshotId()); + verifyFieldsAreFromSource(trackedFile, source); + } + + @Test + public void updateDVWhenBuildingDataFileFromSource() { + TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L); + + DeletionVector dv = + DeletionVectorStruct.builder() + .location("s3://bucket/data/new_dv.puffin") + .offset(5L) + .sizeInBytes(256L) + .cardinality(40L) + .build(); + + TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).deletionVector(dv).build(); + + assertThat(trackedFile.deletionVector()).isNotSameAs(source.deletionVector()).isSameAs(dv); + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(10L); + assertThat(trackedFile.tracking().dataSequenceNumber()).isEqualTo(45L); + assertThat(trackedFile.tracking().fileSequenceNumber()).isEqualTo(45L); + assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L); + } + + @Test + public void addingSameDeletionVectorFails() { + TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L); + + DeletionVector dv = + DeletionVectorStruct.builder() + .location("s3://bucket/data/new_dv.puffin") + .offset(5L) + .sizeInBytes(256L) + .cardinality(40L) + .build(); + + DeletionVector dvCopy = dv.copy(); + + TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).deletionVector(dv).build(); + + assertThatThrownBy(() -> TrackedFileBuilder.from(trackedFile, 30L).deletionVector(dv)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("The same deletion vector already added"); + assertThatThrownBy(() -> TrackedFileBuilder.from(trackedFile, 30L).deletionVector(dvCopy)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("The same deletion vector already added"); + } + + private static Stream nonManifestSources() { + return Stream.of( + Arguments.of(sourceData(10L), FileContent.DATA), + Arguments.of(sourceEqualityDelete(10L), FileContent.EQUALITY_DELETES)); + } + + private static Stream allSources() { + return Stream.concat(nonManifestSources(), manifestSources()); + } + + @ParameterizedTest + @MethodSource("allSources") + public void deletedFromSource(TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 15L); + + TrackedFile deleted = TrackedFileBuilder.deleted(source, 20L); + + assertThat(deleted.contentType()).isEqualTo(contentType); + assertThat(deleted.tracking().status()).isEqualTo(EntryStatus.DELETED); + assertThat(deleted.tracking().snapshotId()).isEqualTo(20L); + verifyFieldsAreFromSource(deleted, source); + } + + @ParameterizedTest + @MethodSource("nonManifestSources") + public void replacedFromNonManifestSource(TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 15L); + + TrackedFile replaced = TrackedFileBuilder.replaced(source, 20L); + + assertThat(replaced.contentType()).isEqualTo(contentType); + assertThat(replaced.tracking().status()).isEqualTo(EntryStatus.REPLACED); + assertThat(replaced.tracking().snapshotId()).isEqualTo(20L); + verifyFieldsAreFromSource(replaced, source); + } + + @ParameterizedTest + @MethodSource("manifestSources") + public void replacedFromManifestSourceFails(TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 15L); + + assertThatThrownBy(() -> TrackedFileBuilder.replaced(source, 20L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Manifest entries cannot transition to REPLACED, but entry type is: " + contentType); + } + + private static TrackedFile sourceData(long snapshotId) { + return TrackedFileBuilder.data(snapshotId) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .specId(7) + .contentStats(CONTENT_STATS) + .sortOrderId(3) + .deletionVector(DELETION_VECTOR) + .keyMetadata(KEY_METADATA) + .splitOffsets(SPLIT_OFFSETS) + .build(); + } + + private static TrackedFile sourceEqualityDelete(long snapshotId) { + return TrackedFileBuilder.equalityDelete(snapshotId) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/eq_delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .equalityIds(ImmutableList.of(1)) + .build(); + } + + private static TrackedFile sourceDataManifest(long snapshotId) { + return TrackedFileBuilder.dataManifest(snapshotId) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/data_manifest.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(420L) + .fileSizeInBytes(556L) + .partition(PARTITION_DATA) + .manifestInfo(MANIFEST_INFO) + .build(); + } + + private static TrackedFile sourceDeleteManifest(long snapshotId) { + return TrackedFileBuilder.deleteManifest(snapshotId) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/delete_manifest.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(100L) + .fileSizeInBytes(543L) + .partition(PARTITION_DATA) + .manifestInfo(MANIFEST_INFO) + .build(); + } + + private static TrackedFile entryWithInheritedSeqNums(TrackedFile entry, long sequenceNumber) { + Tracking manifestTrackingToInheritFrom = + new TrackingStruct( + EntryStatus.EXISTING, 123L, sequenceNumber, sequenceNumber, null, null, null, null); + + ((TrackingStruct) entry.tracking()).inheritFrom(manifestTrackingToInheritFrom); + return entry; + } + + /** + * Verifies that fields in entry are the same as in source. Note, snapshot ID can't be verified + * here, because based on the entry's status it is either carried over or not. + */ + private static void verifyFieldsAreFromSource(TrackedFile entry, TrackedFile source) { + assertThat(entry.writerFormatVersion()).isEqualTo(source.writerFormatVersion()); + assertThat(entry.location()).isEqualTo(source.location()); + assertThat(entry.fileFormat()).isEqualTo(source.fileFormat()); + assertThat(entry.recordCount()).isEqualTo(source.recordCount()); + assertThat(entry.fileSizeInBytes()).isEqualTo(source.fileSizeInBytes()); + assertThat(entry.specId()).isEqualTo(source.specId()); + assertThat(entry.partition()).isSameAs(source.partition()); + assertThat(entry.contentStats()).isSameAs(source.contentStats()); + assertThat(entry.sortOrderId()).isEqualTo(source.sortOrderId()); + assertThat(entry.deletionVector()).isSameAs(source.deletionVector()); + assertThat(entry.keyMetadata()).isEqualTo(source.keyMetadata()); + assertThat(entry.splitOffsets()).isEqualTo(source.splitOffsets()); + assertThat(entry.manifestInfo()).isSameAs(source.manifestInfo()); + assertThat(entry.equalityIds()).isEqualTo(source.equalityIds()); + + assertThat(entry.tracking().dataSequenceNumber()) + .isEqualTo(source.tracking().dataSequenceNumber()); + assertThat(entry.tracking().fileSequenceNumber()) + .isEqualTo(source.tracking().fileSequenceNumber()); + assertThat(entry.tracking().dvSnapshotId()).isEqualTo(source.tracking().dvSnapshotId()); + assertThat(entry.tracking().firstRowId()).isEqualTo(source.tracking().firstRowId()); + assertThat(entry.tracking().deletedPositions()).isEqualTo(source.tracking().deletedPositions()); + assertThat(entry.tracking().replacedPositions()) + .isEqualTo(source.tracking().replacedPositions()); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 0b725a39fb6b..4f5452e31455 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -374,10 +374,6 @@ void testKryoSerializationRoundTrip() throws IOException { } static TrackedFileStruct createFullTrackedFile() { - TrackingStruct tracking = (TrackingStruct) TrackingBuilder.added(42L).build(); - tracking.setManifestLocation("s3://bucket/manifest.avro"); - tracking.set(MANIFEST_POS_ORDINAL, 3L); - DeletionVectorStruct dv = DeletionVectorStruct.builder() .location("s3://bucket/dv.puffin") @@ -387,20 +383,24 @@ static TrackedFileStruct createFullTrackedFile() { .build(); TrackedFileStruct file = - new TrackedFileStruct( - tracking, - FileContent.DATA, - WRITER_FORMAT_VERSION_V4, - "s3://bucket/data/file.parquet", - FileFormat.PARQUET, - newPartition(7, "music"), - 100L, - 1024L); - file.set(SPEC_ID_ORDINAL, 0); - file.set(SORT_ORDER_ID_ORDINAL, 1); - file.set(DELETION_VECTOR_ORDINAL, dv); - file.set(KEY_METADATA_ORDINAL, ByteBuffer.wrap(new byte[] {1, 2, 3})); - file.set(SPLIT_OFFSETS_ORDINAL, ImmutableList.of(50L)); + (TrackedFileStruct) + TrackedFileBuilder.data(42L) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .partition(newPartition(7, "music")) + .recordCount(100L) + .fileSizeInBytes(1024L) + .specId(0) + .sortOrderId(1) + .deletionVector(dv) + .keyMetadata(ByteBuffer.wrap(new byte[] {1, 2, 3})) + .splitOffsets(ImmutableList.of(50L)) + .build(); + + TrackingStruct tracking = (TrackingStruct) file.tracking(); + tracking.setManifestLocation("s3://bucket/manifest.avro"); + tracking.set(MANIFEST_POS_ORDINAL, 3L); return file; } @@ -460,19 +460,16 @@ static TrackedFileStruct createTrackedFileWithStats() { .withFieldStats(fieldStatsList) .build(); - TrackedFileStruct file = - new TrackedFileStruct( - null, - FileContent.DATA, - WRITER_FORMAT_VERSION_V4, - "s3://bucket/data/file.parquet", - FileFormat.PARQUET, - new PartitionData(Types.StructType.of()), - 100L, - 1024L); - file.set(SPEC_ID_ORDINAL, 0); - file.set(CONTENT_STATS_ORDINAL, stats); - - return file; + return (TrackedFileStruct) + TrackedFileBuilder.data(0L) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .partition(new PartitionData(Types.StructType.of())) + .recordCount(100L) + .fileSizeInBytes(1024L) + .specId(0) + .contentStats(stats) + .build(); } } From ef07755dd32b59e62346e8599753f75076e57151 Mon Sep 17 00:00:00 2001 From: Eunbin Son <58901024+thswlsqls@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:28:38 +0900 Subject: [PATCH 25/73] ORC: Fix lower/upper bounds for timestamp_ns columns in OrcMetrics (#16922) --- .../apache/iceberg/orc/TestOrcMetrics.java | 35 +++++++++++++++++++ .../org/apache/iceberg/orc/OrcMetrics.java | 20 +++++++---- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java b/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java index 940cf4522291..c1647bb716c5 100644 --- a/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java +++ b/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java @@ -22,6 +22,8 @@ import java.io.File; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.UUID; import org.apache.iceberg.FileFormat; @@ -31,6 +33,7 @@ import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Schema; import org.apache.iceberg.TestMetrics; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.orc.GenericOrcWriter; import org.apache.iceberg.io.FileAppender; @@ -39,7 +42,10 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; /** Test Metrics for ORC. */ @@ -107,6 +113,35 @@ private boolean isBinaryType(Type type) { return BINARY_TYPES.contains(type.typeId()); } + @TestTemplate + public void timestampNanoBoundsKeepNanoPrecision() throws IOException { + Types.TimestampNanoType nanoType = Types.TimestampNanoType.withoutZone(); + Schema schema = new Schema(Types.NestedField.optional(1, "tsNano", nanoType)); + + // sub-microsecond nanos that would change if truncated to micros + LocalDateTime lower = LocalDateTime.parse("1970-01-01T00:00:00.000001500"); + LocalDateTime upper = LocalDateTime.parse("2024-01-02T03:04:05.123456789"); + + GenericRecord lowerRecord = GenericRecord.create(schema); + lowerRecord.set(0, lower); + GenericRecord upperRecord = GenericRecord.create(schema); + upperRecord.set(0, upper); + + Metrics metrics = getMetrics(schema, lowerRecord, upperRecord); + + LocalDateTime epoch = LocalDateTime.parse("1970-01-01T00:00:00"); + long expectedLower = ChronoUnit.NANOS.between(epoch, lower); + long expectedUpper = ChronoUnit.NANOS.between(epoch, upper); + + long actualLower = Conversions.fromByteBuffer(nanoType, metrics.lowerBounds().get(1)); + long actualUpper = Conversions.fromByteBuffer(nanoType, metrics.upperBounds().get(1)); + + assertThat(actualLower).isEqualTo(expectedLower); + assertThat(actualUpper).isEqualTo(expectedUpper); + // guard against the micros regression: the bound must not be ~1000x smaller + assertThat(actualLower).isEqualTo(1500L); + } + @Override protected void assertBounds( int fieldId, Type type, T lowerBound, T upperBound, MetricsWithStats metricsWithStats) { diff --git a/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java b/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java index 867828820d2b..7498d6e1558f 100644 --- a/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java +++ b/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.sql.Timestamp; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; import java.util.Objects; @@ -267,9 +270,7 @@ private static Optional fromOrcMin( TimestampColumnStatistics tColStats = (TimestampColumnStatistics) columnStats; Timestamp minValue = tColStats.getMinimumUTC(); min = - Optional.ofNullable(minValue) - .map(v -> DateTimeUtil.microsFromInstant(v.toInstant())) - .orElse(null); + Optional.ofNullable(minValue).map(v -> timestampBound(type, v.toInstant())).orElse(null); } else if (columnStats instanceof BooleanColumnStatistics) { BooleanColumnStatistics booleanStats = (BooleanColumnStatistics) columnStats; min = booleanStats.getFalseCount() <= 0; @@ -322,9 +323,7 @@ private static Optional fromOrcMax( TimestampColumnStatistics tColStats = (TimestampColumnStatistics) columnStats; Timestamp maxValue = tColStats.getMaximumUTC(); max = - Optional.ofNullable(maxValue) - .map(v -> DateTimeUtil.microsFromInstant(v.toInstant())) - .orElse(null); + Optional.ofNullable(maxValue).map(v -> timestampBound(type, v.toInstant())).orElse(null); } else if (columnStats instanceof BooleanColumnStatistics) { BooleanColumnStatistics booleanStats = (BooleanColumnStatistics) columnStats; max = booleanStats.getTrueCount() > 0; @@ -333,6 +332,15 @@ private static Optional fromOrcMax( Conversions.toByteBuffer(type, truncateIfNeeded(Bound.UPPER, type, max, metricsMode))); } + private static long timestampBound(Type type, Instant instant) { + // timestamp_ns columns store bounds as nanoseconds from epoch, while timestamp columns store + // microseconds (see Conversions for TIMESTAMP and TIMESTAMP_NANO). + if (type.typeId() == Type.TypeID.TIMESTAMP_NANO) { + return ChronoUnit.NANOS.between(DateTimeUtil.EPOCH, instant.atOffset(ZoneOffset.UTC)); + } + return DateTimeUtil.microsFromInstant(instant); + } + private static Object replaceNaN(double value, double replacement) { return Double.isNaN(value) ? replacement : value; } From d7612ae7b776ae55fb28a0852a3777317ca7dfba Mon Sep 17 00:00:00 2001 From: GuoYu <511955993@qq.com> Date: Wed, 24 Jun 2026 00:10:20 +0800 Subject: [PATCH 26/73] Data: Add TCK for Encrypt in FileFormat API (#16724) --- .../iceberg/data/BaseFormatModelTests.java | 111 +++++++++++++++++- 1 file changed, 108 insertions(+), 3 deletions(-) diff --git a/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java b/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java index 8e9eb55940ee..bde37fdca18d 100644 --- a/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java +++ b/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java @@ -60,7 +60,11 @@ import org.apache.iceberg.deletes.PositionDeleteWriter; import org.apache.iceberg.encryption.EncryptedFiles; import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptingFileIO; import org.apache.iceberg.encryption.EncryptionKeyMetadata; +import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.encryption.EncryptionTestHelpers; +import org.apache.iceberg.encryption.NativeEncryptionKeyMetadata; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.ValidationException; @@ -179,6 +183,8 @@ private static List project(List records, Schema targetSchema) { static final String FEATURE_REUSE_CONTAINERS = "reuseContainers"; static final String FEATURE_COLUMN_LEVEL_METRICS = "columnLevelMetrics"; static final String FEATURE_COLUMN_METRICS_TRUNCATE_BINARY = "columnMetricsTruncateBinary"; + static final String FEATURE_NATIVE_ENCRYPTION = "nativeEncryption"; + static final String FEATURE_AES_STREAM_ENCRYPTION = "aesStreamEncryption"; private static final Map MISSING_FEATURES = Map.of( @@ -188,12 +194,19 @@ private static List project(List records, Schema targetSchema) { FEATURE_CASE_SENSITIVE, FEATURE_SPLIT, FEATURE_COLUMN_LEVEL_METRICS, - FEATURE_COLUMN_METRICS_TRUNCATE_BINARY + FEATURE_COLUMN_METRICS_TRUNCATE_BINARY, + FEATURE_NATIVE_ENCRYPTION }, FileFormat.ORC, new String[] { - FEATURE_REUSE_CONTAINERS, FEATURE_COLUMN_METRICS_TRUNCATE_BINARY, FEATURE_READER_DEFAULT - }); + FEATURE_REUSE_CONTAINERS, + FEATURE_COLUMN_METRICS_TRUNCATE_BINARY, + FEATURE_READER_DEFAULT, + FEATURE_AES_STREAM_ENCRYPTION, + FEATURE_NATIVE_ENCRYPTION + }, + FileFormat.PARQUET, + new String[] {FEATURE_AES_STREAM_ENCRYPTION}); private InMemoryFileIO fileIO; private EncryptedOutputFile encryptedFile; @@ -2067,6 +2080,98 @@ void testDataWriterMetaMap(FileFormat fileFormat) throws IOException { }); } + @ParameterizedTest + @FieldSource("FILE_FORMATS") + void testDataWriterAesStreamEncryption(FileFormat fileFormat) throws IOException { + assumeSupports(fileFormat, FEATURE_AES_STREAM_ENCRYPTION); + + EncryptionManager encryptionManager = EncryptionTestHelpers.createEncryptionManager(); + EncryptingFileIO encryptingFileIO = EncryptingFileIO.combine(fileIO, encryptionManager); + EncryptedOutputFile encryptedOutputFile = + encryptingFileIO.newEncryptingOutputFile("test-file-" + UUID.randomUUID()); + + FileWriterBuilder, ?> writerBuilder = + FormatModelRegistry.dataWriteBuilder(fileFormat, engineType(), encryptedOutputFile) + .keyMetadata(encryptedOutputFile.keyMetadata()); + + writeAndAssertEncryptedDataWriter(fileFormat, encryptingFileIO, writerBuilder); + } + + @ParameterizedTest + @FieldSource("FILE_FORMATS") + void testDataWriterNativeEncryption(FileFormat fileFormat) throws IOException { + assumeSupports(fileFormat, FEATURE_NATIVE_ENCRYPTION); + + EncryptionManager encryptionManager = EncryptionTestHelpers.createEncryptionManager(); + EncryptingFileIO encryptingFileIO = EncryptingFileIO.combine(fileIO, encryptionManager); + String location = "test-file-" + UUID.randomUUID(); + NativeEncryptionKeyMetadata keyMetadata = + (NativeEncryptionKeyMetadata) + encryptingFileIO.newEncryptingOutputFile(location).keyMetadata(); + + // Use a plain encrypted output file so Parquet cannot auto-inject the native encryption key + // and AAD prefix from the output file metadata. This ensures encryption is driven only by + // withFileEncryptionKey and withAADPrefix below. + EncryptedOutputFile encryptedOutputFile = + EncryptedFiles.plainAsEncryptedOutput(fileIO.newOutputFile(location)); + + // keyMetadata is mainly used for parsing during reading, so this call is required here. + FileWriterBuilder, ?> writerBuilder = + FormatModelRegistry.dataWriteBuilder(fileFormat, engineType(), encryptedOutputFile) + .keyMetadata(keyMetadata) + .withFileEncryptionKey(keyMetadata.encryptionKey().duplicate()) + .withAADPrefix(keyMetadata.aadPrefix().duplicate()); + + writeAndAssertEncryptedDataWriter(fileFormat, encryptingFileIO, writerBuilder); + } + + @SuppressWarnings("checkstyle:AssertThatThrownByWithMessageCheck") + private void writeAndAssertEncryptedDataWriter( + FileFormat fileFormat, + EncryptingFileIO encryptingFileIO, + FileWriterBuilder, ?> writerBuilder) + throws IOException { + DataGenerator dataGenerator = new DataGenerators.DefaultSchema(); + Schema schema = dataGenerator.schema(); + List genericRecords = dataGenerator.generateRecords(); + List engineRecords = convertToEngineRecords(genericRecords, schema); + + DataWriter writer = writerBuilder.schema(schema).spec(PartitionSpec.unpartitioned()).build(); + + try (writer) { + engineRecords.forEach(writer::write); + } + + DataFile dataFile = writer.toDataFile(); + assertThat(dataFile).isNotNull(); + assertThat(dataFile.recordCount()).isEqualTo(engineRecords.size()); + assertThat(dataFile.format()).isEqualTo(fileFormat); + assertThat(dataFile.keyMetadata()).isNotNull(); + + assertThatThrownBy( + () -> + readAndAssertGenericRecords( + fileFormat, schema, genericRecords, fileIO.newInputFile(dataFile.location()))) + .isInstanceOf(RuntimeException.class); + + readAndAssertGenericRecords( + fileFormat, schema, genericRecords, encryptingFileIO.newInputFile(dataFile)); + } + + private void readAndAssertGenericRecords( + FileFormat fileFormat, Schema schema, List expected, InputFile inputFile) + throws IOException { + List readRecords; + try (CloseableIterable reader = + FormatModelRegistry.readBuilder(fileFormat, Record.class, inputFile) + .project(schema) + .build()) { + readRecords = ImmutableList.copyOf(reader); + } + + DataTestHelpers.assertEquals(schema.asStruct(), expected, readRecords); + } + private void readAndAssertGenericRecords( FileFormat fileFormat, Schema schema, List expected) throws IOException { InputFile inputFile = encryptedFile.encryptingOutputFile().toInputFile(); From df6ea0820030d217d5f645469419cdbb3f54a41f Mon Sep 17 00:00:00 2001 From: Aleksandr Efimov Date: Tue, 23 Jun 2026 19:24:38 +0300 Subject: [PATCH 27/73] Data: Add metrics reporter to generic scan builder (#16664) --- .../apache/iceberg/data/IcebergGenerics.java | 6 ++++++ .../apache/iceberg/data/TestLocalScan.java | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/data/src/main/java/org/apache/iceberg/data/IcebergGenerics.java b/data/src/main/java/org/apache/iceberg/data/IcebergGenerics.java index 352658752d98..7bf8f5c7780b 100644 --- a/data/src/main/java/org/apache/iceberg/data/IcebergGenerics.java +++ b/data/src/main/java/org/apache/iceberg/data/IcebergGenerics.java @@ -24,6 +24,7 @@ import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.metrics.MetricsReporter; public class IcebergGenerics { private IcebergGenerics() {} @@ -96,6 +97,11 @@ public ScanBuilder appendsAfter(long fromSnapshotId) { return this; } + public ScanBuilder metricsReporter(MetricsReporter reporter) { + this.tableScan = tableScan.metricsReporter(reporter); + return this; + } + public CloseableIterable build() { return new TableScanIterable(tableScan, reuseContainers); } diff --git a/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java b/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java index 0f85596afbdc..aee5eb5fa76c 100644 --- a/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java +++ b/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java @@ -58,7 +58,10 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopInputFile; import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.DataWriter; +import org.apache.iceberg.metrics.InMemoryMetricsReporter; +import org.apache.iceberg.metrics.ScanReport; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -289,6 +292,22 @@ public void testFullScan() { assertThat(records).as("Random record set should match").isEqualTo(expected); } + @TestTemplate + public void testScanMetricsReporter() throws IOException { + InMemoryMetricsReporter reporter = new InMemoryMetricsReporter(); + + try (CloseableIterable results = + IcebergGenerics.read(sharedTable).metricsReporter(reporter).build()) { + assertThat(Sets.newHashSet(results)).hasSize(9); + } + + ScanReport scanReport = reporter.scanReport(); + assertThat(scanReport).isNotNull(); + assertThat(scanReport.tableName()).isEqualTo(sharedTable.name()); + assertThat(scanReport.snapshotId()).isEqualTo(sharedTable.currentSnapshot().snapshotId()); + assertThat(scanReport.scanMetrics().resultDataFiles().value()).isEqualTo(3); + } + @TestTemplate public void testFilter() { Iterable result = IcebergGenerics.read(sharedTable).where(lessThan("id", 3)).build(); From f07b404b060d2bdf06799161de6c12aa43235e11 Mon Sep 17 00:00:00 2001 From: Xin Huang <42597328+huan233usc@users.noreply.github.com> Date: Tue, 23 Jun 2026 17:01:21 -0700 Subject: [PATCH 28/73] Parquet: Skip geo footer bounds (#16850) --- .../iceberg/parquet/ParquetMetrics.java | 9 +++- .../apache/iceberg/parquet/TestParquet.java | 46 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java index 57bfc4295f12..42950bc073ec 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java @@ -43,6 +43,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.BinaryUtil; import org.apache.iceberg.util.NaNUtil; @@ -264,13 +265,19 @@ private FieldMetrics metricsFromFooter( int truncateLength) { if (primitive.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) { return null; - } else if (truncateLength <= 0) { + } else if (truncateLength <= 0 + || (icebergType != null && isGeospatial(icebergType.typeId()))) { + // Parquet lexicographic min/max is not meaningful for spatial WKB. return counts(fieldId); } else { return bounds(fieldId, icebergType, primitive, truncateLength); } } + private static boolean isGeospatial(TypeID typeId) { + return typeId == TypeID.GEOMETRY || typeId == TypeID.GEOGRAPHY; + } + private FieldMetrics counts(int fieldId) { ColumnPath path = ColumnPath.get(currentPath()); long valueCount = 0; diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java index afabd2c6b1bd..c1aabab0b251 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java @@ -284,6 +284,52 @@ public void testColumnStatisticsEnabled() throws Exception { } } + @Test + public void testGeospatialFooterMetricsSkipParquetBounds() throws IOException { + Schema binarySchema = new Schema(optional(1, "geom", Types.BinaryType.get())); + Schema geometrySchema = new Schema(optional(1, "geom", Types.GeometryType.crs84())); + Schema geographySchema = new Schema(optional(1, "geom", Types.GeographyType.crs84())); + + File file = createTempFile(temp); + org.apache.avro.Schema avroSchema = AvroSchemaUtil.convert(binarySchema.asStruct()); + GenericData.Record first = new GenericData.Record(avroSchema); + first.put("geom", ByteBuffer.wrap(new byte[] {0x01, 0x02, 0x03})); + GenericData.Record second = new GenericData.Record(avroSchema); + second.put("geom", ByteBuffer.wrap(new byte[] {0x04, 0x05, 0x06})); + + write( + file, binarySchema, Collections.emptyMap(), ParquetAvroWriter::buildWriter, first, second); + + InputFile inputFile = Files.localInput(file); + try (ParquetFileReader reader = ParquetFileReader.open(ParquetIO.file(inputFile))) { + Metrics geometryMetrics = + ParquetMetrics.metrics( + geometrySchema, + reader.getFooter().getFileMetaData().getSchema(), + MetricsConfig.getDefault(), + reader.getFooter(), + Stream.empty()); + + Metrics geographyMetrics = + ParquetMetrics.metrics( + geographySchema, + reader.getFooter().getFileMetaData().getSchema(), + MetricsConfig.getDefault(), + reader.getFooter(), + Stream.empty()); + + assertThat(geometryMetrics.valueCounts()).containsEntry(1, 2L); + assertThat(geometryMetrics.nullValueCounts()).containsEntry(1, 0L); + assertThat(geometryMetrics.lowerBounds()).doesNotContainKey(1); + assertThat(geometryMetrics.upperBounds()).doesNotContainKey(1); + + assertThat(geographyMetrics.valueCounts()).containsEntry(1, 2L); + assertThat(geographyMetrics.nullValueCounts()).containsEntry(1, 0L); + assertThat(geographyMetrics.lowerBounds()).doesNotContainKey(1); + assertThat(geographyMetrics.upperBounds()).doesNotContainKey(1); + } + } + @Test public void testPerColumnDictionaryEncoding() throws Exception { Schema schema = From 47148f380eaf3e806ec0cc628b9661a8f57a8fc3 Mon Sep 17 00:00:00 2001 From: Wyatt H <129568037+Wyatt-Hawes@users.noreply.github.com> Date: Tue, 23 Jun 2026 17:43:42 -0700 Subject: [PATCH 29/73] AWS: Handle duplicate column names in IcebergToGlueConverter comment map (#16853) * AWS: Tolerate duplicate column names in IcebergToGlueConverter comment map ## What changed `IcebergToGlueConverter.setTableInputInformation(TableInput.Builder, TableMetadata, Table)` builds an `existingColumnMap` (column name -> comment) from the existing Glue table's columns to recover comments for fields whose new Iceberg schema has no `doc()`. The current `Collectors.toMap(Column::name, Column::comment)` throws `IllegalStateException` on duplicate keys. This change adds a first-seen-wins merge function so duplicate keys collapse instead of throwing. ## Why Duplicate `Column.name` entries can legitimately appear in an existing Glue table's `Columns[]` after a case-only `RENAME COLUMN` cycle (e.g. `id -> id__tmp -> Id`): 1. `RENAME COLUMN` only changes the field name; `field.doc()` is unchanged. 2. The converter walks `metadata.schemas()` in `toColumns` and emits historical names alongside the current name; case-sensitive dedupe lets `Id` and `id` through as distinct entries with the same comment. 3. Glue normalizes `Column.name` case-insensitively on persist, collapsing `Id` and `id` into two `name="id"` rows that both carry a non-null comment. 4. Next commit: the stream above hits `Collectors.toMap` -> duplicate-key throw. 5. The throw is swallowed by the outer `catch (RuntimeException e) { LOG.warn(...) }`, leaving `tableInputBuilder.storageDescriptor` unset. 6. `glue.updateTable()` is then called with `TableInput.storageDescriptor=null`, producing a destructive write -- every subsequent read sees a null storage descriptor. The case-collision alone is not the bug; without comments the `column.comment() != null` filter strips the colliding entries and the throw never fires. The failure surfaces only when both colliding columns also have non-null comments, because that's what the comment-recovery map keys on. ## Fix First-seen-wins is the right semantic here: `toColumns` writes current-schema entries before historical ones, so the current entry's comment is preserved on the read-back. The converter's emit behavior is unchanged -- Glue still receives duplicate-name `Columns[]` after a case-only rename, so time-travel and Hive-fallback consumers depending on historical names appearing in `Columns[]` are unaffected. `addColumnWithDedupe` is also untouched. A case-insensitive write-side dedupe was considered and rejected -- it would drop historical names from `Columns[]` and change the converter's documented projection. ## Properties of this fix - Resolves the `IllegalStateException` and the resulting null-storage-descriptor `UpdateTable` calls. - Heals already-broken tables on the next commit -- no manual dedupe required. - No change to the converter's emit behavior. - `addColumnWithDedupe` is untouched. ## Test Adds `TestIcebergToGlueConverter.testSetTableInputInformationWithDuplicateExistingColumnComments`, which constructs an `existingTable` with the broken `Columns[]` shape (two `name="id"` entries, both with comments) and asserts that `setTableInputInformation` produces a non-null `StorageDescriptor` with the correct first-seen comment. Pre-fix, the test fails on a null `storageDescriptor()` (the destructive bug surfaced in-process); post-fix it passes. All 13 tests in `TestIcebergToGlueConverter` pass; spotless clean. * AWS: Rename existingColumnMap to existingColumnNameToComment to match variable contents. --- .../aws/glue/IcebergToGlueConverter.java | 28 ++++++---- .../aws/glue/TestIcebergToGlueConverter.java | 53 +++++++++++++++++++ 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java index 56b38a47e968..2363dc4e2a87 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java +++ b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java @@ -263,18 +263,22 @@ static void setTableInputInformation( Optional.ofNullable(existingTable.description()).ifPresent(tableInputBuilder::description); } - Map existingColumnMap = null; + Map existingColumnNameToComment = null; if (existingTable != null) { List existingColumns = existingTable.storageDescriptor().columns(); - existingColumnMap = + // First-seen-wins on duplicate names. toColumns writes current-schema columns before + // historical-schema columns. Preserve the current-schema comment. + existingColumnNameToComment = existingColumns.stream() .filter(column -> column.comment() != null) - .collect(Collectors.toMap(Column::name, Column::comment)); + .collect( + Collectors.toMap( + Column::name, Column::comment, (existing, duplicate) -> existing)); } else { - existingColumnMap = Collections.emptyMap(); + existingColumnNameToComment = Collections.emptyMap(); } - List columns = toColumns(metadata, existingColumnMap); + List columns = toColumns(metadata, existingColumnNameToComment); tableInputBuilder.storageDescriptor( storageDescriptor.location(metadata.location()).columns(columns).build()); @@ -340,19 +344,20 @@ private static String toTypeString(Type type) { } private static List toColumns( - TableMetadata metadata, Map existingColumnMap) { + TableMetadata metadata, Map existingColumnNameToComment) { List columns = Lists.newArrayList(); Set addedNames = Sets.newHashSet(); for (NestedField field : metadata.schema().columns()) { - addColumnWithDedupe(columns, addedNames, field, true /* is current */, existingColumnMap); + addColumnWithDedupe( + columns, addedNames, field, true /* is current */, existingColumnNameToComment); } for (Schema schema : metadata.schemas()) { if (schema.schemaId() != metadata.currentSchemaId()) { for (NestedField field : schema.columns()) { addColumnWithDedupe( - columns, addedNames, field, false /* is not current */, existingColumnMap); + columns, addedNames, field, false /* is not current */, existingColumnNameToComment); } } } @@ -365,7 +370,7 @@ private static void addColumnWithDedupe( Set dedupe, NestedField field, boolean isCurrent, - Map existingColumnMap) { + Map existingColumnNameToComment) { if (!dedupe.contains(field.name())) { Column.Builder builder = Column.builder() @@ -379,8 +384,9 @@ private static void addColumnWithDedupe( if (field.doc() != null && !field.doc().isEmpty()) { builder.comment(field.doc()); - } else if (existingColumnMap != null && existingColumnMap.containsKey(field.name())) { - builder.comment(existingColumnMap.get(field.name())); + } else if (existingColumnNameToComment != null + && existingColumnNameToComment.containsKey(field.name())) { + builder.comment(existingColumnNameToComment.get(field.name())); } columns.add(builder.build()); diff --git a/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java b/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java index edebfd3420e2..a797ac7b2ffa 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java +++ b/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java @@ -384,4 +384,57 @@ public void testSetTableInputInformationWithExistingTable() { .as("Columns should match") .isEqualTo(expectedTableInput.storageDescriptor().columns()); } + + @Test + public void testDuplicateExistingColumnsKeepFirstComment() { + TableInput.Builder actualTableInputBuilder = TableInput.builder(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.StringType.get())); + PartitionSpec partitionSpec = + PartitionSpec.builderFor(schema).identity("id").withSpecId(1000).build(); + TableMetadata tableMetadata = + TableMetadata.newTableMetadata(schema, partitionSpec, "s3://test", tableLocationProperties); + + Table existingGlueTable = + Table.builder() + .storageDescriptor( + StorageDescriptor.builder() + .columns( + ImmutableList.of( + Column.builder().name("id").comment("current comment").build(), + Column.builder().name("id").comment("historical comment").build())) + .build()) + .build(); + + IcebergToGlueConverter.setTableInputInformation( + actualTableInputBuilder, tableMetadata, existingGlueTable); + TableInput actualTableInput = actualTableInputBuilder.build(); + + TableInput expectedTableInput = + TableInput.builder() + .storageDescriptor( + StorageDescriptor.builder() + .location("s3://test") + .additionalLocations(Sets.newHashSet(tableLocationProperties.values())) + .columns( + ImmutableList.of( + Column.builder() + .name("id") + .type("string") + .comment("current comment") + .parameters( + ImmutableMap.of( + IcebergToGlueConverter.ICEBERG_FIELD_ID, "1", + IcebergToGlueConverter.ICEBERG_FIELD_OPTIONAL, "false", + IcebergToGlueConverter.ICEBERG_FIELD_CURRENT, "true")) + .build())) + .build()) + .build(); + + assertThat(actualTableInput.storageDescriptor()) + .as("Storage descriptor should be set") + .isNotNull(); + assertThat(actualTableInput.storageDescriptor().columns()) + .as("Columns should match") + .isEqualTo(expectedTableInput.storageDescriptor().columns()); + } } From 6bef83a6bc24a2ba4af12329395d5162162c1df5 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Wed, 24 Jun 2026 06:49:30 +0200 Subject: [PATCH 30/73] Flink: Add equality delete conversion planner (#16889) * Flink: Add equality delete conversion planner This adds EqualityConvertPlanner, the operator that drives the equality delete conversion pipeline. For every trigger it picks the next unconverted staging snapshot and emits the read commands and plan metadata that drive the rest of the pipeline. Equality-delete resolution is only correct against an index that reflects the current target branch, so each trigger reconciles the worker index before resolving. It builds the index on the first run and rebuilds it when external commits (compaction, concurrent writes) have moved the branch, evicting entries the rebuild won't overwrite. Read commands are split into watermark-separated phases because the readers run in parallel: without an explicit ordering the worker could resolve a delete against a stale or half-built index. Idempotency is ensured by the planner, not the committer. It recognizes already-converted snapshots from the committer's marker on the target branch, so a restart re-derives its position and never reprocesses a committed snapshot. The planner fails the cycle on inputs it cannot convert correctly rather than dropping them silently: data-file rewrites, V2 positional deletes, and equality deletes whose field IDs don't match the configured set. When there is nothing to convert it still emits a no-op plan so the maintenance task run completes. * fixup! Update comments and add test for rolled back table * fixup! Re-trigger CI to run checks skipped during github.com outage * fixup! Verify planner conversion metrics in tests * fixup! Assert timestamps and watermarks --- .../operator/EqualityConvertPlanner.java | 762 +++++++++++ .../operator/TestEqualityConvertPlanner.java | 1115 +++++++++++++++++ 2 files changed, 1877 insertions(+) create mode 100644 flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java create mode 100644 flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java new file mode 100644 index 000000000000..c3c1785290cf --- /dev/null +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -0,0 +1,762 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.OutputTag; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotChanges; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Planner for the equality delete conversion pipeline. For each trigger, it picks the oldest + * staging snapshot that hasn't been converted yet and emits {@link ReadCommand}s describing the + * files its downstream readers and workers must process. + * + *

Each trigger runs two steps in order: + * + *

    + *
  1. {@link #ensureIndexCurrent}: updates {@link #lastStagingSnapshotId} from main's history, + * bootstraps the worker index from main on first run, and reindexes when external commits + * (e.g. compaction) have advanced main past the currently-indexed snapshot. + *
  2. {@link #processStagingSnapshot}: resolve the chosen staging snapshot's eq deletes against + * the (now-current) index, pass through any DV files, and index the snapshot's new data files + * for the next cycle. + *
+ * + * Watermarks separate phases that gate the worker's keyed state. The contract is documented on + * {@link #advancePhase()}. + * + *

An {@link EqualityConvertPlan} with the current cycle's metadata is emitted via the {@link + * #METADATA_STREAM} side output after the read commands. + * + *

Assumes a single equality-field set supplied via the builder; staging eq-deletes with a + * different {@code equalityFieldIds} fail fast in {@link #retrieveStagingFiles}. Concurrent writes + * on the target branch are handled by {@link #ensureIndexCurrent} reindexing from the new main + * snapshot; commit-time conflicts are caught by {@code RowDelta.validateFromSnapshot}. + */ +@Internal +public class EqualityConvertPlanner extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertPlanner.class); + + public static final OutputTag METADATA_STREAM = + new OutputTag<>("metadata-stream") {}; + + public static final OutputTag CLEAR_BROADCAST_STREAM = + new OutputTag<>("clear-broadcast-stream") {}; + + private static final String PROCESSED_EQ_DELETE_FILE_NUM_METRIC = "processedEqDeleteFileNum"; + private static final String PROCESSED_STAGING_SNAPSHOT_NUM_METRIC = "processedStagingSnapshotNum"; + private static final String SKIPPED_NO_OP_CYCLES_METRIC = "skippedNoOpCycles"; + private static final String REINDEX_COUNT_METRIC = "reindexCount"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String stagingBranch; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + // Equality-field-id set the worker keys on. Supplied via the builder; every staging + // eq-delete's equalityFieldIds() must match exactly. + private final Set eqFieldIds; + + // Main snapshot id the worker's index reflects. + private transient ListState indexSnapshotState; + // Main sequence number the worker's index reflects. + private transient ListState indexedSequenceNumberState; + // Equality field IDs the index was built with, allows to detect reconfiguration. + private transient ListState eqFieldIdsState; + + private transient Table table; + + private transient Long lastMainSnapshotId; + private transient Long lastStagingSnapshotId; + private transient Long indexSnapshotId; + private transient Long indexedSequenceNumber; + + private transient long nextPhaseTs; + + private transient Counter processedEqDeleteFileNumCounter; + private transient Counter processedStagingSnapshotNumCounter; + private transient Counter skippedNoOpCyclesCounter; + private transient Counter reindexCounter; + private transient Counter errorCounter; + + public EqualityConvertPlanner( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch, + Set eqFieldIds) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.stagingBranch = stagingBranch; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + Preconditions.checkArgument( + eqFieldIds != null && !eqFieldIds.isEmpty(), "eqFieldIds must not be null or empty"); + this.eqFieldIds = ImmutableSet.copyOf(eqFieldIds); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.processedEqDeleteFileNumCounter = + taskMetricGroup.counter(PROCESSED_EQ_DELETE_FILE_NUM_METRIC); + this.processedStagingSnapshotNumCounter = + taskMetricGroup.counter(PROCESSED_STAGING_SNAPSHOT_NUM_METRIC); + this.skippedNoOpCyclesCounter = taskMetricGroup.counter(SKIPPED_NO_OP_CYCLES_METRIC); + this.reindexCounter = taskMetricGroup.counter(REINDEX_COUNT_METRIC); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + indexSnapshotState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexSnapshotId", Types.LONG)); + + indexSnapshotId = null; + for (Long stateValue : indexSnapshotState.get()) { + Preconditions.checkState( + indexSnapshotId == null, "indexSnapshotId state should hold at most one value"); + indexSnapshotId = stateValue; + } + + indexedSequenceNumberState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexedSequenceNumber", Types.LONG)); + + indexedSequenceNumber = null; + for (Long stateValue : indexedSequenceNumberState.get()) { + Preconditions.checkState( + indexedSequenceNumber == null, + "indexedSequenceNumber state should hold at most one value"); + indexedSequenceNumber = stateValue; + } + + eqFieldIdsState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("eqFieldIds", Types.INT)); + Set restoredEqFieldIds = Sets.newHashSet(eqFieldIdsState.get()); + Preconditions.checkState( + restoredEqFieldIds.isEmpty() || restoredEqFieldIds.equals(eqFieldIds), + "Equality field IDs changed across restart: restored=%s, configured=%s. " + + "Reconfiguring equality-field columns is not supported; " + + "restart from a clean state (no savepoint).", + restoredEqFieldIds, + eqFieldIds); + } + + @Override + public void snapshotState(StateSnapshotContext context) throws Exception { + super.snapshotState(context); + indexSnapshotState.clear(); + if (indexSnapshotId != null) { + indexSnapshotState.add(indexSnapshotId); + } + + indexedSequenceNumberState.clear(); + if (indexedSequenceNumber != null) { + indexedSequenceNumberState.add(indexedSequenceNumber); + } + + eqFieldIdsState.clear(); + for (int id : eqFieldIds) { + eqFieldIdsState.add(id); + } + } + + @Override + public void processElement(StreamRecord element) throws Exception { + long triggerTs = element.getTimestamp(); + nextPhaseTs = Math.max(triggerTs, nextPhaseTs + 1); + + Long currentMainSnapshotId = lastMainSnapshotId; + try { + table.refresh(); + Snapshot mainSnapshot = table.snapshot(targetBranch); + currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + ensureIndexCurrent(mainSnapshot); + + Snapshot nextToProcess = + nextUnprocessedStagingSnapshot(table.snapshot(stagingBranch), mainSnapshot); + + if (nextToProcess == null) { + LOG.info("Nothing new to convert on staging branch '{}'.", stagingBranch); + emitNoOpResult(triggerTs, currentMainSnapshotId); + return; + } + + processStagingSnapshot(nextToProcess, triggerTs, currentMainSnapshotId); + } catch (Exception e) { + LOG.error("Error processing equality deletes for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + emitDrainResult(triggerTs, currentMainSnapshotId); + } + } + + /** + * Brings the worker's index up to date with the current state of the target branch: + * + *

    + *
  • Updates {@link #lastStagingSnapshotId} from the most recent committer marker on main. + *
  • Bootstraps the index from main on the first trigger with a non-null main snapshot. + *
  • Reindexes from main when external commits (e.g. compaction or direct writes) have + * advanced main past the currently-indexed snapshot. + *
+ * + *

No-op when main hasn't moved since the last trigger. Otherwise the history walk is bounded + * to commits added since {@link #lastMainSnapshotId}. + */ + private void ensureIndexCurrent(Snapshot mainSnapshot) { + Long currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + if (Objects.equals(lastMainSnapshotId, currentMainSnapshotId)) { + return; + } + + LastCommittedWork info = discoverLastCommittedWork(mainSnapshot); + updateLastStagingSnapshotId(info); + + boolean bootstrap = mainSnapshot != null && indexSnapshotId == null; + boolean reindex = indexSnapshotId != null && info.externalCommitCount() > 0; + if (bootstrap || reindex) { + LOG.info( + "{} worker index from main snapshot {} for field IDs {}.", + bootstrap ? "Bootstrapping" : "Reindexing", + currentMainSnapshotId, + eqFieldIds); + if (reindex) { + // Evict keyed entries the reindex will not re-add (e.g. data file removed by CoW). + output.collect( + CLEAR_BROADCAST_STREAM, + new StreamRecord<>( + IndexCommand.clearBeforeReindex( + currentMainSnapshotId, mainSnapshot.sequenceNumber()))); + reindexCounter.inc(); + } + + indexSnapshotId = currentMainSnapshotId; + indexedSequenceNumber = mainSnapshot.sequenceNumber(); + emitMainDataReadCommands(mainSnapshot); + } + + lastMainSnapshotId = currentMainSnapshotId; + } + + private void updateLastStagingSnapshotId(LastCommittedWork info) { + if (info.lastCommittedStaging() != null) { + lastStagingSnapshotId = info.lastCommittedStaging(); + return; + } + + Preconditions.checkState( + lastMainSnapshotId == null || lastStagingSnapshotId == null || info.reachedLastInspected(), + "No COMMITTED_STAGING_SNAPSHOT marker reachable on target branch '%s' for table %s, " + + "but a prior marker was seen (lastStagingSnapshotId=%s). Target may have been " + + "rewritten (rollback, replace_main, or snapshot expiration). " + + "Manual intervention required.", + targetBranch, + tableName, + lastStagingSnapshotId); + } + + /** + * Walks main back from head looking for the most recent snapshot tagged with {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}. Returns the staging snapshot id + * recorded there (or {@code null} if not reached) and the count of intervening external commits. + * + *

The walk stops at whichever comes first: + * + *

    + *
  • The first snapshot carrying the committer marker. + *
  • {@link #lastMainSnapshotId} — anything older was inspected on a previous trigger. + *
+ */ + private LastCommittedWork discoverLastCommittedWork(Snapshot mainSnapshot) { + Long lastCommittedStaging = null; + int externalCount = 0; + boolean reachedLastInspected = false; + Snapshot current = mainSnapshot; + while (current != null) { + if (lastMainSnapshotId != null && current.snapshotId() == lastMainSnapshotId) { + reachedLastInspected = true; + break; + } + + String prop = + current.summary().get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY); + if (prop != null) { + lastCommittedStaging = Long.parseLong(prop); + break; + } + + externalCount++; + current = parentOf(current); + } + + return new LastCommittedWork(lastCommittedStaging, externalCount, reachedLastInspected); + } + + private record LastCommittedWork( + Long lastCommittedStaging, int externalCommitCount, boolean reachedLastInspected) {} + + /** + * Walks staging history from head back to the stop point and returns the oldest unprocessed + * snapshot to convert this cycle, or {@code null} if there's nothing new. + * + *

The stop point is: + * + *

    + *
  • the last-processed staging snapshot, if known; + *
  • otherwise, on cold start with {@code stagingBranch != targetBranch}, the common ancestor + * with target; + *
  • otherwise, on cold start with {@code stagingBranch == targetBranch}, the root of history + * ({@code null}). {@link #shouldSkip} filters already-converted snapshots in O(1) via the + * {@code COMMITTED_STAGING_SNAPSHOT_PROPERTY} marker, and pure-insert snapshots via the + * same-branch eq-delete predicate. + *
+ * + *

When {@code stagingBranch == targetBranch}, the writer commits eq-deletes and data files + * directly to target; the converter performs in-place eq-delete-to-DV compaction. On cold start + * we walk the full history so eq-deletes committed before the converter started are still picked + * up. + */ + private Snapshot nextUnprocessedStagingSnapshot(Snapshot stagingHead, Snapshot mainSnapshot) { + if (stagingHead == null) { + return null; + } + + Long stopAt; + if (lastStagingSnapshotId != null) { + stopAt = lastStagingSnapshotId; + } else if (stagingOnTargetBranch) { + stopAt = null; + } else { + stopAt = findCommonAncestor(stagingHead, mainSnapshot); + } + + Snapshot current = stagingHead; + Snapshot oldestUnprocessed = null; + while (current != null) { + if (stopAt != null && current.snapshotId() == stopAt) { + break; + } + + if (!shouldSkip(current)) { + oldestUnprocessed = current; + } + + current = parentOf(current); + } + + return oldestUnprocessed; + } + + /** + * Resolves the eq deletes in {@code stagingSnapshot} against the current index and emits the + * cycle's metadata. Phase ordering (separated by watermarks): + * + *

    + *
  1. Eq delete read commands. Eq deletes resolve in the worker. + *
  2. Staging data files. Indexed for the NEXT cycle's eq-delete resolution. + *
+ * + * Cold-start bootstrap of the index from main data is handled separately in {@link + * #processElement}, which runs once before the first cycle. + */ + private void processStagingSnapshot( + Snapshot stagingSnapshot, long triggerTs, Long currentMainSnapshotId) { + + StagingInputs inputs = retrieveStagingFiles(stagingSnapshot); + Preconditions.checkState( + !inputs.isEmpty(), + "Staging snapshot %s has no convertible inputs; shouldSkip should have filtered it.", + stagingSnapshot.snapshotId()); + + emitDeletePhase(inputs.eqDeleteFiles()); + emitSnapshotDataPhase(inputs.newDataFiles()); + + LOG.info( + "Emitted read commands for {} new data files from staging branch '{}'.", + inputs.newDataFiles().size(), + stagingBranch); + + processedStagingSnapshotNumCounter.inc(); + + output.collect( + METADATA_STREAM, + new StreamRecord<>( + new EqualityConvertPlan( + inputs.newDataFiles(), + inputs.stagingDVFiles(), + stagingSnapshot.snapshotId(), + currentMainSnapshotId, + triggerTs, + nextPhaseTs))); + + advancePhase(); + } + + /** + * Classifies the files added by {@code stagingSnapshot} into data files, eq delete files, and DV + * files. Throws if the snapshot: + * + *
    + *
  • Removes data files (rewrites on the staging branch aren't supported). + *
  • Contains V2 positional delete files (the converter expects a V3 staging branch written by + * Flink, which produces only deletion vectors for deletes). + *
  • Contains an eq-delete file whose {@code equalityFieldIds()} doesn't match the + * builder-configured set (silent wrong-key serialization otherwise). + *
+ */ + private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { + SnapshotChanges changes = SnapshotChanges.builderFor(table).snapshot(stagingSnapshot).build(); + + // Rewrites on the staging branch would require rewriting the corresponding DVs against new + // data files on target. Not implemented; fail fast instead of silently dropping work. + if (changes.removedDataFiles().iterator().hasNext()) { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' removes data files; " + + "equality delete conversion does not support rewrites on the staging branch. " + + "Run compaction on the target branch instead.", + stagingSnapshot.snapshotId(), stagingBranch)); + } + + List newDataFiles = Lists.newArrayList(); + List stagingDVFiles = Lists.newArrayList(); + List eqDeleteFiles = Lists.newArrayList(); + + for (DataFile dataFile : changes.addedDataFiles()) { + newDataFiles.add(dataFile); + } + + for (DeleteFile deleteFile : changes.addedDeleteFiles()) { + if (deleteFile.content() == FileContent.EQUALITY_DELETES) { + Set deleteFieldIds = Sets.newHashSet(deleteFile.equalityFieldIds()); + Preconditions.checkState( + deleteFieldIds.equals(eqFieldIds), + "Staging snapshot %s on branch '%s' contains an equality delete file %s with " + + "equalityFieldIds=%s, which does not match the configured eqFieldIds=%s. " + + "The writer must use the same equality field IDs as the converter.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + deleteFieldIds, + eqFieldIds); + eqDeleteFiles.add(deleteFile); + } else if (ContentFileUtil.isDV(deleteFile)) { + stagingDVFiles.add(deleteFile); + } else { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' contains a V2 positional delete file (%s); " + + "equality delete conversion expects a V3 staging branch written by Flink, " + + "which produces only deletion vectors for deletes.", + stagingSnapshot.snapshotId(), stagingBranch, deleteFile.location())); + } + } + + return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); + } + + /** Files added by one staging snapshot, classified for cycle emission. */ + private record StagingInputs( + List newDataFiles, + List stagingDVFiles, + List eqDeleteFiles) { + + boolean isEmpty() { + return newDataFiles.isEmpty() && eqDeleteFiles.isEmpty() && stagingDVFiles.isEmpty(); + } + } + + private void emitDeletePhase(List eqDeleteFiles) { + for (DeleteFile deleteFile : eqDeleteFiles) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.eqDeleteFile( + deleteFile, + spec, + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(deleteFile)), + nextPhaseTs)); + processedEqDeleteFileNumCounter.inc(); + } + + advancePhase(); + } + + private void emitSnapshotDataPhase(List snapshotDataFiles) { + // Shared-branch skip: when stagingBranch == targetBranch, these files are already on target + // and were indexed by bootstrap/reindex. Re-emitting would duplicate entries in + // dataRowPositions. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : snapshotDataFiles) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.stagingDataFile( + new FlinkAddedRowsScanTask(dataFile, spec), + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(dataFile)), + nextPhaseTs)); + } + } + + advancePhase(); + } + + private void emitNoOpResult(long triggerTimestamp, Long currentMainSnapshotId) { + skippedNoOpCyclesCounter.inc(); + emitDrainResult(triggerTimestamp, currentMainSnapshotId); + } + + /** Emits an empty plan result and advances the phase so the pipeline drains. */ + private void emitDrainResult(long triggerTimestamp, Long currentMainSnapshotId) { + output.collect( + METADATA_STREAM, + new StreamRecord<>( + EqualityConvertPlan.noOp(currentMainSnapshotId, triggerTimestamp, nextPhaseTs))); + advancePhase(); + } + + /** + * Emits {@link ReadCommand}s for every data file on {@code mainSnapshot} so the worker indexes + * them for the configured equality-field set. Existing DVs attached to a data file are loaded by + * the reader and their positions are skipped. V2 positional deletes are not expected on main; the + * reader throws if it encounters one. Equality deletes attached to the scan task are skipped + * during indexing (they are processed via the planner's eq-delete read commands). + */ + private void emitMainDataReadCommands(Snapshot mainSnapshot) { + long commitSnapshotId = mainSnapshot.snapshotId(); + + try (CloseableIterable tasks = + table.newScan().useSnapshot(commitSnapshotId).planFiles()) { + for (FileScanTask task : tasks) { + output.collect( + new StreamRecord<>( + ReadCommand.dataFile( + task, indexSnapshotId, indexedSequenceNumber, dataSequenceNumber(task.file())), + nextPhaseTs)); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to plan files for main index", e); + } + + LOG.info( + "Emitted main data read commands for field IDs {} from snapshot {}.", + eqFieldIds, + commitSnapshotId); + + advancePhase(); + } + + /** + * Emits a phase-end watermark and bumps the phase timestamp. Every phase-emitting method must + * call this exactly once after its records; the worker uses these watermarks to gate keyed-state + * transitions. Missing or extra calls silently break ordering. + */ + private void advancePhase() { + output.emitWatermark(new Watermark(nextPhaseTs)); + nextPhaseTs++; + } + + /** + * Returns {@code true} if {@code snapshot} can be skipped: + * + *
    + *
  • It was already committed by us (carries {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}), OR + *
  • it adds no data or delete files (e.g. delete-file-only removals), OR + *
  • {@code stagingBranch == targetBranch} and the snapshot adds no equality-delete files. + * Pure-insert (and data-file-only) commits on the shared branch don't need a conversion + * cycle: their data is already on target, and the worker's index stays fresh via {@link + * #ensureIndexCurrent} when the next eq-delete arrives. + *
+ * + *

Filter checks read per-snapshot counts from {@link Snapshot#summary()}; we don't parse + * manifests here. Manifest parsing happens later in {@link #retrieveStagingFiles} only for the + * chosen snapshot. + */ + private boolean shouldSkip(Snapshot snapshot) { + Map summary = snapshot.summary(); + if (summary.containsKey(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) { + return true; + } + + long addedDataFiles = summaryCount(summary, SnapshotSummary.ADDED_FILES_PROP); + long addedDeleteFiles = summaryCount(summary, SnapshotSummary.ADDED_DELETE_FILES_PROP); + if (addedDataFiles == 0 && addedDeleteFiles == 0) { + LOG.info( + "Skipping staging snapshot {}: no added data or delete files.", snapshot.snapshotId()); + return true; + } + + if (stagingOnTargetBranch + && summaryCount(summary, SnapshotSummary.ADD_EQ_DELETE_FILES_PROP) == 0) { + return true; + } + + return false; + } + + private static long summaryCount(Map summary, String key) { + String value = summary.get(key); + return value == null ? 0L : Long.parseLong(value); + } + + /** + * Returns the id of the newest snapshot that is reachable from both the staging and target branch + * heads (i.e. the most recent common ancestor where the two branches last matched), or {@code + * null} if they share no history. Used on cold start to know where staging diverged from target + * so we can skip converting snapshots that already exist on target. + */ + private Long findCommonAncestor(Snapshot stagingHead, Snapshot mainHead) { + if (mainHead == null) { + return null; + } + + Set stagingSeen = Sets.newHashSet(); + Set mainSeen = Sets.newHashSet(); + Snapshot stagingCurrent = stagingHead; + Snapshot mainCurrent = mainHead; + + while (stagingCurrent != null || mainCurrent != null) { + if (stagingCurrent != null) { + long id = stagingCurrent.snapshotId(); + if (mainSeen.contains(id)) { + return id; + } + + stagingSeen.add(id); + stagingCurrent = parentOf(stagingCurrent); + } + + if (mainCurrent != null) { + long id = mainCurrent.snapshotId(); + if (stagingSeen.contains(id)) { + return id; + } + + mainSeen.add(id); + mainCurrent = parentOf(mainCurrent); + } + } + + return null; + } + + /** Returns the parent snapshot, or {@code null} if {@code snapshot} has no parent. */ + private Snapshot parentOf(Snapshot snapshot) { + Long parentId = snapshot.parentId(); + return parentId != null ? table.snapshot(parentId) : null; + } + + private static long dataSequenceNumber(ContentFile file) { + Long sequenceNumber = file.dataSequenceNumber(); + Preconditions.checkNotNull( + sequenceNumber, "Missing data sequence number for committed file %s", file.location()); + return sequenceNumber; + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + @VisibleForTesting + long reindexCount() { + return reindexCounter.getCount(); + } + + @VisibleForTesting + long skippedNoOpCycles() { + return skippedNoOpCyclesCounter.getCount(); + } + + @VisibleForTesting + long processedStagingSnapshotNum() { + return processedStagingSnapshotNumCounter.getCount(); + } + + @VisibleForTesting + long processedEqDeleteFileNum() { + return processedEqDeleteFileNumCounter.getCount(); + } +} diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java new file mode 100644 index 000000000000..82860a0de881 --- /dev/null +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -0,0 +1,1115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestEqualityConvertPlanner extends OperatorTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void bootstrapsIndexFromBuilderEqualityFieldIds() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Staging branch exists but contains no eq-delete files yet. The planner still populates the + // worker index from main using the builder-configured equality field set so the index is ready + // when the first delete arrives. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isZero(); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Builder configured for [1, 2], but writer produces an eq delete with [1] only. + DeleteFile mismatched = writeIdOnlyEqualityDelete(table, 1); + table.newRowDelta().addDeletes(mismatched).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { + // When stagingBranch == targetBranch, the writer commits new data files directly to main. + // Bootstrap scans the main snapshot (which already includes those files) and indexes them. + // The planner must NOT also emit the staging-data phase for the same files — that would + // duplicate ADD_DATA_ROW commands for the same (PK, file, position) in the worker's index. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Writer commits (new-data id=3) + (eq-delete id=1) in one RowDelta directly to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(SnapshotRef.MAIN_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + + // Bootstrap from main emits one data-file command per file (id=1, id=2, id=3) = 3. + // Without the same-branch guard, emitSnapshotDataPhase would also emit newDataFile → 4. + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + long newDataFileCount = + commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .filter(c -> c.task().file().location().equals(newDataFile.location())) + .count(); + assertThat(newDataFileCount).isEqualTo(1); + } + } + + @Test + void emitsReadCommandsForEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 3 DATA_FILE (from main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).isEmpty(); + } + } + + @Test + void emitsDataFileFromInsertOnlyStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: eq delete targeting main data + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s1SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // S2: insert-only (no eq deletes), but its data must still be indexed for the configured + // equality fields + DataFile insertS2 = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(insertS2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: processes S1 (eq delete) + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + // Record S1's conversion on main, so the planner walks past S1 before processing S2. + simulateConvertCommit(table, s1SnapshotId); + + // Trigger 2: processes S2 (insert-only). Must emit its data file so the configured equality + // fields stay indexed. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + + Set dataFilePaths = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .map(TestEqualityConvertPlanner::filePath) + .collect(Collectors.toSet()); + + assertThat(dataFilePaths).contains(insertS2.location()); + } + } + + @Test + void includesNewDataFilesFromStaging() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 1 main data file + 1 eq delete + 1 staging data file + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).hasSize(1); + } + } + + @Test + void findsIntersectionWhenMainAdvancedAfterStagingFork() throws Exception { + Table table = createTableWithDelete(3); + // Pre-fork main: two snapshots. + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Fork staging from current main head. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Main advances with two more snapshots after the fork. These are on main's + // lineage but not on staging's. + insert(table, 3, "c"); + insert(table, 4, "d"); + + // Staging gets one new snapshot containing an equality delete. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Planner identifies the fork point and processes only the staging-only + // commit (the eq delete), and emits all four current main data files + // for the index refresh. + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(countDataFileTasks(commands)).isEqualTo(4); + } + } + + @Test + void skipsAlreadyProcessedStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + assertThat(firstTriggerCount).isGreaterThan(0); + assertThat(planner(harness).skippedNoOpCycles()).isZero(); + + // Simulate the committer committing to main with the staging snapshot property + // (the planner promotes pending only after confirming the commit landed on main). + simulateConvertCommit(table, stagingSnapshotId); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(firstTriggerCount); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + } + } + + @Test + void noOutputWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Bootstrap from main for the configured field set: 1 main DATA_FILE; no-op metadata still + // emitted because there's nothing on staging to convert. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(1); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isZero(); + } + } + + @Test + void propagatesStagingOnlyPositionalDeletes() throws Exception { + Table table = createTableWithDelete(3); + DataFile mainData = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(mainData).commit(); + table.refresh(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot has only a DV (no eq deletes, no new data files). Without the + // planner forwarding stagingDVFiles in this case, the committer would never + // see the DV and it would be lost. + DeleteFile stagingDV = writeStagingDV(table, mainData.location(), 0L); + table.newRowDelta().addDeletes(stagingDV).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(commands)).isZero(); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + EqualityConvertPlan result = metadata.get(0).getValue(); + assertThat(result.dataFiles()).isEmpty(); + assertThat(result.stagingDVFiles()).hasSize(1); + assertThat(result.stagingDVFiles().get(0).location()).isEqualTo(stagingDV.location()); + } + } + + @Test + void phaseTimestampsAreMonotonicallyIncreasing() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging: eq delete + new data file (triggers 3 phases: main data, eq delete, staging data) + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + long triggerTs = 100L; + sendTrigger(harness, triggerTs); + + List output = Lists.newArrayList(harness.getOutput()); + assertThat(((StreamRecord) output.get(0)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(1)).isEqualTo(new Watermark(triggerTs)); + + assertThat(((StreamRecord) output.get(2)).getValue().task()) + .isInstanceOf(EqualityDeleteFileScanTask.class); + assertThat(output.get(3)).isEqualTo(new Watermark(triggerTs + 1)); + + assertThat(((StreamRecord) output.get(4)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(5)).isEqualTo(new Watermark(triggerTs + 2)); + } + } + + @Test + void routesExceptionToErrorStream() throws Exception { + createTableWithDelete(3); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + dropTable(); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + sendTrigger(harness); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsOnRemovedDataFilesOnStagingBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + // Rewrite on the staging branch (not main). Equality delete conversion does not support + // rewrites on staging; the planner must fail the cycle instead of silently dropping the + // removed files. + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + // Bootstrap runs before processCycle's failure, so the main data commands are already on + // the wire. Only the cycle itself fails. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + } + } + + @Test + void reEmitsMainDataAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + + // Compact data files on main: rewrite 2 files into 1 + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // 1 DATA_FILE (compacted main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + // DATA_FILE should reference the compacted file + List dataCmds = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .collect(Collectors.toList()); + for (ReadCommand cmd : dataCmds) { + assertThat(cmd.task().file().location()).isEqualTo(compactedFile.location()); + } + } + } + + @Test + void refreshesIndexBeforeFirstCycleProcessesEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Stage MULTIPLE eq-delete snapshots before the first trigger arrives. The first trigger + // bootstraps the index from main for the configured equality fields before processing any eq + // deletes, then processes one snapshot per trigger (oldest first). + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s2SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + + // First trigger processes the OLDER staging snapshot (eqDelete1). Bootstrap built the index + // from main once: 2 main DATA_FILE + 1 EQ_DELETE_FILE = 3 commands. + assertThat(afterFirst).isEqualTo(3); + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + simulateConvertCommit(table, table.snapshot(STAGING_BRANCH).parentId()); + + // Second trigger processes the newer staging snapshot. The index is already up-to-date. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + assertThat(countDataFileTasks(trigger2Commands)).isZero(); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + simulateConvertCommit(table, s2SnapshotId); + } + } + + @Test + void noMainReEmitWhenUnchanged() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Only 1 EQ_DELETE_FILE, no main re-emission + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(2); + } + } + + @Test + void noMainReEmitAfterOwnCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long indexedStagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + + // Record the converter's own commit (COMMITTED_STAGING_SNAPSHOT property) for the staging + // snapshot the first trigger processed. The next trigger reads the property off main and + // advances lastStagingSnapshotId. No reindex should be triggered. + simulateConvertCommit(table, indexedStagingSnapshotId); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Own commits don't trigger index rebuild. + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + } + } + + @Test + void reIndexesAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 1 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(2); + assertThat(planner(harness).reindexCount()).isZero(); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + // External commit: no COMMITTED_STAGING_SNAPSHOT_PROPERTY + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // External commit triggers re-index: main data files are re-emitted + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(2); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(2); + } + } + + @Test + void emitsClearIndexBroadcastOnReindex() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1 bootstraps the index. Bootstrap does not broadcast CLEAR_INDEX. + sendTrigger(harness); + assertThat(harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM)) + .isNullOrEmpty(); + + // External commit advances main. The next trigger reindexes and must broadcast CLEAR_INDEX + // so workers evict ghost keys (e.g. a PK whose data file was removed by external CoW). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + long mainAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).snapshotId(); + long mainSeqAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).sequenceNumber(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + + List clears = + harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM).stream() + .map(StreamRecord::getValue) + .collect(Collectors.toList()); + assertThat(clears).hasSize(1); + assertThat(clears.get(0).type()).isEqualTo(IndexCommand.Type.CLEAR_INDEX); + assertThat(clears.get(0).mainSnapshotId()).isEqualTo(mainAfterExternal); + assertThat(clears.get(0).mainSequenceNumber()).isEqualTo(mainSeqAfterExternal); + } + } + + @Test + void detectsMainBranchChangeWithoutNewStagingSnapshots() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: build the index and process the existing eq delete. + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(afterFirst).isGreaterThan(0); + + // External commit on main (no COMMITTED_STAGING_SNAPSHOT_PROPERTY). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + // Trigger 2: nothing new on staging, but the planner must still notice the + // external main change and immediately re-emit main data for index rebuild. + sendTrigger(harness); + List allAfterTrigger2 = harness.extractOutputValues(); + List trigger2Commands = + allAfterTrigger2.subList(afterFirst, allAfterTrigger2.size()); + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + + // A subsequent staging eq delete should NOT re-emit main data because the index + // was already rebuilt in trigger 2. + int afterSecond = allAfterTrigger2.size(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 10, "z"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List all = harness.extractOutputValues(); + List trigger3 = all.subList(afterSecond, all.size()); + assertThat(countDataFileTasks(trigger3)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger3)).isEqualTo(1); + } + } + + @Test + void stateRestoredAfterCheckpoint() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // First trigger bootstraps the index and processes the staging snapshot. + sendTrigger(harness); + int commandCount = harness.extractOutputValues().size(); + assertThat(commandCount).isGreaterThan(0); + + // Record S1's conversion on main (committer marker) so the planner advances + // lastStagingSnapshotId on the second trigger. + simulateConvertCommit(table, stagingSnapshotId); + + // Second trigger reads the marker and advances lastStagingSnapshotId. + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(commandCount); + + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // Restore from checkpoint: the restored index is not re-indexed because its state matches the + // snapshot COMMITTED_STAGING_SNAPSHOT_PROPERTY property. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void failsOnV2PosDeleteOnStagingBranch() throws Exception { + Table table = createTableWithDelete(2); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Flink-written staging branches are V3 and only produce DVs for deletes. A V2 positional + // delete on staging is a writer-side misconfiguration and should fail the cycle rather than + // be silently absorbed. + DataFile dataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile posDelete = writePosDeleteFile(table, dataFile.location(), 0); + table.newRowDelta().addRows(dataFile).addDeletes(posDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void skipsCommittedSnapshotAfterCommitLandsButCheckpointDoesNot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // Capture the planner's checkpoint before any cycle has run. + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // The staging commit landed on main but lastStagingSnapshotId was not checkpointed. + simulateConvertCommit(table, stagingSnapshotId); + + // On restore, the planner walks main, finds the COMMITTED_STAGING_SNAPSHOT_PROPERTY, + // advances lastStagingSnapshotId, and skips re-emitting the already-committed snapshot. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(0); + // Bootstrap from main creates the index from the main data files on the first trigger even + // though the staging snapshot itself is already committed and skipped. Main now has the + // original insert plus simulateConvertCommit's marker file = 2 data files. + assertThat(countDataFileTasks(commands)).isEqualTo(2); + } + } + + @Test + void failsWhenCommitMarkerDisappears() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + long startSnapshotId = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + simulateConvertCommit(table, stagingSnapshotId); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + table.manageSnapshots().rollbackTo(startSnapshotId).commit(); + table.refresh(); + + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + assertThat( + harness + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .poll() + .getValue() + .getMessage()) + .contains("No COMMITTED_STAGING_SNAPSHOT marker reachable"); + } + } + + @Test + void failsOnEqFieldIdsChangeAcrossRestart() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + assertThatThrownBy(() -> harness.initializeState(state)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Equality field IDs changed across restart"); + } + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch) throws Exception { + // Default matches the [1, 2] equality-field-id list produced by writeEqualityDelete. + return createHarness(stagingBranch, Lists.newArrayList(1, 2)); + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch, List eqFieldIds) throws Exception { + return new OneInputStreamOperatorTestHarness<>( + new EqualityConvertPlanner( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + stagingBranch, + SnapshotRef.MAIN_BRANCH, + Sets.newHashSet(eqFieldIds))); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness) + throws Exception { + sendTrigger(harness, System.currentTimeMillis()); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness, long time) + throws Exception { + harness.processElement(new StreamRecord<>(Trigger.create(time, 0), time)); + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(record), + idOnly); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private static long countDataFileTasks(List commands) { + return commands.stream().filter(c -> c.task() instanceof FileScanTask).count(); + } + + private static long countEqDeleteTasks(List commands) { + return commands.stream().filter(TestEqualityConvertPlanner::isEqDelete).count(); + } + + private static EqualityConvertPlanner planner( + OneInputStreamOperatorTestHarness harness) { + return (EqualityConvertPlanner) harness.getOperator(); + } + + private static boolean isEqDelete(ReadCommand cmd) { + return cmd.task() instanceof EqualityDeleteFileScanTask; + } + + private static String filePath(ReadCommand cmd) { + return cmd.task().file().location(); + } + + private void simulateConvertCommit(Table table, long stagingSnapshotId) throws IOException { + DataFile dummy = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(-1, "marker" + stagingSnapshotId))); + table + .newRowDelta() + .addRows(dummy) + .set( + EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY, + String.valueOf(stagingSnapshotId)) + .commit(); + table.refresh(); + } +} From aa4abedd2573b9fec5567213195ce208bd91609e Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Wed, 24 Jun 2026 14:12:45 +0900 Subject: [PATCH 31/73] Build: Bump com.google.cloud:libraries-bom from 26.83.0 to 26.84.0 (#16906) Bumps [com.google.cloud:libraries-bom](https://github.com/googleapis/java-cloud-bom) from 26.83.0 to 26.84.0. - [Release notes](https://github.com/googleapis/java-cloud-bom/releases) - [Commits](https://github.com/googleapis/java-cloud-bom/compare/v26.83.0...v26.84.0) --- updated-dependencies: - dependency-name: com.google.cloud:libraries-bom dependency-version: 26.84.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gcp-bundle/runtime-deps.txt | 28 ++++---- gradle/libs.versions.toml | 2 +- .../kafka-connect-runtime/runtime-deps.txt | 72 +++++++++---------- 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/gcp-bundle/runtime-deps.txt b/gcp-bundle/runtime-deps.txt index 5577e8669e4f..48d68619d892 100644 --- a/gcp-bundle/runtime-deps.txt +++ b/gcp-bundle/runtime-deps.txt @@ -8,17 +8,17 @@ com.github.ben-manes.caffeine:caffeine:3.2 com.google.android:annotations:4.1 com.google.api-client:google-api-client:2.7 com.google.api.grpc:gapic-google-cloud-storage-v2:2.69 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.28 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.200 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.200 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.29 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:3.29 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:3.29 com.google.api.grpc:grpc-google-cloud-storage-v2:2.69 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.28 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.28 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.200 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.200 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.28 -com.google.api.grpc:proto-google-cloud-kms-v1:0.186 -com.google.api.grpc:proto-google-cloud-monitoring-v3:3.93 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.29 +com.google.api.grpc:proto-google-cloud-kms-v1:2.96 +com.google.api.grpc:proto-google-cloud-monitoring-v3:3.94 com.google.api.grpc:proto-google-cloud-storage-v2:2.69 com.google.api.grpc:proto-google-common-protos:2.72 com.google.api.grpc:proto-google-iam-v1:1.67 @@ -37,13 +37,13 @@ com.google.cloud.gcs.analytics:gcs-analytics-core:1.3 com.google.cloud.opentelemetry:detector-resources-support:0.33 com.google.cloud.opentelemetry:exporter-metrics:0.33 com.google.cloud.opentelemetry:shared-resourcemapping:0.33 -com.google.cloud:google-cloud-bigquery:2.66 -com.google.cloud:google-cloud-bigquerystorage:3.28 +com.google.cloud:google-cloud-bigquery:2.67 +com.google.cloud:google-cloud-bigquerystorage:3.29 com.google.cloud:google-cloud-core-grpc:2.71 com.google.cloud:google-cloud-core-http:2.71 com.google.cloud:google-cloud-core:2.71 -com.google.cloud:google-cloud-kms:2.95 -com.google.cloud:google-cloud-monitoring:3.93 +com.google.cloud:google-cloud-kms:2.96 +com.google.cloud:google-cloud-monitoring:3.94 com.google.cloud:google-cloud-storage:2.69 com.google.code.gson:gson:2.13 com.google.errorprone:error_prone_annotations:2.49 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 662dc6196935..b859567560a0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -51,7 +51,7 @@ findbugs-jsr305 = "3.0.2" flink120 = { strictly = "1.20.1"} flink20 = { strictly = "2.0.0"} flink21 = { strictly = "2.1.0"} -google-libraries-bom = "26.83.0" +google-libraries-bom = "26.84.0" gcs-analytics-core = "1.3.1" guava = "33.6.0-jre" hadoop3 = "3.4.3" diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt index 2aef692c4f6f..1cb307041212 100644 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -18,39 +18,39 @@ com.github.luben:zstd-jni:1.5 com.github.pjfanning:jersey-json:1.22 com.google.android:annotations:4.1 com.google.api-client:google-api-client:2.7 -com.google.api.grpc:gapic-google-cloud-storage-v2:2.68 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.28 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.200 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.200 -com.google.api.grpc:grpc-google-cloud-storage-v2:2.68 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.28 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.28 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.200 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.200 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.28 -com.google.api.grpc:proto-google-cloud-monitoring-v3:3.93 -com.google.api.grpc:proto-google-cloud-storage-v2:2.68 -com.google.api.grpc:proto-google-common-protos:2.71 -com.google.api.grpc:proto-google-iam-v1:1.66 -com.google.api:api-common:2.63 -com.google.api:gax-grpc:2.80 -com.google.api:gax-httpjson:2.80 -com.google.api:gax:2.80 +com.google.api.grpc:gapic-google-cloud-storage-v2:2.69 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.29 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:3.29 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:3.29 +com.google.api.grpc:grpc-google-cloud-storage-v2:2.69 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.29 +com.google.api.grpc:proto-google-cloud-monitoring-v3:3.94 +com.google.api.grpc:proto-google-cloud-storage-v2:2.69 +com.google.api.grpc:proto-google-common-protos:2.72 +com.google.api.grpc:proto-google-iam-v1:1.67 +com.google.api:api-common:2.64 +com.google.api:gax-grpc:2.81 +com.google.api:gax-httpjson:2.81 +com.google.api:gax:2.81 com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0 com.google.apis:google-api-services-storage:v1-rev20260204-2.0 -com.google.auth:google-auth-library-credentials:1.47 -com.google.auth:google-auth-library-oauth2-http:1.47 +com.google.auth:google-auth-library-credentials:1.48 +com.google.auth:google-auth-library-oauth2-http:1.48 com.google.auto.value:auto-value-annotations:1.11 com.google.cloud.opentelemetry:detector-resources-support:0.33 com.google.cloud.opentelemetry:exporter-metrics:0.33 com.google.cloud.opentelemetry:shared-resourcemapping:0.33 -com.google.cloud:google-cloud-bigquery:2.66 -com.google.cloud:google-cloud-bigquerystorage:3.28 -com.google.cloud:google-cloud-core-grpc:2.70 -com.google.cloud:google-cloud-core-http:2.70 -com.google.cloud:google-cloud-core:2.70 -com.google.cloud:google-cloud-monitoring:3.93 -com.google.cloud:google-cloud-storage:2.68 +com.google.cloud:google-cloud-bigquery:2.67 +com.google.cloud:google-cloud-bigquerystorage:3.29 +com.google.cloud:google-cloud-core-grpc:2.71 +com.google.cloud:google-cloud-core-http:2.71 +com.google.cloud:google-cloud-core:2.71 +com.google.cloud:google-cloud-monitoring:3.94 +com.google.cloud:google-cloud-storage:2.69 com.google.code.findbugs:jsr305:3.0 com.google.code.gson:gson:2.13 com.google.errorprone:error_prone_annotations:2.48 @@ -128,15 +128,15 @@ io.opencensus:opencensus-api:0.31 io.opencensus:opencensus-contrib-http-util:0.31 io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37 io.opentelemetry.semconv:opentelemetry-semconv:1.29 -io.opentelemetry:opentelemetry-api:1.57 -io.opentelemetry:opentelemetry-common:1.57 -io.opentelemetry:opentelemetry-context:1.57 -io.opentelemetry:opentelemetry-sdk-common:1.57 -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.57 -io.opentelemetry:opentelemetry-sdk-logs:1.57 -io.opentelemetry:opentelemetry-sdk-metrics:1.57 -io.opentelemetry:opentelemetry-sdk-trace:1.57 -io.opentelemetry:opentelemetry-sdk:1.57 +io.opentelemetry:opentelemetry-api:1.62 +io.opentelemetry:opentelemetry-common:1.62 +io.opentelemetry:opentelemetry-context:1.62 +io.opentelemetry:opentelemetry-sdk-common:1.62 +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.62 +io.opentelemetry:opentelemetry-sdk-logs:1.62 +io.opentelemetry:opentelemetry-sdk-metrics:1.62 +io.opentelemetry:opentelemetry-sdk-trace:1.62 +io.opentelemetry:opentelemetry-sdk:1.62 io.perfmark:perfmark-api:0.27 io.projectreactor.netty:reactor-netty-core:1.2 io.projectreactor.netty:reactor-netty-http:1.2 From 534c3fd9427b7037b5fa683d51b4dd5fc255c472 Mon Sep 17 00:00:00 2001 From: Neelesh Salian Date: Tue, 23 Jun 2026 23:54:00 -0700 Subject: [PATCH 32/73] Core: Migrate switch statements to switch expressions (#16881) Co-authored-by: Maximilian Michels Co-authored-by: Manu Zhang --- .../iceberg/BaseDistributedDataScan.java | 23 +- .../java/org/apache/iceberg/BaseFile.java | 168 +++++-------- .../iceberg/BaseIncrementalChangelogScan.java | 46 ++-- .../iceberg/BasePartitionStatistics.java | 101 +++----- .../java/org/apache/iceberg/BaseScan.java | 12 +- .../java/org/apache/iceberg/BaseSnapshot.java | 25 +- .../org/apache/iceberg/BaseTransaction.java | 19 +- .../java/org/apache/iceberg/CatalogUtil.java | 37 +-- .../org/apache/iceberg/ContentFileParser.java | 18 +- .../org/apache/iceberg/DeleteFileIndex.java | 12 +- .../apache/iceberg/DeletionVectorStruct.java | 41 ++-- .../java/org/apache/iceberg/FileMetadata.java | 14 +- .../apache/iceberg/GenericManifestEntry.java | 45 ++-- .../apache/iceberg/GenericManifestFile.java | 133 ++++------- .../iceberg/GenericPartitionFieldSummary.java | 38 +-- .../org/apache/iceberg/ManifestFiles.java | 74 +++--- .../apache/iceberg/ManifestInfoStruct.java | 87 +++---- .../org/apache/iceberg/ManifestLists.java | 52 ++-- .../org/apache/iceberg/ManifestWriter.java | 12 +- .../iceberg/MergingSnapshotProducer.java | 31 +-- .../apache/iceberg/MetadataTableUtils.java | 56 ++--- .../apache/iceberg/MetadataUpdateParser.java | 224 +++++++----------- .../org/apache/iceberg/PartitionData.java | 21 +- .../apache/iceberg/PartitionStatsHandler.java | 36 +-- .../org/apache/iceberg/PartitionsTable.java | 18 +- .../apache/iceberg/RewriteTablePathUtil.java | 27 +-- .../java/org/apache/iceberg/ScanSummary.java | 26 +- .../org/apache/iceberg/ScanTaskParser.java | 21 +- .../java/org/apache/iceberg/SchemaParser.java | 15 +- .../java/org/apache/iceberg/SchemaUpdate.java | 18 +- .../org/apache/iceberg/SingleValueParser.java | 164 +++++++------ .../org/apache/iceberg/SnapshotChanges.java | 16 +- .../org/apache/iceberg/SnapshotProducer.java | 12 +- .../org/apache/iceberg/SnapshotSummary.java | 44 ++-- .../org/apache/iceberg/SortOrderParser.java | 14 +- .../org/apache/iceberg/TrackedFileStruct.java | 125 +++------- .../org/apache/iceberg/TrackingStruct.java | 73 ++---- .../iceberg/UpdateRequirementParser.java | 103 ++++---- .../java/org/apache/iceberg/V1Metadata.java | 114 ++++----- .../java/org/apache/iceberg/V2Metadata.java | 151 +++++------- .../java/org/apache/iceberg/V3Metadata.java | 181 ++++++-------- .../java/org/apache/iceberg/V4Metadata.java | 177 ++++++-------- .../iceberg/actions/RewriteFileGroup.java | 22 +- .../actions/RewritePositionDeletesGroup.java | 25 +- .../java/org/apache/iceberg/avro/Avro.java | 31 +-- .../avro/AvroCustomOrderSchemaVisitor.java | 29 +-- .../apache/iceberg/avro/AvroFormatModel.java | 15 +- .../apache/iceberg/avro/AvroSchemaUtil.java | 15 +- .../iceberg/avro/AvroSchemaVisitor.java | 33 ++- .../avro/AvroSchemaWithTypeVisitor.java | 25 +- .../AvroWithPartnerByStructureVisitor.java | 29 +-- .../iceberg/avro/AvroWithPartnerVisitor.java | 42 ++-- .../apache/iceberg/avro/BaseWriteBuilder.java | 67 ++---- .../iceberg/avro/BuildAvroProjection.java | 23 +- .../iceberg/avro/GenericAvroReader.java | 88 +++---- .../apache/iceberg/avro/InternalReader.java | 86 +++---- .../org/apache/iceberg/avro/SchemaToType.java | 36 +-- .../org/apache/iceberg/avro/TypeToSchema.java | 64 ++--- .../org/apache/iceberg/avro/ValueReaders.java | 16 +- .../apache/iceberg/data/GenericDataUtil.java | 30 ++- .../data/IdentityPartitionConverters.java | 30 ++- .../apache/iceberg/data/avro/DataReader.java | 89 +++---- .../apache/iceberg/data/avro/DataWriter.java | 75 +++--- .../iceberg/data/avro/PlannedDataReader.java | 97 ++++---- .../iceberg/deletes/DeleteGranularity.java | 12 +- .../iceberg/deletes/PositionDelete.java | 31 +-- .../SortingPositionOnlyDeleteWriter.java | 13 +- .../encryption/StandardKeyMetadata.java | 31 +-- .../iceberg/expressions/ExpressionParser.java | 74 +++--- .../iceberg/hadoop/HadoopMetricsContext.java | 52 ++-- .../io/ClusteredPositionDeleteWriter.java | 12 +- .../iceberg/metrics/TimerResultParser.java | 30 +-- .../apache/iceberg/puffin/PuffinFormat.java | 41 ++-- .../apache/iceberg/puffin/PuffinReader.java | 12 +- .../apache/iceberg/rest/CatalogHandlers.java | 27 +-- .../apache/iceberg/rest/ErrorHandlers.java | 107 ++++----- .../iceberg/rest/RESTTableOperations.java | 21 +- .../apache/iceberg/rest/RESTTableScan.java | 55 ++--- .../iceberg/rest/auth/AuthManagers.java | 29 +-- .../apache/iceberg/rest/auth/OAuth2Util.java | 22 +- .../schema/SchemaWithPartnerVisitor.java | 29 +-- .../iceberg/variants/PrimitiveWrapper.java | 169 ++++++------- .../iceberg/variants/VariantVisitor.java | 19 +- .../view/ViewRepresentationParser.java | 25 +- 84 files changed, 1766 insertions(+), 2636 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java b/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java index 027a3d2298f0..94c3b02e56a3 100644 --- a/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java @@ -240,21 +240,14 @@ private boolean shouldPlanLocally(PlanningMode mode, List manifest return true; } - switch (mode) { - case LOCAL: - return true; - - case DISTRIBUTED: - return manifests.isEmpty(); - - case AUTO: - return remoteParallelism() <= localParallelism - || manifests.size() <= 2 * localParallelism - || totalSize(manifests) <= localPlanningSizeThreshold; - - default: - throw new IllegalArgumentException("Unknown planning mode: " + mode); - } + return switch (mode) { + case LOCAL -> true; + case DISTRIBUTED -> manifests.isEmpty(); + case AUTO -> + remoteParallelism() <= localParallelism + || manifests.size() <= 2 * localParallelism + || totalSize(manifests) <= localPlanningSizeThreshold; + }; } private long totalSize(List manifests) { diff --git a/core/src/main/java/org/apache/iceberg/BaseFile.java b/core/src/main/java/org/apache/iceberg/BaseFile.java index 7147ba58787b..e42989ea1f31 100644 --- a/core/src/main/java/org/apache/iceberg/BaseFile.java +++ b/core/src/main/java/org/apache/iceberg/BaseFile.java @@ -316,79 +316,40 @@ public void put(int i, Object value) { @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0: - this.content = value != null ? FileContent.fromId((Integer) value) : FileContent.DATA; - return; - case 1: - // always coerce to String for Serializable - this.filePath = value.toString(); - return; - case 2: - this.format = FileFormat.fromString(value.toString()); - return; - case 3: - this.partitionSpecId = (value != null) ? (Integer) value : -1; - return; - case 4: + case 0 -> + this.content = value != null ? FileContent.fromId((Integer) value) : FileContent.DATA; + case 1 -> this.filePath = value.toString(); // always coerce to String for Serializable + case 2 -> this.format = FileFormat.fromString(value.toString()); + case 3 -> this.partitionSpecId = (value != null) ? (Integer) value : -1; + case 4 -> { // Preserve the constructor-initialized partitionData when the reader returns null // (e.g., v4 Parquet manifests for unpartitioned tables omit the partition field). if (value != null) { this.partitionData = (PartitionData) value; } - return; - case 5: - this.recordCount = (Long) value; - return; - case 6: - this.fileSizeInBytes = (Long) value; - return; - case 7: - this.columnSizes = (Map) value; - return; - case 8: - this.valueCounts = (Map) value; - return; - case 9: - this.nullValueCounts = (Map) value; - return; - case 10: - this.nanValueCounts = (Map) value; - return; - case 11: - this.lowerBounds = SerializableByteBufferMap.wrap((Map) value); - return; - case 12: - this.upperBounds = SerializableByteBufferMap.wrap((Map) value); - return; - case 13: - this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); - return; - case 14: - this.splitOffsets = ArrayUtil.toLongArray((List) value); - return; - case 15: - this.equalityIds = ArrayUtil.toIntArray((List) value); - return; - case 16: - this.sortOrderId = (Integer) value; - return; - case 17: - this.firstRowId = (Long) value; - return; - case 18: - this.referencedDataFile = value != null ? value.toString() : null; - return; - case 19: - this.contentOffset = (Long) value; - return; - case 20: - this.contentSizeInBytes = (Long) value; - return; - case 21: - this.fileOrdinal = (long) value; - return; - default: + } + case 5 -> this.recordCount = (Long) value; + case 6 -> this.fileSizeInBytes = (Long) value; + case 7 -> this.columnSizes = (Map) value; + case 8 -> this.valueCounts = (Map) value; + case 9 -> this.nullValueCounts = (Map) value; + case 10 -> this.nanValueCounts = (Map) value; + case 11 -> + this.lowerBounds = SerializableByteBufferMap.wrap((Map) value); + case 12 -> + this.upperBounds = SerializableByteBufferMap.wrap((Map) value); + case 13 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + case 14 -> this.splitOffsets = ArrayUtil.toLongArray((List) value); + case 15 -> this.equalityIds = ArrayUtil.toIntArray((List) value); + case 16 -> this.sortOrderId = (Integer) value; + case 17 -> this.firstRowId = (Long) value; + case 18 -> this.referencedDataFile = value != null ? value.toString() : null; + case 19 -> this.contentOffset = (Long) value; + case 20 -> this.contentSizeInBytes = (Long) value; + case 21 -> this.fileOrdinal = (long) value; + default -> { // ignore the object, it must be from a newer version of the format + } } } @@ -398,54 +359,31 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int basePos) { - switch (basePos) { - case 0: - return content.id(); - case 1: - return filePath; - case 2: - return format != null ? format.toString() : null; - case 3: - return partitionSpecId; - case 4: - return partitionData; - case 5: - return recordCount; - case 6: - return fileSizeInBytes; - case 7: - return columnSizes; - case 8: - return valueCounts; - case 9: - return nullValueCounts; - case 10: - return nanValueCounts; - case 11: - return lowerBounds; - case 12: - return upperBounds; - case 13: - return keyMetadata(); - case 14: - return splitOffsets(); - case 15: - return equalityFieldIds(); - case 16: - return sortOrderId; - case 17: - return firstRowId; - case 18: - return referencedDataFile; - case 19: - return contentOffset; - case 20: - return contentSizeInBytes; - case 21: - return fileOrdinal; - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + basePos); - } + return switch (basePos) { + case 0 -> content.id(); + case 1 -> filePath; + case 2 -> format != null ? format.toString() : null; + case 3 -> partitionSpecId; + case 4 -> partitionData; + case 5 -> recordCount; + case 6 -> fileSizeInBytes; + case 7 -> columnSizes; + case 8 -> valueCounts; + case 9 -> nullValueCounts; + case 10 -> nanValueCounts; + case 11 -> lowerBounds; + case 12 -> upperBounds; + case 13 -> keyMetadata(); + case 14 -> splitOffsets(); + case 15 -> equalityFieldIds(); + case 16 -> sortOrderId; + case 17 -> firstRowId; + case 18 -> referencedDataFile; + case 19 -> contentOffset; + case 20 -> contentSizeInBytes; + case 21 -> fileOrdinal; + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + basePos); + }; } @Override diff --git a/core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java b/core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java index 2d54a94e8d73..ee16ae6d2a18 100644 --- a/core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java @@ -153,30 +153,28 @@ public CloseableIterable apply( int changeOrdinal = snapshotOrdinals.get(commitSnapshotId); DataFile dataFile = entry.file().copy(context.shouldKeepStats()); - switch (entry.status()) { - case ADDED: - return new BaseAddedRowsScanTask( - changeOrdinal, - commitSnapshotId, - dataFile, - NO_DELETES, - context.schemaAsString(), - context.specAsString(), - context.residuals()); - - case DELETED: - return new BaseDeletedDataFileScanTask( - changeOrdinal, - commitSnapshotId, - dataFile, - NO_DELETES, - context.schemaAsString(), - context.specAsString(), - context.residuals()); - - default: - throw new IllegalArgumentException("Unexpected entry status: " + entry.status()); - } + return switch (entry.status()) { + case ADDED -> + new BaseAddedRowsScanTask( + changeOrdinal, + commitSnapshotId, + dataFile, + NO_DELETES, + context.schemaAsString(), + context.specAsString(), + context.residuals()); + case DELETED -> + new BaseDeletedDataFileScanTask( + changeOrdinal, + commitSnapshotId, + dataFile, + NO_DELETES, + context.schemaAsString(), + context.specAsString(), + context.residuals()); + default -> + throw new IllegalArgumentException("Unexpected entry status: " + entry.status()); + }; }); } } diff --git a/core/src/main/java/org/apache/iceberg/BasePartitionStatistics.java b/core/src/main/java/org/apache/iceberg/BasePartitionStatistics.java index 1cdb4a49e341..56b020bb9b55 100644 --- a/core/src/main/java/org/apache/iceberg/BasePartitionStatistics.java +++ b/core/src/main/java/org/apache/iceberg/BasePartitionStatistics.java @@ -145,36 +145,22 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - switch (pos) { - case PARTITION_POSITION: - return partition; - case SPEC_ID_POSITION: - return specId; - case DATA_RECORD_COUNT_POSITION: - return dataRecordCount; - case DATA_FILE_COUNT_POSITION: - return dataFileCount; - case TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION: - return totalDataFileSizeInBytes; - case POSITION_DELETE_RECORD_COUNT_POSITION: - return positionDeleteRecordCount; - case POSITION_DELETE_FILE_COUNT_POSITION: - return positionDeleteFileCount; - case EQUALITY_DELETE_RECORD_COUNT_POSITION: - return equalityDeleteRecordCount; - case EQUALITY_DELETE_FILE_COUNT_POSITION: - return equalityDeleteFileCount; - case TOTAL_RECORD_COUNT_POSITION: - return totalRecordCount; - case LAST_UPDATED_AT_POSITION: - return lastUpdatedAt; - case LAST_UPDATED_SNAPSHOT_ID_POSITION: - return lastUpdatedSnapshotId; - case DV_COUNT_POSITION: - return dvCount; - default: - throw new UnsupportedOperationException("Unknown position: " + pos); - } + return switch (pos) { + case PARTITION_POSITION -> partition; + case SPEC_ID_POSITION -> specId; + case DATA_RECORD_COUNT_POSITION -> dataRecordCount; + case DATA_FILE_COUNT_POSITION -> dataFileCount; + case TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION -> totalDataFileSizeInBytes; + case POSITION_DELETE_RECORD_COUNT_POSITION -> positionDeleteRecordCount; + case POSITION_DELETE_FILE_COUNT_POSITION -> positionDeleteFileCount; + case EQUALITY_DELETE_RECORD_COUNT_POSITION -> equalityDeleteRecordCount; + case EQUALITY_DELETE_FILE_COUNT_POSITION -> equalityDeleteFileCount; + case TOTAL_RECORD_COUNT_POSITION -> totalRecordCount; + case LAST_UPDATED_AT_POSITION -> lastUpdatedAt; + case LAST_UPDATED_SNAPSHOT_ID_POSITION -> lastUpdatedSnapshotId; + case DV_COUNT_POSITION -> dvCount; + default -> throw new UnsupportedOperationException("Unknown position: " + pos); + }; } @Override @@ -184,47 +170,20 @@ protected void internalSet(int pos, T value) { } switch (pos) { - case PARTITION_POSITION: - this.partition = (StructLike) value; - break; - case SPEC_ID_POSITION: - this.specId = (int) value; - break; - case DATA_RECORD_COUNT_POSITION: - this.dataRecordCount = (long) value; - break; - case DATA_FILE_COUNT_POSITION: - this.dataFileCount = (int) value; - break; - case TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION: - this.totalDataFileSizeInBytes = (long) value; - break; - case POSITION_DELETE_RECORD_COUNT_POSITION: - this.positionDeleteRecordCount = (long) value; - break; - case POSITION_DELETE_FILE_COUNT_POSITION: - this.positionDeleteFileCount = (int) value; - break; - case EQUALITY_DELETE_RECORD_COUNT_POSITION: - this.equalityDeleteRecordCount = (long) value; - break; - case EQUALITY_DELETE_FILE_COUNT_POSITION: - this.equalityDeleteFileCount = (int) value; - break; - case TOTAL_RECORD_COUNT_POSITION: - this.totalRecordCount = (Long) value; - break; - case LAST_UPDATED_AT_POSITION: - this.lastUpdatedAt = (Long) value; - break; - case LAST_UPDATED_SNAPSHOT_ID_POSITION: - this.lastUpdatedSnapshotId = (Long) value; - break; - case DV_COUNT_POSITION: - this.dvCount = (int) value; - break; - default: - throw new UnsupportedOperationException("Unknown position: " + pos); + case PARTITION_POSITION -> this.partition = (StructLike) value; + case SPEC_ID_POSITION -> this.specId = (int) value; + case DATA_RECORD_COUNT_POSITION -> this.dataRecordCount = (long) value; + case DATA_FILE_COUNT_POSITION -> this.dataFileCount = (int) value; + case TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION -> this.totalDataFileSizeInBytes = (long) value; + case POSITION_DELETE_RECORD_COUNT_POSITION -> this.positionDeleteRecordCount = (long) value; + case POSITION_DELETE_FILE_COUNT_POSITION -> this.positionDeleteFileCount = (int) value; + case EQUALITY_DELETE_RECORD_COUNT_POSITION -> this.equalityDeleteRecordCount = (long) value; + case EQUALITY_DELETE_FILE_COUNT_POSITION -> this.equalityDeleteFileCount = (int) value; + case TOTAL_RECORD_COUNT_POSITION -> this.totalRecordCount = (Long) value; + case LAST_UPDATED_AT_POSITION -> this.lastUpdatedAt = (Long) value; + case LAST_UPDATED_SNAPSHOT_ID_POSITION -> this.lastUpdatedSnapshotId = (Long) value; + case DV_COUNT_POSITION -> this.dvCount = (int) value; + default -> throw new UnsupportedOperationException("Unknown position: " + pos); } } } diff --git a/core/src/main/java/org/apache/iceberg/BaseScan.java b/core/src/main/java/org/apache/iceberg/BaseScan.java index 242a5aaacc09..82fb5a3fdc2a 100644 --- a/core/src/main/java/org/apache/iceberg/BaseScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseScan.java @@ -319,13 +319,9 @@ public ThisT minRowsRequested(long numRows) { * @return a list of column names corresponding to the specified manifest content type. */ static List scanColumns(ManifestContent content) { - switch (content) { - case DATA: - return BaseScan.SCAN_COLUMNS; - case DELETES: - return BaseScan.DELETE_SCAN_COLUMNS; - default: - throw new UnsupportedOperationException("Cannot read unknown manifest type: " + content); - } + return switch (content) { + case DATA -> BaseScan.SCAN_COLUMNS; + case DELETES -> BaseScan.DELETE_SCAN_COLUMNS; + }; } } diff --git a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java index b8ea6db22938..d095fd2f1249 100644 --- a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java +++ b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java @@ -276,14 +276,11 @@ private void cacheDeleteFileChanges(FileIO fileIO) { ManifestFiles.readDeleteManifest(manifest, fileIO, null)) { for (ManifestEntry entry : reader.entries()) { switch (entry.status()) { - case ADDED: - adds.add(entry.file().copy()); - break; - case DELETED: - deletes.add(entry.file().copyWithoutStats()); - break; - default: + case ADDED -> adds.add(entry.file().copy()); + case DELETED -> deletes.add(entry.file().copyWithoutStats()); + default -> { // ignore existing + } } } } catch (IOException e) { @@ -309,15 +306,11 @@ private void cacheDataFileChanges(FileIO fileIO) { new ManifestGroup(fileIO, changedManifests).ignoreExisting().entries()) { for (ManifestEntry entry : entries) { switch (entry.status()) { - case ADDED: - adds.add(entry.file().copy()); - break; - case DELETED: - deletes.add(entry.file().copyWithoutStats()); - break; - default: - throw new IllegalStateException( - "Unexpected entry status, not added or deleted: " + entry); + case ADDED -> adds.add(entry.file().copy()); + case DELETED -> deletes.add(entry.file().copyWithoutStats()); + default -> + throw new IllegalStateException( + "Unexpected entry status, not added or deleted: " + entry); } } } catch (IOException e) { diff --git a/core/src/main/java/org/apache/iceberg/BaseTransaction.java b/core/src/main/java/org/apache/iceberg/BaseTransaction.java index 9884ac297079..2227676a7968 100644 --- a/core/src/main/java/org/apache/iceberg/BaseTransaction.java +++ b/core/src/main/java/org/apache/iceberg/BaseTransaction.java @@ -253,21 +253,10 @@ public void commitTransaction() { hasLastOpCommitted, "Cannot commit transaction: last operation has not committed"); switch (type) { - case CREATE_TABLE: - commitCreateTransaction(); - break; - - case REPLACE_TABLE: - commitReplaceTransaction(false); - break; - - case CREATE_OR_REPLACE_TABLE: - commitReplaceTransaction(true); - break; - - case SIMPLE: - commitSimpleTransaction(); - break; + case CREATE_TABLE -> commitCreateTransaction(); + case REPLACE_TABLE -> commitReplaceTransaction(false); + case CREATE_OR_REPLACE_TABLE -> commitReplaceTransaction(true); + case SIMPLE -> commitSimpleTransaction(); } } diff --git a/core/src/main/java/org/apache/iceberg/CatalogUtil.java b/core/src/main/java/org/apache/iceberg/CatalogUtil.java index 2b400ccebc8b..e48c4a7dc905 100644 --- a/core/src/main/java/org/apache/iceberg/CatalogUtil.java +++ b/core/src/main/java/org/apache/iceberg/CatalogUtil.java @@ -314,31 +314,18 @@ public static Catalog buildIcebergCatalog(String name, Map optio if (catalogImpl == null) { String catalogType = PropertyUtil.propertyAsString(options, ICEBERG_CATALOG_TYPE, ICEBERG_CATALOG_TYPE_HIVE); - switch (catalogType.toLowerCase(Locale.ENGLISH)) { - case ICEBERG_CATALOG_TYPE_HIVE: - catalogImpl = ICEBERG_CATALOG_HIVE; - break; - case ICEBERG_CATALOG_TYPE_HADOOP: - catalogImpl = ICEBERG_CATALOG_HADOOP; - break; - case ICEBERG_CATALOG_TYPE_REST: - catalogImpl = ICEBERG_CATALOG_REST; - break; - case ICEBERG_CATALOG_TYPE_GLUE: - catalogImpl = ICEBERG_CATALOG_GLUE; - break; - case ICEBERG_CATALOG_TYPE_NESSIE: - catalogImpl = ICEBERG_CATALOG_NESSIE; - break; - case ICEBERG_CATALOG_TYPE_JDBC: - catalogImpl = ICEBERG_CATALOG_JDBC; - break; - case ICEBERG_CATALOG_TYPE_BIGQUERY: - catalogImpl = ICEBERG_CATALOG_BIGQUERY; - break; - default: - throw new UnsupportedOperationException("Unknown catalog type: " + catalogType); - } + catalogImpl = + switch (catalogType.toLowerCase(Locale.ENGLISH)) { + case ICEBERG_CATALOG_TYPE_HIVE -> ICEBERG_CATALOG_HIVE; + case ICEBERG_CATALOG_TYPE_HADOOP -> ICEBERG_CATALOG_HADOOP; + case ICEBERG_CATALOG_TYPE_REST -> ICEBERG_CATALOG_REST; + case ICEBERG_CATALOG_TYPE_GLUE -> ICEBERG_CATALOG_GLUE; + case ICEBERG_CATALOG_TYPE_NESSIE -> ICEBERG_CATALOG_NESSIE; + case ICEBERG_CATALOG_TYPE_JDBC -> ICEBERG_CATALOG_JDBC; + case ICEBERG_CATALOG_TYPE_BIGQUERY -> ICEBERG_CATALOG_BIGQUERY; + default -> + throw new UnsupportedOperationException("Unknown catalog type: " + catalogType); + }; } else { String catalogType = options.get(ICEBERG_CATALOG_TYPE); Preconditions.checkArgument( diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java b/core/src/main/java/org/apache/iceberg/ContentFileParser.java index f024a24b18ce..d626407fb1b6 100644 --- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java +++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java @@ -355,21 +355,19 @@ private static PartitionData partitionFromJson( } private static FileContent fileContentFromJson(String content) { - switch (content) { - case CONTENT_DATA: - return FileContent.DATA; - case CONTENT_POSITION_DELETES: - return FileContent.POSITION_DELETES; - case CONTENT_EQUALITY_DELETES: - return FileContent.EQUALITY_DELETES; - default: + return switch (content) { + case CONTENT_DATA -> FileContent.DATA; + case CONTENT_POSITION_DELETES -> FileContent.POSITION_DELETES; + case CONTENT_EQUALITY_DELETES -> FileContent.EQUALITY_DELETES; + default -> { // In 1.10 and before, file content is serialized as the FileContent enum value try { - return FileContent.valueOf(content); + yield FileContent.valueOf(content); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( String.format("Invalid file content value: '%s'", content), e); } - } + } + }; } } diff --git a/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java b/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java index 872fcd212b8a..9d566860089a 100644 --- a/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java +++ b/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java @@ -501,18 +501,16 @@ DeleteFileIndex build() { for (DeleteFile file : files) { switch (file.content()) { - case POSITION_DELETES: + case POSITION_DELETES -> { if (ContentFileUtil.isDV(file)) { add(dvByPath, file); } else { add(posDeletesByPath, posDeletesByPartition, file); } - break; - case EQUALITY_DELETES: - add(globalDeletes, eqDeletesByPartition, file, fieldLookup); - break; - default: - throw new UnsupportedOperationException("Unsupported content: " + file.content()); + } + case EQUALITY_DELETES -> add(globalDeletes, eqDeletesByPartition, file, fieldLookup); + default -> + throw new UnsupportedOperationException("Unsupported content: " + file.content()); } ScanMetricsUtil.indexedDeleteFile(scanMetrics, file); } diff --git a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java index 3f5be0756fad..4708942a88b9 100644 --- a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java +++ b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java @@ -90,38 +90,27 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - switch (pos) { - case 0: - return location; - case 1: - return offset; - case 2: - return sizeInBytes; - case 3: - return cardinality; - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + return switch (pos) { + case 0 -> location; + case 1 -> offset; + case 2 -> sizeInBytes; + case 3 -> cardinality; + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0: - // always coerce to String for Serializable - this.location = value.toString(); - break; - case 1: - this.offset = (Long) value; - break; - case 2: - this.sizeInBytes = (Long) value; - break; - case 3: - this.cardinality = (Long) value; - break; - default: + case 0 -> + // always coerce to String for Serializable + this.location = value.toString(); + case 1 -> this.offset = (Long) value; + case 2 -> this.sizeInBytes = (Long) value; + case 3 -> this.cardinality = (Long) value; + default -> { // ignore the object, it must be from a newer version of the format + } } } diff --git a/core/src/main/java/org/apache/iceberg/FileMetadata.java b/core/src/main/java/org/apache/iceberg/FileMetadata.java index a5266101c252..4f9ac72df2c7 100644 --- a/core/src/main/java/org/apache/iceberg/FileMetadata.java +++ b/core/src/main/java/org/apache/iceberg/FileMetadata.java @@ -272,17 +272,15 @@ public DeleteFile build() { } switch (content) { - case POSITION_DELETES: - Preconditions.checkArgument( - sortOrderId == null, "Position delete file should not have sort order"); - break; - case EQUALITY_DELETES: + case POSITION_DELETES -> + Preconditions.checkArgument( + sortOrderId == null, "Position delete file should not have sort order"); + case EQUALITY_DELETES -> { if (sortOrderId == null) { sortOrderId = SortOrder.unsorted().orderId(); } - break; - default: - throw new IllegalStateException("Unknown content type " + content); + } + default -> throw new IllegalStateException("Unknown content type " + content); } return new GenericDeleteFile( diff --git a/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java b/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java index f154c982d1c7..99f5e8ba7f5e 100644 --- a/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java +++ b/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java @@ -157,23 +157,14 @@ public void setFileSequenceNumber(long newFileSequenceNumber) { @SuppressWarnings("unchecked") public void put(int i, Object v) { switch (i) { - case 0: - this.status = Status.fromId((Integer) v); - return; - case 1: - this.snapshotId = (Long) v; - return; - case 2: - this.dataSequenceNumber = (Long) v; - return; - case 3: - this.fileSequenceNumber = (Long) v; - return; - case 4: - this.file = (F) v; - return; - default: + case 0 -> this.status = Status.fromId((Integer) v); + case 1 -> this.snapshotId = (Long) v; + case 2 -> this.dataSequenceNumber = (Long) v; + case 3 -> this.fileSequenceNumber = (Long) v; + case 4 -> this.file = (F) v; + default -> { // ignore the object, it must be from a newer version of the format + } } } @@ -184,20 +175,14 @@ public void set(int pos, T value) { @Override public Object get(int i) { - switch (i) { - case 0: - return status.id(); - case 1: - return snapshotId; - case 2: - return dataSequenceNumber; - case 3: - return fileSequenceNumber; - case 4: - return file; - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + i); - } + return switch (i) { + case 0 -> status.id(); + case 1 -> snapshotId; + case 2 -> dataSequenceNumber; + case 3 -> fileSequenceNumber; + case 4 -> file; + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + i); + }; } @Override diff --git a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java index 9624484ffe0c..2131a60b8be6 100644 --- a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java +++ b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java @@ -288,102 +288,55 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int basePos) { - switch (basePos) { - case 0: - return manifestPath; - case 1: - return lazyLength(); - case 2: - return specId; - case 3: - return content.id(); - case 4: - return sequenceNumber; - case 5: - return minSequenceNumber; - case 6: - return snapshotId; - case 7: - return addedFilesCount; - case 8: - return existingFilesCount; - case 9: - return deletedFilesCount; - case 10: - return addedRowsCount; - case 11: - return existingRowsCount; - case 12: - return deletedRowsCount; - case 13: - return partitions(); - case 14: - return keyMetadata(); - case 15: - return firstRowId(); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + basePos); - } + return switch (basePos) { + case 0 -> manifestPath; + case 1 -> lazyLength(); + case 2 -> specId; + case 3 -> content.id(); + case 4 -> sequenceNumber; + case 5 -> minSequenceNumber; + case 6 -> snapshotId; + case 7 -> addedFilesCount; + case 8 -> existingFilesCount; + case 9 -> deletedFilesCount; + case 10 -> addedRowsCount; + case 11 -> existingRowsCount; + case 12 -> deletedRowsCount; + case 13 -> partitions(); + case 14 -> keyMetadata(); + case 15 -> firstRowId(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + basePos); + }; } @Override protected void internalSet(int basePos, T value) { switch (basePos) { - case 0: - // always coerce to String for Serializable - this.manifestPath = value.toString(); - return; - case 1: - this.length = (Long) value; - return; - case 2: - this.specId = (Integer) value; - return; - case 3: - this.content = - value != null ? ManifestContent.fromId((Integer) value) : ManifestContent.DATA; - return; - case 4: - this.sequenceNumber = value != null ? (Long) value : 0; - return; - case 5: - this.minSequenceNumber = value != null ? (Long) value : 0; - return; - case 6: - this.snapshotId = (Long) value; - return; - case 7: - this.addedFilesCount = (Integer) value; - return; - case 8: - this.existingFilesCount = (Integer) value; - return; - case 9: - this.deletedFilesCount = (Integer) value; - return; - case 10: - this.addedRowsCount = (Long) value; - return; - case 11: - this.existingRowsCount = (Long) value; - return; - case 12: - this.deletedRowsCount = (Long) value; - return; - case 13: - this.partitions = - value == null - ? null - : ((List) value).toArray(new PartitionFieldSummary[0]); - return; - case 14: - this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); - return; - case 15: - this.firstRowId = (Long) value; - return; - default: + case 0 -> this.manifestPath = value.toString(); // always coerce to String for Serializable + case 1 -> this.length = (Long) value; + case 2 -> this.specId = (Integer) value; + case 3 -> + this.content = + value != null ? ManifestContent.fromId((Integer) value) : ManifestContent.DATA; + case 4 -> this.sequenceNumber = value != null ? (Long) value : 0; + case 5 -> this.minSequenceNumber = value != null ? (Long) value : 0; + case 6 -> this.snapshotId = (Long) value; + case 7 -> this.addedFilesCount = (Integer) value; + case 8 -> this.existingFilesCount = (Integer) value; + case 9 -> this.deletedFilesCount = (Integer) value; + case 10 -> this.addedRowsCount = (Long) value; + case 11 -> this.existingRowsCount = (Long) value; + case 12 -> this.deletedRowsCount = (Long) value; + case 13 -> + this.partitions = + value == null + ? null + : ((List) value).toArray(new PartitionFieldSummary[0]); + case 14 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + case 15 -> this.firstRowId = (Long) value; + default -> { // ignore the object, it must be from a newer version of the format + } } } diff --git a/core/src/main/java/org/apache/iceberg/GenericPartitionFieldSummary.java b/core/src/main/java/org/apache/iceberg/GenericPartitionFieldSummary.java index e75f37d2ec12..62e7f080eb3f 100644 --- a/core/src/main/java/org/apache/iceberg/GenericPartitionFieldSummary.java +++ b/core/src/main/java/org/apache/iceberg/GenericPartitionFieldSummary.java @@ -149,18 +149,13 @@ public Object get(int i) { if (fromProjectionPos != null) { pos = fromProjectionPos[i]; } - switch (pos) { - case 0: - return containsNull; - case 1: - return containsNaN; - case 2: - return lowerBound(); - case 3: - return upperBound(); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + return switch (pos) { + case 0 -> containsNull; + case 1 -> containsNaN; + case 2 -> lowerBound(); + case 3 -> upperBound(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -172,20 +167,13 @@ public void set(int i, T value) { pos = fromProjectionPos[i]; } switch (pos) { - case 0: - this.containsNull = (Boolean) value; - return; - case 1: - this.containsNaN = (Boolean) value; - return; - case 2: - this.lowerBound = ByteBuffers.toByteArray((ByteBuffer) value); - return; - case 3: - this.upperBound = ByteBuffers.toByteArray((ByteBuffer) value); - return; - default: + case 0 -> this.containsNull = (Boolean) value; + case 1 -> this.containsNaN = (Boolean) value; + case 2 -> this.lowerBound = ByteBuffers.toByteArray((ByteBuffer) value); + case 3 -> this.upperBound = ByteBuffers.toByteArray((ByteBuffer) value); + default -> { // ignore the object, it must be from a newer version of the format + } } } diff --git a/core/src/main/java/org/apache/iceberg/ManifestFiles.java b/core/src/main/java/org/apache/iceberg/ManifestFiles.java index dae46e5ec49e..e5ab7b94add6 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestFiles.java +++ b/core/src/main/java/org/apache/iceberg/ManifestFiles.java @@ -303,20 +303,21 @@ static ManifestWriter newWriter( Long snapshotId, Long firstRowId, Map writerProperties) { - switch (formatVersion) { - case 1: - return new ManifestWriter.V1Writer(spec, encryptedOutputFile, snapshotId, writerProperties); - case 2: - return new ManifestWriter.V2Writer(spec, encryptedOutputFile, snapshotId, writerProperties); - case 3: - return new ManifestWriter.V3Writer( - spec, encryptedOutputFile, snapshotId, firstRowId, writerProperties); - case 4: - return new ManifestWriter.V4Writer( - spec, encryptedOutputFile, snapshotId, firstRowId, writerProperties); - } - throw new UnsupportedOperationException( - "Cannot write manifest for table version: " + formatVersion); + return switch (formatVersion) { + case 1 -> + new ManifestWriter.V1Writer(spec, encryptedOutputFile, snapshotId, writerProperties); + case 2 -> + new ManifestWriter.V2Writer(spec, encryptedOutputFile, snapshotId, writerProperties); + case 3 -> + new ManifestWriter.V3Writer( + spec, encryptedOutputFile, snapshotId, firstRowId, writerProperties); + case 4 -> + new ManifestWriter.V4Writer( + spec, encryptedOutputFile, snapshotId, firstRowId, writerProperties); + default -> + throw new UnsupportedOperationException( + "Cannot write manifest for table version: " + formatVersion); + }; } /** @@ -408,18 +409,15 @@ public static ManifestWriter writeDeleteManifest( EncryptedOutputFile outputFile, Long snapshotId, Map writerProperties) { - switch (formatVersion) { - case 1: - throw new IllegalArgumentException("Cannot write delete files in a v1 table"); - case 2: - return new ManifestWriter.V2DeleteWriter(spec, outputFile, snapshotId, writerProperties); - case 3: - return new ManifestWriter.V3DeleteWriter(spec, outputFile, snapshotId, writerProperties); - case 4: - return new ManifestWriter.V4DeleteWriter(spec, outputFile, snapshotId, writerProperties); - } - throw new UnsupportedOperationException( - "Cannot write manifest for table version: " + formatVersion); + return switch (formatVersion) { + case 1 -> throw new IllegalArgumentException("Cannot write delete files in a v1 table"); + case 2 -> new ManifestWriter.V2DeleteWriter(spec, outputFile, snapshotId, writerProperties); + case 3 -> new ManifestWriter.V3DeleteWriter(spec, outputFile, snapshotId, writerProperties); + case 4 -> new ManifestWriter.V4DeleteWriter(spec, outputFile, snapshotId, writerProperties); + default -> + throw new UnsupportedOperationException( + "Cannot write manifest for table version: " + formatVersion); + }; } /** @@ -458,14 +456,10 @@ static ManifestReader open(ManifestFile manifest, FileIO io) { static ManifestReader open( ManifestFile manifest, FileIO io, Map specsById) { - switch (manifest.content()) { - case DATA: - return ManifestFiles.read(manifest, io, specsById); - case DELETES: - return ManifestFiles.readDeleteManifest(manifest, io, specsById); - } - throw new UnsupportedOperationException( - "Cannot read unknown manifest type: " + manifest.content()); + return switch (manifest.content()) { + case DATA -> ManifestFiles.read(manifest, io, specsById); + case DELETES -> ManifestFiles.readDeleteManifest(manifest, io, specsById); + }; } static ManifestFile copyAppendManifest( @@ -543,17 +537,15 @@ private static ManifestFile copyManifestInternal( entry.status(), allowedEntryStatus); switch (entry.status()) { - case ADDED: + case ADDED -> { summaryBuilder.addedFile(reader.spec(), entry.file()); writer.add(entry); - break; - case EXISTING: - writer.existing(entry); - break; - case DELETED: + } + case EXISTING -> writer.existing(entry); + case DELETED -> { summaryBuilder.deletedFile(reader.spec(), entry.file()); writer.delete(entry); - break; + } } } diff --git a/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java b/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java index 6a7ccea6b679..b598b97069b4 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java +++ b/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java @@ -166,72 +166,39 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - switch (pos) { - case 0: - return addedFilesCount; - case 1: - return existingFilesCount; - case 2: - return deletedFilesCount; - case 3: - return replacedFilesCount; - case 4: - return addedRowsCount; - case 5: - return existingRowsCount; - case 6: - return deletedRowsCount; - case 7: - return replacedRowsCount; - case 8: - return minSequenceNumber; - case 9: - return dv(); - case 10: - return dvCardinality; - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + return switch (pos) { + case 0 -> addedFilesCount; + case 1 -> existingFilesCount; + case 2 -> deletedFilesCount; + case 3 -> replacedFilesCount; + case 4 -> addedRowsCount; + case 5 -> existingRowsCount; + case 6 -> deletedRowsCount; + case 7 -> replacedRowsCount; + case 8 -> minSequenceNumber; + case 9 -> dv(); + case 10 -> dvCardinality; + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0: - this.addedFilesCount = (Integer) value; - break; - case 1: - this.existingFilesCount = (Integer) value; - break; - case 2: - this.deletedFilesCount = (Integer) value; - break; - case 3: - this.replacedFilesCount = (Integer) value; - break; - case 4: - this.addedRowsCount = (Long) value; - break; - case 5: - this.existingRowsCount = (Long) value; - break; - case 6: - this.deletedRowsCount = (Long) value; - break; - case 7: - this.replacedRowsCount = (Long) value; - break; - case 8: - this.minSequenceNumber = (Long) value; - break; - case 9: - this.dv = ByteBuffers.toByteArray((ByteBuffer) value); - break; - case 10: - this.dvCardinality = (Long) value; - break; - default: + case 0 -> this.addedFilesCount = (Integer) value; + case 1 -> this.existingFilesCount = (Integer) value; + case 2 -> this.deletedFilesCount = (Integer) value; + case 3 -> this.replacedFilesCount = (Integer) value; + case 4 -> this.addedRowsCount = (Long) value; + case 5 -> this.existingRowsCount = (Long) value; + case 6 -> this.deletedRowsCount = (Long) value; + case 7 -> this.replacedRowsCount = (Long) value; + case 8 -> this.minSequenceNumber = (Long) value; + case 9 -> this.dv = ByteBuffers.toByteArray((ByteBuffer) value); + case 10 -> this.dvCardinality = (Long) value; + default -> { // ignore the object, it must be from a newer version of the format + } } } diff --git a/core/src/main/java/org/apache/iceberg/ManifestLists.java b/core/src/main/java/org/apache/iceberg/ManifestLists.java index dbe080584b13..b1deea83ecf6 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestLists.java +++ b/core/src/main/java/org/apache/iceberg/ManifestLists.java @@ -66,35 +66,37 @@ static ManifestListWriter write( Long parentSnapshotId, long sequenceNumber, Long firstRowId) { - switch (formatVersion) { - case 1: + return switch (formatVersion) { + case 1 -> { Preconditions.checkArgument( sequenceNumber == TableMetadata.INITIAL_SEQUENCE_NUMBER, "Invalid sequence number for v1 manifest list: %s", sequenceNumber); - return new ManifestListWriter.V1Writer( + yield new ManifestListWriter.V1Writer( manifestListFile, encryptionManager, snapshotId, parentSnapshotId); - case 2: - return new ManifestListWriter.V2Writer( - manifestListFile, encryptionManager, snapshotId, parentSnapshotId, sequenceNumber); - case 3: - return new ManifestListWriter.V3Writer( - manifestListFile, - encryptionManager, - snapshotId, - parentSnapshotId, - sequenceNumber, - firstRowId); - case 4: - return new ManifestListWriter.V4Writer( - manifestListFile, - encryptionManager, - snapshotId, - parentSnapshotId, - sequenceNumber, - firstRowId); - } - throw new UnsupportedOperationException( - "Cannot write manifest list for table version: " + formatVersion); + } + case 2 -> + new ManifestListWriter.V2Writer( + manifestListFile, encryptionManager, snapshotId, parentSnapshotId, sequenceNumber); + case 3 -> + new ManifestListWriter.V3Writer( + manifestListFile, + encryptionManager, + snapshotId, + parentSnapshotId, + sequenceNumber, + firstRowId); + case 4 -> + new ManifestListWriter.V4Writer( + manifestListFile, + encryptionManager, + snapshotId, + parentSnapshotId, + sequenceNumber, + firstRowId); + default -> + throw new UnsupportedOperationException( + "Cannot write manifest list for table version: " + formatVersion); + }; } } diff --git a/core/src/main/java/org/apache/iceberg/ManifestWriter.java b/core/src/main/java/org/apache/iceberg/ManifestWriter.java index 321bcd89d8b1..d1524fbfd4a1 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestWriter.java +++ b/core/src/main/java/org/apache/iceberg/ManifestWriter.java @@ -110,18 +110,18 @@ protected ManifestContent content() { void addEntry(ManifestEntry entry) { switch (entry.status()) { - case ADDED: + case ADDED -> { addedFiles += 1; addedRows += entry.file().recordCount(); - break; - case EXISTING: + } + case EXISTING -> { existingFiles += 1; existingRows += entry.file().recordCount(); - break; - case DELETED: + } + case DELETED -> { deletedFiles += 1; deletedRows += entry.file().recordCount(); - break; + } } stats.update(entry.file().partition()); diff --git a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java index 1a70b4f90b8f..a4d45fd640f6 100644 --- a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java @@ -294,24 +294,19 @@ protected void validateNewDeleteFile(DeleteFile file) { private static void validateDeleteFileForVersion(DeleteFile file, int formatVersion) { switch (formatVersion) { - case 1: - throw new IllegalArgumentException("Deletes are supported in V2 and above"); - case 2: - Preconditions.checkArgument( - file.content() == FileContent.EQUALITY_DELETES || !ContentFileUtil.isDV(file), - "Must not use DVs for position deletes in V2: %s", - ContentFileUtil.dvDesc(file)); - break; - case 3: - case 4: - Preconditions.checkArgument( - file.content() == FileContent.EQUALITY_DELETES || ContentFileUtil.isDV(file), - "Must use DVs for position deletes in V%s: %s", - formatVersion, - file.location()); - break; - default: - throw new IllegalArgumentException("Unsupported format version: " + formatVersion); + case 1 -> throw new IllegalArgumentException("Deletes are supported in V2 and above"); + case 2 -> + Preconditions.checkArgument( + file.content() == FileContent.EQUALITY_DELETES || !ContentFileUtil.isDV(file), + "Must not use DVs for position deletes in V2: %s", + ContentFileUtil.dvDesc(file)); + case 3, 4 -> + Preconditions.checkArgument( + file.content() == FileContent.EQUALITY_DELETES || ContentFileUtil.isDV(file), + "Must use DVs for position deletes in V%s: %s", + formatVersion, + file.location()); + default -> throw new IllegalArgumentException("Unsupported format version: " + formatVersion); } } diff --git a/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java b/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java index adb0f18ba1ad..735d62362a4c 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java +++ b/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java @@ -20,7 +20,6 @@ import java.util.Locale; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.NoSuchTableException; public class MetadataTableUtils { private MetadataTableUtils() {} @@ -55,43 +54,24 @@ public static Table createMetadataTableInstance( private static Table createMetadataTableInstance( Table baseTable, String metadataTableName, MetadataTableType type) { - switch (type) { - case ENTRIES: - return new ManifestEntriesTable(baseTable, metadataTableName); - case FILES: - return new FilesTable(baseTable, metadataTableName); - case DATA_FILES: - return new DataFilesTable(baseTable, metadataTableName); - case DELETE_FILES: - return new DeleteFilesTable(baseTable, metadataTableName); - case HISTORY: - return new HistoryTable(baseTable, metadataTableName); - case SNAPSHOTS: - return new SnapshotsTable(baseTable, metadataTableName); - case METADATA_LOG_ENTRIES: - return new MetadataLogEntriesTable(baseTable, metadataTableName); - case REFS: - return new RefsTable(baseTable, metadataTableName); - case MANIFESTS: - return new ManifestsTable(baseTable, metadataTableName); - case PARTITIONS: - return new PartitionsTable(baseTable, metadataTableName); - case ALL_DATA_FILES: - return new AllDataFilesTable(baseTable, metadataTableName); - case ALL_DELETE_FILES: - return new AllDeleteFilesTable(baseTable, metadataTableName); - case ALL_FILES: - return new AllFilesTable(baseTable, metadataTableName); - case ALL_MANIFESTS: - return new AllManifestsTable(baseTable, metadataTableName); - case ALL_ENTRIES: - return new AllEntriesTable(baseTable, metadataTableName); - case POSITION_DELETES: - return new PositionDeletesTable(baseTable, metadataTableName); - default: - throw new NoSuchTableException( - "Unknown metadata table type: %s for %s", type, metadataTableName); - } + return switch (type) { + case ENTRIES -> new ManifestEntriesTable(baseTable, metadataTableName); + case FILES -> new FilesTable(baseTable, metadataTableName); + case DATA_FILES -> new DataFilesTable(baseTable, metadataTableName); + case DELETE_FILES -> new DeleteFilesTable(baseTable, metadataTableName); + case HISTORY -> new HistoryTable(baseTable, metadataTableName); + case SNAPSHOTS -> new SnapshotsTable(baseTable, metadataTableName); + case METADATA_LOG_ENTRIES -> new MetadataLogEntriesTable(baseTable, metadataTableName); + case REFS -> new RefsTable(baseTable, metadataTableName); + case MANIFESTS -> new ManifestsTable(baseTable, metadataTableName); + case PARTITIONS -> new PartitionsTable(baseTable, metadataTableName); + case ALL_DATA_FILES -> new AllDataFilesTable(baseTable, metadataTableName); + case ALL_DELETE_FILES -> new AllDeleteFilesTable(baseTable, metadataTableName); + case ALL_FILES -> new AllFilesTable(baseTable, metadataTableName); + case ALL_MANIFESTS -> new AllManifestsTable(baseTable, metadataTableName); + case ALL_ENTRIES -> new AllEntriesTable(baseTable, metadataTableName); + case POSITION_DELETES -> new PositionDeletesTable(baseTable, metadataTableName); + }; } public static Table createMetadataTableInstance( diff --git a/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java b/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java index 280fe7565d75..a0490b3c5d8e 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java +++ b/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java @@ -192,89 +192,62 @@ public static void toJson(MetadataUpdate metadataUpdate, JsonGenerator generator generator.writeStringField(ACTION, updateAction); switch (updateAction) { - case ASSIGN_UUID: - writeAssignUUID((MetadataUpdate.AssignUUID) metadataUpdate, generator); - break; - case UPGRADE_FORMAT_VERSION: - writeUpgradeFormatVersion((MetadataUpdate.UpgradeFormatVersion) metadataUpdate, generator); - break; - case ADD_SCHEMA: - writeAddSchema((MetadataUpdate.AddSchema) metadataUpdate, generator); - break; - case SET_CURRENT_SCHEMA: - writeSetCurrentSchema((MetadataUpdate.SetCurrentSchema) metadataUpdate, generator); - break; - case ADD_PARTITION_SPEC: - writeAddPartitionSpec((MetadataUpdate.AddPartitionSpec) metadataUpdate, generator); - break; - case SET_DEFAULT_PARTITION_SPEC: - writeSetDefaultPartitionSpec( - (MetadataUpdate.SetDefaultPartitionSpec) metadataUpdate, generator); - break; - case ADD_SORT_ORDER: - writeAddSortOrder((MetadataUpdate.AddSortOrder) metadataUpdate, generator); - break; - case SET_DEFAULT_SORT_ORDER: - writeSetDefaultSortOrder((MetadataUpdate.SetDefaultSortOrder) metadataUpdate, generator); - break; - case SET_STATISTICS: - writeSetStatistics((MetadataUpdate.SetStatistics) metadataUpdate, generator); - break; - case REMOVE_STATISTICS: - writeRemoveStatistics((MetadataUpdate.RemoveStatistics) metadataUpdate, generator); - break; - case SET_PARTITION_STATISTICS: - writeSetPartitionStatistics( - (MetadataUpdate.SetPartitionStatistics) metadataUpdate, generator); - break; - case REMOVE_PARTITION_STATISTICS: - writeRemovePartitionStatistics( - (MetadataUpdate.RemovePartitionStatistics) metadataUpdate, generator); - break; - case ADD_SNAPSHOT: - writeAddSnapshot((MetadataUpdate.AddSnapshot) metadataUpdate, generator); - break; - case REMOVE_SNAPSHOTS: - writeRemoveSnapshots((MetadataUpdate.RemoveSnapshots) metadataUpdate, generator); - break; - case REMOVE_SNAPSHOT_REF: - writeRemoveSnapshotRef((MetadataUpdate.RemoveSnapshotRef) metadataUpdate, generator); - break; - case SET_SNAPSHOT_REF: - writeSetSnapshotRef((MetadataUpdate.SetSnapshotRef) metadataUpdate, generator); - break; - case SET_PROPERTIES: - writeSetProperties((MetadataUpdate.SetProperties) metadataUpdate, generator); - break; - case REMOVE_PROPERTIES: - writeRemoveProperties((MetadataUpdate.RemoveProperties) metadataUpdate, generator); - break; - case SET_LOCATION: - writeSetLocation((MetadataUpdate.SetLocation) metadataUpdate, generator); - break; - case ADD_VIEW_VERSION: - writeAddViewVersion((MetadataUpdate.AddViewVersion) metadataUpdate, generator); - break; - case SET_CURRENT_VIEW_VERSION: - writeSetCurrentViewVersionId( - (MetadataUpdate.SetCurrentViewVersion) metadataUpdate, generator); - break; - case REMOVE_PARTITION_SPECS: - writeRemovePartitionSpecs((MetadataUpdate.RemovePartitionSpecs) metadataUpdate, generator); - break; - case REMOVE_SCHEMAS: - writeRemoveSchemas((MetadataUpdate.RemoveSchemas) metadataUpdate, generator); - break; - case ADD_ENCRYPTION_KEY: - writeAddEncryptionKey((MetadataUpdate.AddEncryptionKey) metadataUpdate, generator); - break; - case REMOVE_ENCRYPTION_KEY: - writeRemoveEncryptionKey((MetadataUpdate.RemoveEncryptionKey) metadataUpdate, generator); - break; - default: - throw new IllegalArgumentException( - String.format( - "Cannot convert metadata update to json. Unrecognized action: %s", updateAction)); + case ASSIGN_UUID -> writeAssignUUID((MetadataUpdate.AssignUUID) metadataUpdate, generator); + case UPGRADE_FORMAT_VERSION -> + writeUpgradeFormatVersion( + (MetadataUpdate.UpgradeFormatVersion) metadataUpdate, generator); + case ADD_SCHEMA -> writeAddSchema((MetadataUpdate.AddSchema) metadataUpdate, generator); + case SET_CURRENT_SCHEMA -> + writeSetCurrentSchema((MetadataUpdate.SetCurrentSchema) metadataUpdate, generator); + case ADD_PARTITION_SPEC -> + writeAddPartitionSpec((MetadataUpdate.AddPartitionSpec) metadataUpdate, generator); + case SET_DEFAULT_PARTITION_SPEC -> + writeSetDefaultPartitionSpec( + (MetadataUpdate.SetDefaultPartitionSpec) metadataUpdate, generator); + case ADD_SORT_ORDER -> + writeAddSortOrder((MetadataUpdate.AddSortOrder) metadataUpdate, generator); + case SET_DEFAULT_SORT_ORDER -> + writeSetDefaultSortOrder((MetadataUpdate.SetDefaultSortOrder) metadataUpdate, generator); + case SET_STATISTICS -> + writeSetStatistics((MetadataUpdate.SetStatistics) metadataUpdate, generator); + case REMOVE_STATISTICS -> + writeRemoveStatistics((MetadataUpdate.RemoveStatistics) metadataUpdate, generator); + case SET_PARTITION_STATISTICS -> + writeSetPartitionStatistics( + (MetadataUpdate.SetPartitionStatistics) metadataUpdate, generator); + case REMOVE_PARTITION_STATISTICS -> + writeRemovePartitionStatistics( + (MetadataUpdate.RemovePartitionStatistics) metadataUpdate, generator); + case ADD_SNAPSHOT -> writeAddSnapshot((MetadataUpdate.AddSnapshot) metadataUpdate, generator); + case REMOVE_SNAPSHOTS -> + writeRemoveSnapshots((MetadataUpdate.RemoveSnapshots) metadataUpdate, generator); + case REMOVE_SNAPSHOT_REF -> + writeRemoveSnapshotRef((MetadataUpdate.RemoveSnapshotRef) metadataUpdate, generator); + case SET_SNAPSHOT_REF -> + writeSetSnapshotRef((MetadataUpdate.SetSnapshotRef) metadataUpdate, generator); + case SET_PROPERTIES -> + writeSetProperties((MetadataUpdate.SetProperties) metadataUpdate, generator); + case REMOVE_PROPERTIES -> + writeRemoveProperties((MetadataUpdate.RemoveProperties) metadataUpdate, generator); + case SET_LOCATION -> writeSetLocation((MetadataUpdate.SetLocation) metadataUpdate, generator); + case ADD_VIEW_VERSION -> + writeAddViewVersion((MetadataUpdate.AddViewVersion) metadataUpdate, generator); + case SET_CURRENT_VIEW_VERSION -> + writeSetCurrentViewVersionId( + (MetadataUpdate.SetCurrentViewVersion) metadataUpdate, generator); + case REMOVE_PARTITION_SPECS -> + writeRemovePartitionSpecs( + (MetadataUpdate.RemovePartitionSpecs) metadataUpdate, generator); + case REMOVE_SCHEMAS -> + writeRemoveSchemas((MetadataUpdate.RemoveSchemas) metadataUpdate, generator); + case ADD_ENCRYPTION_KEY -> + writeAddEncryptionKey((MetadataUpdate.AddEncryptionKey) metadataUpdate, generator); + case REMOVE_ENCRYPTION_KEY -> + writeRemoveEncryptionKey((MetadataUpdate.RemoveEncryptionKey) metadataUpdate, generator); + default -> + throw new IllegalArgumentException( + String.format( + "Cannot convert metadata update to json. Unrecognized action: %s", updateAction)); } generator.writeEndObject(); @@ -299,61 +272,36 @@ public static MetadataUpdate fromJson(JsonNode jsonNode) { jsonNode.hasNonNull(ACTION), "Cannot parse metadata update. Missing field: action"); String action = JsonUtil.getString(ACTION, jsonNode).toLowerCase(Locale.ROOT); - switch (action) { - case ASSIGN_UUID: - return readAssignUUID(jsonNode); - case UPGRADE_FORMAT_VERSION: - return readUpgradeFormatVersion(jsonNode); - case ADD_SCHEMA: - return readAddSchema(jsonNode); - case SET_CURRENT_SCHEMA: - return readSetCurrentSchema(jsonNode); - case ADD_PARTITION_SPEC: - return readAddPartitionSpec(jsonNode); - case SET_DEFAULT_PARTITION_SPEC: - return readSetDefaultPartitionSpec(jsonNode); - case ADD_SORT_ORDER: - return readAddSortOrder(jsonNode); - case SET_DEFAULT_SORT_ORDER: - return readSetDefaultSortOrder(jsonNode); - case SET_STATISTICS: - return readSetStatistics(jsonNode); - case REMOVE_STATISTICS: - return readRemoveStatistics(jsonNode); - case SET_PARTITION_STATISTICS: - return readSetPartitionStatistics(jsonNode); - case REMOVE_PARTITION_STATISTICS: - return readRemovePartitionStatistics(jsonNode); - case ADD_SNAPSHOT: - return readAddSnapshot(jsonNode); - case REMOVE_SNAPSHOTS: - return readRemoveSnapshots(jsonNode); - case REMOVE_SNAPSHOT_REF: - return readRemoveSnapshotRef(jsonNode); - case SET_SNAPSHOT_REF: - return readSetSnapshotRef(jsonNode); - case SET_PROPERTIES: - return readSetProperties(jsonNode); - case REMOVE_PROPERTIES: - return readRemoveProperties(jsonNode); - case SET_LOCATION: - return readSetLocation(jsonNode); - case ADD_VIEW_VERSION: - return readAddViewVersion(jsonNode); - case SET_CURRENT_VIEW_VERSION: - return readCurrentViewVersionId(jsonNode); - case REMOVE_PARTITION_SPECS: - return readRemovePartitionSpecs(jsonNode); - case REMOVE_SCHEMAS: - return readRemoveSchemas(jsonNode); - case ADD_ENCRYPTION_KEY: - return readAddEncryptionKey(jsonNode); - case REMOVE_ENCRYPTION_KEY: - return readRemoveEncryptionKey(jsonNode); - default: - throw new UnsupportedOperationException( - String.format("Cannot convert metadata update action to json: %s", action)); - } + return switch (action) { + case ASSIGN_UUID -> readAssignUUID(jsonNode); + case UPGRADE_FORMAT_VERSION -> readUpgradeFormatVersion(jsonNode); + case ADD_SCHEMA -> readAddSchema(jsonNode); + case SET_CURRENT_SCHEMA -> readSetCurrentSchema(jsonNode); + case ADD_PARTITION_SPEC -> readAddPartitionSpec(jsonNode); + case SET_DEFAULT_PARTITION_SPEC -> readSetDefaultPartitionSpec(jsonNode); + case ADD_SORT_ORDER -> readAddSortOrder(jsonNode); + case SET_DEFAULT_SORT_ORDER -> readSetDefaultSortOrder(jsonNode); + case SET_STATISTICS -> readSetStatistics(jsonNode); + case REMOVE_STATISTICS -> readRemoveStatistics(jsonNode); + case SET_PARTITION_STATISTICS -> readSetPartitionStatistics(jsonNode); + case REMOVE_PARTITION_STATISTICS -> readRemovePartitionStatistics(jsonNode); + case ADD_SNAPSHOT -> readAddSnapshot(jsonNode); + case REMOVE_SNAPSHOTS -> readRemoveSnapshots(jsonNode); + case REMOVE_SNAPSHOT_REF -> readRemoveSnapshotRef(jsonNode); + case SET_SNAPSHOT_REF -> readSetSnapshotRef(jsonNode); + case SET_PROPERTIES -> readSetProperties(jsonNode); + case REMOVE_PROPERTIES -> readRemoveProperties(jsonNode); + case SET_LOCATION -> readSetLocation(jsonNode); + case ADD_VIEW_VERSION -> readAddViewVersion(jsonNode); + case SET_CURRENT_VIEW_VERSION -> readCurrentViewVersionId(jsonNode); + case REMOVE_PARTITION_SPECS -> readRemovePartitionSpecs(jsonNode); + case REMOVE_SCHEMAS -> readRemoveSchemas(jsonNode); + case ADD_ENCRYPTION_KEY -> readAddEncryptionKey(jsonNode); + case REMOVE_ENCRYPTION_KEY -> readRemoveEncryptionKey(jsonNode); + default -> + throw new UnsupportedOperationException( + String.format("Cannot convert metadata update action to json: %s", action)); + }; } private static void writeAssignUUID(MetadataUpdate.AssignUUID update, JsonGenerator gen) diff --git a/core/src/main/java/org/apache/iceberg/PartitionData.java b/core/src/main/java/org/apache/iceberg/PartitionData.java index 41ad72bf0bac..1f95bf820bdb 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionData.java +++ b/core/src/main/java/org/apache/iceberg/PartitionData.java @@ -203,21 +203,16 @@ public static Object[] copyData(Types.StructType type, Object[] data) { } else { Types.NestedField field = fields.get(i); switch (field.type().typeId()) { - case STRUCT: - case LIST: - case MAP: - throw new IllegalArgumentException("Unsupported type in partition data: " + type); - case BINARY: - case FIXED: + case STRUCT, LIST, MAP -> + throw new IllegalArgumentException("Unsupported type in partition data: " + type); + case BINARY, FIXED -> { byte[] buffer = (byte[]) data[i]; copy[i] = Arrays.copyOf(buffer, buffer.length); - break; - case STRING: - copy[i] = data[i].toString(); - break; - default: - // no need to copy the object - copy[i] = data[i]; + } + case STRING -> copy[i] = data[i].toString(); + default -> + // no need to copy the object + copy[i] = data[i]; } } } diff --git a/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java b/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java index 611bd3cd0783..cff78ecd7b71 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java +++ b/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java @@ -355,7 +355,7 @@ private static void liveEntry(PartitionStatistics stats, ContentFile file, Sn Preconditions.checkArgument(stats.specId() == file.specId(), "Spec IDs must match"); switch (file.content()) { - case DATA: + case DATA -> { stats.set( PartitionStatistics.DATA_RECORD_COUNT_POSITION, stats.dataRecordCount() + file.recordCount()); @@ -363,8 +363,8 @@ private static void liveEntry(PartitionStatistics stats, ContentFile file, Sn stats.set( PartitionStatistics.TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION, stats.totalDataFileSizeInBytes() + file.fileSizeInBytes()); - break; - case POSITION_DELETES: + } + case POSITION_DELETES -> { stats.set( PartitionStatistics.POSITION_DELETE_RECORD_COUNT_POSITION, stats.positionDeleteRecordCount() + file.recordCount()); @@ -375,18 +375,18 @@ private static void liveEntry(PartitionStatistics stats, ContentFile file, Sn PartitionStatistics.POSITION_DELETE_FILE_COUNT_POSITION, stats.positionDeleteFileCount() + 1); } - - break; - case EQUALITY_DELETES: + } + case EQUALITY_DELETES -> { stats.set( PartitionStatistics.EQUALITY_DELETE_RECORD_COUNT_POSITION, stats.equalityDeleteRecordCount() + file.recordCount()); stats.set( PartitionStatistics.EQUALITY_DELETE_FILE_COUNT_POSITION, stats.equalityDeleteFileCount() + 1); - break; - default: - throw new UnsupportedOperationException("Unsupported file content type: " + file.content()); + } + default -> + throw new UnsupportedOperationException( + "Unsupported file content type: " + file.content()); } if (snapshot != null) { @@ -420,7 +420,7 @@ private static void deletedEntryForIncrementalCompute( Preconditions.checkArgument(stats.specId() == file.specId(), "Spec IDs must match"); switch (file.content()) { - case DATA: + case DATA -> { stats.set( PartitionStatistics.DATA_RECORD_COUNT_POSITION, stats.dataRecordCount() - file.recordCount()); @@ -428,8 +428,8 @@ private static void deletedEntryForIncrementalCompute( stats.set( PartitionStatistics.TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION, stats.totalDataFileSizeInBytes() - file.fileSizeInBytes()); - break; - case POSITION_DELETES: + } + case POSITION_DELETES -> { stats.set( PartitionStatistics.POSITION_DELETE_RECORD_COUNT_POSITION, stats.positionDeleteRecordCount() - file.recordCount()); @@ -440,18 +440,18 @@ private static void deletedEntryForIncrementalCompute( PartitionStatistics.POSITION_DELETE_FILE_COUNT_POSITION, stats.positionDeleteFileCount() - 1); } - - break; - case EQUALITY_DELETES: + } + case EQUALITY_DELETES -> { stats.set( PartitionStatistics.EQUALITY_DELETE_RECORD_COUNT_POSITION, stats.equalityDeleteRecordCount() - file.recordCount()); stats.set( PartitionStatistics.EQUALITY_DELETE_FILE_COUNT_POSITION, stats.equalityDeleteFileCount() - 1); - break; - default: - throw new UnsupportedOperationException("Unsupported file content type: " + file.content()); + } + default -> + throw new UnsupportedOperationException( + "Unsupported file content type: " + file.content()); } if (snapshot != null) { diff --git a/core/src/main/java/org/apache/iceberg/PartitionsTable.java b/core/src/main/java/org/apache/iceberg/PartitionsTable.java index b5dd5c284ce2..b3506ffbde9c 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionsTable.java +++ b/core/src/main/java/org/apache/iceberg/PartitionsTable.java @@ -340,22 +340,22 @@ void update(ContentFile file, Snapshot snapshot) { } switch (file.content()) { - case DATA: + case DATA -> { this.dataRecordCount += file.recordCount(); this.dataFileCount += 1; this.dataFileSizeInBytes += file.fileSizeInBytes(); - break; - case POSITION_DELETES: + } + case POSITION_DELETES -> { this.posDeleteRecordCount += file.recordCount(); this.posDeleteFileCount += 1; - break; - case EQUALITY_DELETES: + } + case EQUALITY_DELETES -> { this.eqDeleteRecordCount += file.recordCount(); this.eqDeleteFileCount += 1; - break; - default: - throw new UnsupportedOperationException( - "Unsupported file content type: " + file.content()); + } + default -> + throw new UnsupportedOperationException( + "Unsupported file content type: " + file.content()); } } diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java index 69f82931833e..a091d349d096 100644 --- a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java +++ b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java @@ -451,7 +451,7 @@ private static RewriteResult writeDeleteFileEntry( RewriteResult result = new RewriteResult<>(); switch (file.content()) { - case POSITION_DELETES: + case POSITION_DELETES -> { DeleteFile posDeleteFile = newPositionDeleteEntry(file, spec, sourcePrefix, targetPrefix); appendEntryWithFile(entry, writer, posDeleteFile); // keep the following entries in metadata but exclude them from copyPlan @@ -467,7 +467,8 @@ private static RewriteResult writeDeleteFileEntry( } result.toRewrite().add(file.copy()); return result; - case EQUALITY_DELETES: + } + case EQUALITY_DELETES -> { DeleteFile eqDeleteFile = newEqualityDeleteEntry(file, spec, sourcePrefix, targetPrefix); appendEntryWithFile(entry, writer, eqDeleteFile); // keep the following entries in metadata but exclude them from copyPlan @@ -478,9 +479,10 @@ private static RewriteResult writeDeleteFileEntry( result.copyPlan().add(Pair.of(file.location(), eqDeleteFile.location())); } return result; - - default: - throw new UnsupportedOperationException("Unsupported delete file type: " + file.content()); + } + default -> + throw new UnsupportedOperationException( + "Unsupported delete file type: " + file.content()); } } @@ -488,16 +490,11 @@ private static > void appendEntryWithFile( ManifestEntry entry, ManifestWriter writer, F file) { switch (entry.status()) { - case ADDED: - writer.add(file); - break; - case EXISTING: - writer.existing( - file, entry.snapshotId(), entry.dataSequenceNumber(), entry.fileSequenceNumber()); - break; - case DELETED: - writer.delete(file, entry.dataSequenceNumber(), entry.fileSequenceNumber()); - break; + case ADDED -> writer.add(file); + case EXISTING -> + writer.existing( + file, entry.snapshotId(), entry.dataSequenceNumber(), entry.fileSequenceNumber()); + case DELETED -> writer.delete(file, entry.dataSequenceNumber(), entry.fileSequenceNumber()); } } diff --git a/core/src/main/java/org/apache/iceberg/ScanSummary.java b/core/src/main/java/org/apache/iceberg/ScanSummary.java index 5f8e66c0b450..99b5bd87c6f9 100644 --- a/core/src/main/java/org/apache/iceberg/ScanSummary.java +++ b/core/src/main/java/org/apache/iceberg/ScanSummary.java @@ -388,37 +388,37 @@ static Pair timestampRange(List> timeFilters) for (UnboundPredicate pred : timeFilters) { long value = pred.literal().value(); switch (pred.op()) { - case LT: + case LT -> { if (value - 1 < maxTimestamp) { maxTimestamp = value - 1; } - break; - case LT_EQ: + } + case LT_EQ -> { if (value < maxTimestamp) { maxTimestamp = value; } - break; - case GT: + } + case GT -> { if (value + 1 > minTimestamp) { minTimestamp = value + 1; } - break; - case GT_EQ: + } + case GT_EQ -> { if (value > minTimestamp) { minTimestamp = value; } - break; - case EQ: + } + case EQ -> { if (value < maxTimestamp) { maxTimestamp = value; } if (value > minTimestamp) { minTimestamp = value; } - break; - default: - throw new UnsupportedOperationException( - "Cannot filter timestamps using predicate: " + pred); + } + default -> + throw new UnsupportedOperationException( + "Cannot filter timestamps using predicate: " + pred); } } diff --git a/core/src/main/java/org/apache/iceberg/ScanTaskParser.java b/core/src/main/java/org/apache/iceberg/ScanTaskParser.java index 67e44cea7d07..cf9faefa247d 100644 --- a/core/src/main/java/org/apache/iceberg/ScanTaskParser.java +++ b/core/src/main/java/org/apache/iceberg/ScanTaskParser.java @@ -113,19 +113,12 @@ private static FileScanTask fromJson(JsonNode jsonNode, boolean caseSensitive) { taskType = TaskType.fromTypeName(taskTypeStr); } - switch (taskType) { - case FILE_SCAN_TASK: - return FileScanTaskParser.fromJson(jsonNode, caseSensitive); - case DATA_TASK: - return DataTaskParser.fromJson(jsonNode); - case FILES_TABLE_TASK: - return FilesTableTaskParser.fromJson(jsonNode); - case ALL_MANIFESTS_TABLE_TASK: - return AllManifestsTableTaskParser.fromJson(jsonNode); - case MANIFEST_ENTRIES_TABLE_TASK: - return ManifestEntriesTableTaskParser.fromJson(jsonNode); - default: - throw new UnsupportedOperationException("Unsupported task type: " + taskType.typeName()); - } + return switch (taskType) { + case FILE_SCAN_TASK -> FileScanTaskParser.fromJson(jsonNode, caseSensitive); + case DATA_TASK -> DataTaskParser.fromJson(jsonNode); + case FILES_TABLE_TASK -> FilesTableTaskParser.fromJson(jsonNode); + case ALL_MANIFESTS_TABLE_TASK -> AllManifestsTableTaskParser.fromJson(jsonNode); + case MANIFEST_ENTRIES_TABLE_TASK -> ManifestEntriesTableTaskParser.fromJson(jsonNode); + }; } } diff --git a/core/src/main/java/org/apache/iceberg/SchemaParser.java b/core/src/main/java/org/apache/iceberg/SchemaParser.java index 7481af0284f6..6b31f0cc9c0d 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaParser.java +++ b/core/src/main/java/org/apache/iceberg/SchemaParser.java @@ -146,17 +146,10 @@ static void toJson(Type type, JsonGenerator generator) throws IOException { } else { Type.NestedType nested = type.asNestedType(); switch (type.typeId()) { - case STRUCT: - toJson(nested.asStructType(), generator); - break; - case LIST: - toJson(nested.asListType(), generator); - break; - case MAP: - toJson(nested.asMapType(), generator); - break; - default: - throw new IllegalArgumentException("Cannot write unknown type: " + type); + case STRUCT -> toJson(nested.asStructType(), generator); + case LIST -> toJson(nested.asListType(), generator); + case MAP -> toJson(nested.asMapType(), generator); + default -> throw new IllegalArgumentException("Cannot write unknown type: " + type); } } } diff --git a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java index 1fa6ebbe8fef..1cb2d020ba4c 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java +++ b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java @@ -790,27 +790,21 @@ private static List moveFields( reordered.remove(toMove); switch (move.type()) { - case FIRST: - reordered.addFirst(toMove); - break; - - case BEFORE: + case FIRST -> reordered.addFirst(toMove); + case BEFORE -> { Types.NestedField before = Iterables.find(reordered, field -> field.fieldId() == move.referenceFieldId()); int beforeIndex = reordered.indexOf(before); // insert the new node at the index of the existing node reordered.add(beforeIndex, toMove); - break; - - case AFTER: + } + case AFTER -> { Types.NestedField after = Iterables.find(reordered, field -> field.fieldId() == move.referenceFieldId()); int afterIndex = reordered.indexOf(after); reordered.add(afterIndex + 1, toMove); - break; - - default: - throw new UnsupportedOperationException("Unknown move type: " + move.type()); + } + default -> throw new UnsupportedOperationException("Unknown move type: " + move.type()); } } diff --git a/core/src/main/java/org/apache/iceberg/SingleValueParser.java b/core/src/main/java/org/apache/iceberg/SingleValueParser.java index c7f07ea1a2d4..0da8f4177532 100644 --- a/core/src/main/java/org/apache/iceberg/SingleValueParser.java +++ b/core/src/main/java/org/apache/iceberg/SingleValueParser.java @@ -46,45 +46,51 @@ private SingleValueParser() {} private static final String KEYS = "keys"; private static final String VALUES = "values"; + @SuppressWarnings("MethodLength") public static Object fromJson(Type type, JsonNode defaultValue) { if (defaultValue == null || defaultValue.isNull()) { return null; } - switch (type.typeId()) { - case BOOLEAN: + return switch (type.typeId()) { + case BOOLEAN -> { Preconditions.checkArgument( defaultValue.isBoolean(), "Cannot parse default as a %s value: %s", type, defaultValue); - return defaultValue.booleanValue(); - case INTEGER: + yield defaultValue.booleanValue(); + } + case INTEGER -> { Preconditions.checkArgument( defaultValue.isIntegralNumber() && defaultValue.canConvertToInt(), "Cannot parse default as a %s value: %s", type, defaultValue); - return defaultValue.intValue(); - case LONG: + yield defaultValue.intValue(); + } + case LONG -> { Preconditions.checkArgument( defaultValue.isIntegralNumber() && defaultValue.canConvertToLong(), "Cannot parse default as a %s value: %s", type, defaultValue); - return defaultValue.longValue(); - case FLOAT: + yield defaultValue.longValue(); + } + case FLOAT -> { Preconditions.checkArgument( defaultValue.isFloatingPointNumber(), "Cannot parse default as a %s value: %s", type, defaultValue); - return defaultValue.floatValue(); - case DOUBLE: + yield defaultValue.floatValue(); + } + case DOUBLE -> { Preconditions.checkArgument( defaultValue.isFloatingPointNumber(), "Cannot parse default as a %s value: %s", type, defaultValue); - return defaultValue.doubleValue(); - case DECIMAL: + yield defaultValue.doubleValue(); + } + case DECIMAL -> { Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); BigDecimal retDecimal; @@ -99,12 +105,14 @@ public static Object fromJson(Type type, JsonNode defaultValue) { "Cannot parse default as a %s value: %s, the scale doesn't match", type, defaultValue); - return retDecimal; - case STRING: + yield retDecimal; + } + case STRING -> { Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); - return defaultValue.textValue(); - case UUID: + yield defaultValue.textValue(); + } + case UUID -> { Preconditions.checkArgument( defaultValue.isTextual() && defaultValue.textValue().length() == 36, "Cannot parse default as a %s value: %s", @@ -117,16 +125,19 @@ public static Object fromJson(Type type, JsonNode defaultValue) { throw new IllegalArgumentException( String.format("Cannot parse default as a %s value: %s", type, defaultValue), e); } - return uuid; - case DATE: + yield uuid; + } + case DATE -> { Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); - return DateTimeUtil.isoDateToDays(defaultValue.textValue()); - case TIME: + yield DateTimeUtil.isoDateToDays(defaultValue.textValue()); + } + case TIME -> { Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); - return DateTimeUtil.isoTimeToMicros(defaultValue.textValue()); - case TIMESTAMP: + yield DateTimeUtil.isoTimeToMicros(defaultValue.textValue()); + } + case TIMESTAMP -> { Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); if (((Types.TimestampType) type).shouldAdjustToUTC()) { @@ -136,11 +147,12 @@ public static Object fromJson(Type type, JsonNode defaultValue) { "Cannot parse default as a %s value: %s, offset must be +00:00", type, defaultValue); - return DateTimeUtil.isoTimestamptzToMicros(timestampTz); + yield DateTimeUtil.isoTimestamptzToMicros(timestampTz); } else { - return DateTimeUtil.isoTimestampToMicros(defaultValue.textValue()); + yield DateTimeUtil.isoTimestampToMicros(defaultValue.textValue()); } - case TIMESTAMP_NANO: + } + case TIMESTAMP_NANO -> { Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); if (((Types.TimestampNanoType) type).shouldAdjustToUTC()) { @@ -150,11 +162,12 @@ public static Object fromJson(Type type, JsonNode defaultValue) { "Cannot parse default as a %s value: %s, offset must be +00:00", type, defaultValue); - return DateTimeUtil.isoTimestamptzToNanos(timestampTzNano); + yield DateTimeUtil.isoTimestamptzToNanos(timestampTzNano); } else { - return DateTimeUtil.isoTimestampToNanos(defaultValue.textValue()); + yield DateTimeUtil.isoTimestampToNanos(defaultValue.textValue()); } - case FIXED: + } + case FIXED -> { Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); int defaultLength = defaultValue.textValue().length(); @@ -167,22 +180,21 @@ public static Object fromJson(Type type, JsonNode defaultValue) { defaultLength); byte[] fixedBytes = BaseEncoding.base16().decode(defaultValue.textValue().toUpperCase(Locale.ROOT)); - return ByteBuffer.wrap(fixedBytes); - case BINARY: + yield ByteBuffer.wrap(fixedBytes); + } + case BINARY -> { Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); byte[] binaryBytes = BaseEncoding.base16().decode(defaultValue.textValue().toUpperCase(Locale.ROOT)); - return ByteBuffer.wrap(binaryBytes); - case LIST: - return listFromJson(type, defaultValue); - case MAP: - return mapFromJson(type, defaultValue); - case STRUCT: - return structFromJson(type, defaultValue); - default: - throw new UnsupportedOperationException(String.format("Type: %s is not supported", type)); - } + yield ByteBuffer.wrap(binaryBytes); + } + case LIST -> listFromJson(type, defaultValue); + case MAP -> mapFromJson(type, defaultValue); + case STRUCT -> structFromJson(type, defaultValue); + default -> + throw new UnsupportedOperationException(String.format("Type: %s is not supported", type)); + }; } private static StructLike structFromJson(Type type, JsonNode defaultValue) { @@ -259,42 +271,42 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } switch (type.typeId()) { - case BOOLEAN: + case BOOLEAN -> { Preconditions.checkArgument( defaultValue instanceof Boolean, "Invalid default %s value: %s", type, defaultValue); generator.writeBoolean((Boolean) defaultValue); - break; - case INTEGER: + } + case INTEGER -> { Preconditions.checkArgument( defaultValue instanceof Integer, "Invalid default %s value: %s", type, defaultValue); generator.writeNumber((Integer) defaultValue); - break; - case LONG: + } + case LONG -> { Preconditions.checkArgument( defaultValue instanceof Long, "Invalid default %s value: %s", type, defaultValue); generator.writeNumber((Long) defaultValue); - break; - case FLOAT: + } + case FLOAT -> { Preconditions.checkArgument( defaultValue instanceof Float, "Invalid default %s value: %s", type, defaultValue); generator.writeNumber((Float) defaultValue); - break; - case DOUBLE: + } + case DOUBLE -> { Preconditions.checkArgument( defaultValue instanceof Double, "Invalid default %s value: %s", type, defaultValue); generator.writeNumber((Double) defaultValue); - break; - case DATE: + } + case DATE -> { Preconditions.checkArgument( defaultValue instanceof Integer, "Invalid default %s value: %s", type, defaultValue); generator.writeString(DateTimeUtil.daysToIsoDate((Integer) defaultValue)); - break; - case TIME: + } + case TIME -> { Preconditions.checkArgument( defaultValue instanceof Long, "Invalid default %s value: %s", type, defaultValue); generator.writeString(DateTimeUtil.microsToIsoTime((Long) defaultValue)); - break; - case TIMESTAMP: + } + case TIMESTAMP -> { Preconditions.checkArgument( defaultValue instanceof Long, "Invalid default %s value: %s", type, defaultValue); if (((Types.TimestampType) type).shouldAdjustToUTC()) { @@ -302,8 +314,8 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } else { generator.writeString(DateTimeUtil.microsToIsoTimestamp((Long) defaultValue)); } - break; - case TIMESTAMP_NANO: + } + case TIMESTAMP_NANO -> { Preconditions.checkArgument( defaultValue instanceof Long, "Invalid default %s value: %s", type, defaultValue); if (((Types.TimestampNanoType) type).shouldAdjustToUTC()) { @@ -311,21 +323,21 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } else { generator.writeString(DateTimeUtil.nanosToIsoTimestamp((Long) defaultValue)); } - break; - case STRING: + } + case STRING -> { Preconditions.checkArgument( defaultValue instanceof CharSequence, "Invalid default %s value: %s", type, defaultValue); generator.writeString(((CharSequence) defaultValue).toString()); - break; - case UUID: + } + case UUID -> { Preconditions.checkArgument( defaultValue instanceof UUID, "Invalid default %s value: %s", type, defaultValue); generator.writeString(defaultValue.toString()); - break; - case FIXED: + } + case FIXED -> { Preconditions.checkArgument( defaultValue instanceof ByteBuffer, "Invalid default %s value: %s", type, defaultValue); ByteBuffer byteBufferValue = (ByteBuffer) defaultValue; @@ -337,14 +349,14 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato byteBufferValue.remaining()); generator.writeString( BaseEncoding.base16().encode(ByteBuffers.toByteArray(byteBufferValue))); - break; - case BINARY: + } + case BINARY -> { Preconditions.checkArgument( defaultValue instanceof ByteBuffer, "Invalid default %s value: %s", type, defaultValue); generator.writeString( BaseEncoding.base16().encode(ByteBuffers.toByteArray((ByteBuffer) defaultValue))); - break; - case DECIMAL: + } + case DECIMAL -> { Preconditions.checkArgument( defaultValue instanceof BigDecimal && ((BigDecimal) defaultValue).scale() == ((Types.DecimalType) type).scale(), @@ -357,8 +369,8 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } else { generator.writeString(decimalValue.toString()); } - break; - case LIST: + } + case LIST -> { Preconditions.checkArgument( defaultValue instanceof List, "Invalid default %s value: %s", type, defaultValue); List defaultList = (List) defaultValue; @@ -368,8 +380,8 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato toJson(elementType, element, generator); } generator.writeEndArray(); - break; - case MAP: + } + case MAP -> { Preconditions.checkArgument( defaultValue instanceof Map, "Invalid default %s value: %s", type, defaultValue); Map defaultMap = (Map) defaultValue; @@ -390,8 +402,8 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } generator.writeEndArray(); generator.writeEndObject(); - break; - case STRUCT: + } + case STRUCT -> { Preconditions.checkArgument( defaultValue instanceof StructLike, "Invalid default %s value: %s", type, defaultValue); Types.StructType structType = type.asStructType(); @@ -409,9 +421,9 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } } generator.writeEndObject(); - break; - default: - throw new UnsupportedOperationException(String.format("Type: %s is not supported", type)); + } + default -> + throw new UnsupportedOperationException(String.format("Type: %s is not supported", type)); } } } diff --git a/core/src/main/java/org/apache/iceberg/SnapshotChanges.java b/core/src/main/java/org/apache/iceberg/SnapshotChanges.java index 38a81bb966c8..857fee5b512a 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotChanges.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotChanges.java @@ -139,12 +139,8 @@ private void cacheDataFileChanges() { iterate(manifestReadTasks)) { for (Pair pair : changedDataFiles) { switch (pair.first()) { - case ADDED: - adds.add(pair.second()); - break; - case DELETED: - deletes.add(pair.second()); - break; + case ADDED -> adds.add(pair.second()); + case DELETED -> deletes.add(pair.second()); } } } catch (IOException e) { @@ -190,12 +186,8 @@ private void cacheDeleteFileChanges() { iterate(manifestReadTasks)) { for (Pair pair : changedDeleteFiles) { switch (pair.first()) { - case ADDED: - adds.add(pair.second()); - break; - case DELETED: - deletes.add(pair.second()); - break; + case ADDED -> adds.add(pair.second()); + case DELETED -> deletes.add(pair.second()); } } } catch (IOException e) { diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index d97c63b61608..676024613568 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -833,24 +833,24 @@ private static ManifestFile addMetadata(TableOperations ops, ManifestFile manife } switch (entry.status()) { - case ADDED: + case ADDED -> { addedFiles += 1; addedRows += entry.file().recordCount(); if (snapshotId == null) { snapshotId = entry.snapshotId(); } - break; - case EXISTING: + } + case EXISTING -> { existingFiles += 1; existingRows += entry.file().recordCount(); - break; - case DELETED: + } + case DELETED -> { deletedFiles += 1; deletedRows += entry.file().recordCount(); if (snapshotId == null) { snapshotId = entry.snapshotId(); } - break; + } } stats.update(entry.file().partition()); diff --git a/core/src/main/java/org/apache/iceberg/SnapshotSummary.java b/core/src/main/java/org/apache/iceberg/SnapshotSummary.java index 392fad623f59..fdecb787e4f7 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotSummary.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotSummary.java @@ -291,11 +291,11 @@ void addTo(ImmutableMap.Builder builder) { void addedFile(ContentFile file) { this.addedSize += ScanTaskUtil.contentSizeInBytes(file); switch (file.content()) { - case DATA: + case DATA -> { this.addedFiles += 1; this.addedRecords += file.recordCount(); - break; - case POSITION_DELETES: + } + case POSITION_DELETES -> { DeleteFile deleteFile = (DeleteFile) file; if (ContentFileUtil.isDV(deleteFile)) { this.addedDVs += 1; @@ -304,26 +304,26 @@ void addedFile(ContentFile file) { } this.addedDeleteFiles += 1; this.addedPosDeletes += file.recordCount(); - break; - case EQUALITY_DELETES: + } + case EQUALITY_DELETES -> { this.addedDeleteFiles += 1; this.addedEqDeleteFiles += 1; this.addedEqDeletes += file.recordCount(); - break; - default: - throw new UnsupportedOperationException( - "Unsupported file content type: " + file.content()); + } + default -> + throw new UnsupportedOperationException( + "Unsupported file content type: " + file.content()); } } void removedFile(ContentFile file) { this.removedSize += ScanTaskUtil.contentSizeInBytes(file); switch (file.content()) { - case DATA: + case DATA -> { this.removedFiles += 1; this.deletedRecords += file.recordCount(); - break; - case POSITION_DELETES: + } + case POSITION_DELETES -> { DeleteFile deleteFile = (DeleteFile) file; if (ContentFileUtil.isDV(deleteFile)) { this.removedDVs += 1; @@ -332,31 +332,31 @@ void removedFile(ContentFile file) { } this.removedDeleteFiles += 1; this.removedPosDeletes += file.recordCount(); - break; - case EQUALITY_DELETES: + } + case EQUALITY_DELETES -> { this.removedDeleteFiles += 1; this.removedEqDeleteFiles += 1; this.removedEqDeletes += file.recordCount(); - break; - default: - throw new UnsupportedOperationException( - "Unsupported file content type: " + file.content()); + } + default -> + throw new UnsupportedOperationException( + "Unsupported file content type: " + file.content()); } } void addedManifest(ManifestFile manifest) { switch (manifest.content()) { - case DATA: + case DATA -> { this.addedFiles += manifest.addedFilesCount(); this.addedRecords += manifest.addedRowsCount(); this.removedFiles += manifest.deletedFilesCount(); this.deletedRecords += manifest.deletedRowsCount(); - break; - case DELETES: + } + case DELETES -> { this.addedDeleteFiles += manifest.addedFilesCount(); this.removedDeleteFiles += manifest.deletedFilesCount(); this.trustSizeAndDeleteCounts = false; - break; + } } } diff --git a/core/src/main/java/org/apache/iceberg/SortOrderParser.java b/core/src/main/java/org/apache/iceberg/SortOrderParser.java index 53d7e5090c76..695621a8e3c7 100644 --- a/core/src/main/java/org/apache/iceberg/SortOrderParser.java +++ b/core/src/main/java/org/apache/iceberg/SortOrderParser.java @@ -168,13 +168,11 @@ private static void buildFromJsonFields(UnboundSortOrder.Builder builder, JsonNo } private static NullOrder toNullOrder(String nullOrderingAsString) { - switch (nullOrderingAsString.toLowerCase(Locale.ROOT)) { - case "nulls-first": - return NULLS_FIRST; - case "nulls-last": - return NULLS_LAST; - default: - throw new IllegalArgumentException("Unexpected null order: " + nullOrderingAsString); - } + return switch (nullOrderingAsString.toLowerCase(Locale.ROOT)) { + case "nulls-first" -> NULLS_FIRST; + case "nulls-last" -> NULLS_LAST; + default -> + throw new IllegalArgumentException("Unexpected null order: " + nullOrderingAsString); + }; } } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 9a44e8045cbb..76b16ad7439a 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -274,98 +274,51 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - switch (pos) { - case 0: - return tracking; - case 1: - return contentType != null ? contentType.id() : null; - case 2: - return writerFormatVersion; - case 3: - return location; - case 4: - return fileFormat != null ? fileFormat.toString() : null; - case 5: - return recordCount; - case 6: - return fileSizeInBytes; - case 7: - return specId; - case 8: - return partitionData; - case 9: - return contentStats; - case 10: - return sortOrderId; - case 11: - return deletionVector; - case 12: - return manifestInfo; - case 13: - return keyMetadata(); - case 14: - return splitOffsets(); - case 15: - return equalityIds(); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + return switch (pos) { + case 0 -> tracking; + case 1 -> contentType != null ? contentType.id() : null; + case 2 -> writerFormatVersion; + case 3 -> location; + case 4 -> fileFormat != null ? fileFormat.toString() : null; + case 5 -> recordCount; + case 6 -> fileSizeInBytes; + case 7 -> specId; + case 8 -> partitionData; + case 9 -> contentStats; + case 10 -> sortOrderId; + case 11 -> deletionVector; + case 12 -> manifestInfo; + case 13 -> keyMetadata(); + case 14 -> splitOffsets(); + case 15 -> equalityIds(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0: - this.tracking = (Tracking) value; - break; - case 1: - this.contentType = FileContent.fromId((Integer) value); - break; - case 2: - this.writerFormatVersion = (int) value; - break; - case 3: - // always coerce to String for Serializable - this.location = value.toString(); - break; - case 4: - this.fileFormat = FileFormat.fromString(value.toString()); - break; - case 5: - this.recordCount = (long) value; - break; - case 6: - this.fileSizeInBytes = (long) value; - break; - case 7: - this.specId = (Integer) value; - break; - case 8: - this.partitionData = (PartitionData) value; - break; - case 9: - this.contentStats = (ContentStats) value; - break; - case 10: - this.sortOrderId = (Integer) value; - break; - case 11: - this.deletionVector = (DeletionVector) value; - break; - case 12: - this.manifestInfo = (ManifestInfo) value; - break; - case 13: - this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); - break; - case 14: - this.splitOffsets = ArrayUtil.toLongArray((List) value); - break; - case 15: - this.equalityIds = ArrayUtil.toIntArray((List) value); - break; - default: + case 0 -> this.tracking = (Tracking) value; + case 1 -> this.contentType = FileContent.fromId((Integer) value); + case 2 -> this.writerFormatVersion = (int) value; + case 3 -> + // always coerce to String for Serializable + this.location = value.toString(); + case 4 -> this.fileFormat = FileFormat.fromString(value.toString()); + case 5 -> this.recordCount = (long) value; + case 6 -> this.fileSizeInBytes = (long) value; + case 7 -> this.specId = (Integer) value; + case 8 -> this.partitionData = (PartitionData) value; + case 9 -> this.contentStats = (ContentStats) value; + case 10 -> this.sortOrderId = (Integer) value; + case 11 -> this.deletionVector = (DeletionVector) value; + case 12 -> this.manifestInfo = (ManifestInfo) value; + case 13 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + case 14 -> this.splitOffsets = ArrayUtil.toLongArray((List) value); + case 15 -> this.equalityIds = ArrayUtil.toIntArray((List) value); + default -> { // ignore the object, it must be from a newer version of the format + } } } diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java index 8ae4b7e4ce88..b4fae388c96b 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java @@ -195,62 +195,35 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - switch (pos) { - case 0: - return status != null ? status.id() : null; - case 1: - return snapshotId(); - case 2: - return dataSequenceNumber(); - case 3: - return fileSequenceNumber(); - case 4: - return dvSnapshotId; - case 5: - return firstRowId; - case 6: - return deletedPositions(); - case 7: - return replacedPositions(); - case 8: - return manifestPos; - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + return switch (pos) { + case 0 -> status != null ? status.id() : null; + case 1 -> snapshotId(); + case 2 -> dataSequenceNumber(); + case 3 -> fileSequenceNumber(); + case 4 -> dvSnapshotId; + case 5 -> firstRowId; + case 6 -> deletedPositions(); + case 7 -> replacedPositions(); + case 8 -> manifestPos; + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0: - this.status = EntryStatus.fromId((Integer) value); - break; - case 1: - this.snapshotId = (Long) value; - break; - case 2: - this.dataSequenceNumber = (Long) value; - break; - case 3: - this.fileSequenceNumber = (Long) value; - break; - case 4: - this.dvSnapshotId = (Long) value; - break; - case 5: - this.firstRowId = (Long) value; - break; - case 6: - this.deletedPositions = ByteBuffers.toByteArray((ByteBuffer) value); - break; - case 7: - this.replacedPositions = ByteBuffers.toByteArray((ByteBuffer) value); - break; - case 8: - this.manifestPos = (long) value; - break; - default: + case 0 -> this.status = EntryStatus.fromId((Integer) value); + case 1 -> this.snapshotId = (Long) value; + case 2 -> this.dataSequenceNumber = (Long) value; + case 3 -> this.fileSequenceNumber = (Long) value; + case 4 -> this.dvSnapshotId = (Long) value; + case 5 -> this.firstRowId = (Long) value; + case 6 -> this.deletedPositions = ByteBuffers.toByteArray((ByteBuffer) value); + case 7 -> this.replacedPositions = ByteBuffers.toByteArray((ByteBuffer) value); + case 8 -> this.manifestPos = (long) value; + default -> { // ignore the object, it must be from a newer version of the format + } } } diff --git a/core/src/main/java/org/apache/iceberg/UpdateRequirementParser.java b/core/src/main/java/org/apache/iceberg/UpdateRequirementParser.java index 5c4dc2221290..d4a6e0d49039 100644 --- a/core/src/main/java/org/apache/iceberg/UpdateRequirementParser.java +++ b/core/src/main/java/org/apache/iceberg/UpdateRequirementParser.java @@ -97,44 +97,36 @@ public static void toJson(UpdateRequirement updateRequirement, JsonGenerator gen generator.writeStringField(TYPE, requirementType); switch (requirementType) { - case ASSERT_TABLE_DOES_NOT_EXIST: + case ASSERT_TABLE_DOES_NOT_EXIST -> { // No fields beyond the requirement itself - break; - case ASSERT_TABLE_UUID: - writeAssertTableUUID((UpdateRequirement.AssertTableUUID) updateRequirement, generator); - break; - case ASSERT_VIEW_UUID: - writeAssertViewUUID((UpdateRequirement.AssertViewUUID) updateRequirement, generator); - break; - case ASSERT_REF_SNAPSHOT_ID: - writeAssertRefSnapshotId( - (UpdateRequirement.AssertRefSnapshotID) updateRequirement, generator); - break; - case ASSERT_LAST_ASSIGNED_FIELD_ID: - writeAssertLastAssignedFieldId( - (UpdateRequirement.AssertLastAssignedFieldId) updateRequirement, generator); - break; - case ASSERT_LAST_ASSIGNED_PARTITION_ID: - writeAssertLastAssignedPartitionId( - (UpdateRequirement.AssertLastAssignedPartitionId) updateRequirement, generator); - break; - case ASSERT_CURRENT_SCHEMA_ID: - writeAssertCurrentSchemaId( - (UpdateRequirement.AssertCurrentSchemaID) updateRequirement, generator); - break; - case ASSERT_DEFAULT_SPEC_ID: - writeAssertDefaultSpecId( - (UpdateRequirement.AssertDefaultSpecID) updateRequirement, generator); - break; - case ASSERT_DEFAULT_SORT_ORDER_ID: - writeAssertDefaultSortOrderId( - (UpdateRequirement.AssertDefaultSortOrderID) updateRequirement, generator); - break; - default: - throw new IllegalArgumentException( - String.format( - "Cannot convert update requirement to json. Unrecognized type: %s", - requirementType)); + } + case ASSERT_TABLE_UUID -> + writeAssertTableUUID((UpdateRequirement.AssertTableUUID) updateRequirement, generator); + case ASSERT_VIEW_UUID -> + writeAssertViewUUID((UpdateRequirement.AssertViewUUID) updateRequirement, generator); + case ASSERT_REF_SNAPSHOT_ID -> + writeAssertRefSnapshotId( + (UpdateRequirement.AssertRefSnapshotID) updateRequirement, generator); + case ASSERT_LAST_ASSIGNED_FIELD_ID -> + writeAssertLastAssignedFieldId( + (UpdateRequirement.AssertLastAssignedFieldId) updateRequirement, generator); + case ASSERT_LAST_ASSIGNED_PARTITION_ID -> + writeAssertLastAssignedPartitionId( + (UpdateRequirement.AssertLastAssignedPartitionId) updateRequirement, generator); + case ASSERT_CURRENT_SCHEMA_ID -> + writeAssertCurrentSchemaId( + (UpdateRequirement.AssertCurrentSchemaID) updateRequirement, generator); + case ASSERT_DEFAULT_SPEC_ID -> + writeAssertDefaultSpecId( + (UpdateRequirement.AssertDefaultSpecID) updateRequirement, generator); + case ASSERT_DEFAULT_SORT_ORDER_ID -> + writeAssertDefaultSortOrderId( + (UpdateRequirement.AssertDefaultSortOrderID) updateRequirement, generator); + default -> + throw new IllegalArgumentException( + String.format( + "Cannot convert update requirement to json. Unrecognized type: %s", + requirementType)); } generator.writeEndObject(); @@ -159,29 +151,20 @@ public static UpdateRequirement fromJson(JsonNode jsonNode) { jsonNode.hasNonNull(TYPE), "Cannot parse update requirement. Missing field: type"); String type = JsonUtil.getString(TYPE, jsonNode).toLowerCase(Locale.ROOT); - switch (type) { - case ASSERT_TABLE_DOES_NOT_EXIST: - return readAssertTableDoesNotExist(jsonNode); - case ASSERT_TABLE_UUID: - return readAssertTableUUID(jsonNode); - case ASSERT_VIEW_UUID: - return readAssertViewUUID(jsonNode); - case ASSERT_REF_SNAPSHOT_ID: - return readAssertRefSnapshotId(jsonNode); - case ASSERT_LAST_ASSIGNED_FIELD_ID: - return readAssertLastAssignedFieldId(jsonNode); - case ASSERT_LAST_ASSIGNED_PARTITION_ID: - return readAssertLastAssignedPartitionId(jsonNode); - case ASSERT_CURRENT_SCHEMA_ID: - return readAssertCurrentSchemaId(jsonNode); - case ASSERT_DEFAULT_SPEC_ID: - return readAssertDefaultSpecId(jsonNode); - case ASSERT_DEFAULT_SORT_ORDER_ID: - return readAssertDefaultSortOrderId(jsonNode); - default: - throw new UnsupportedOperationException( - String.format("Unrecognized update requirement. Cannot convert to json: %s", type)); - } + return switch (type) { + case ASSERT_TABLE_DOES_NOT_EXIST -> readAssertTableDoesNotExist(jsonNode); + case ASSERT_TABLE_UUID -> readAssertTableUUID(jsonNode); + case ASSERT_VIEW_UUID -> readAssertViewUUID(jsonNode); + case ASSERT_REF_SNAPSHOT_ID -> readAssertRefSnapshotId(jsonNode); + case ASSERT_LAST_ASSIGNED_FIELD_ID -> readAssertLastAssignedFieldId(jsonNode); + case ASSERT_LAST_ASSIGNED_PARTITION_ID -> readAssertLastAssignedPartitionId(jsonNode); + case ASSERT_CURRENT_SCHEMA_ID -> readAssertCurrentSchemaId(jsonNode); + case ASSERT_DEFAULT_SPEC_ID -> readAssertDefaultSpecId(jsonNode); + case ASSERT_DEFAULT_SORT_ORDER_ID -> readAssertDefaultSortOrderId(jsonNode); + default -> + throw new UnsupportedOperationException( + String.format("Unrecognized update requirement. Cannot convert to json: %s", type)); + }; } private static void writeAssertTableUUID( diff --git a/core/src/main/java/org/apache/iceberg/V1Metadata.java b/core/src/main/java/org/apache/iceberg/V1Metadata.java index 34bcd691a9cf..de7bcf56d668 100644 --- a/core/src/main/java/org/apache/iceberg/V1Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V1Metadata.java @@ -74,34 +74,21 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return path(); - case 1: - return length(); - case 2: - return partitionSpecId(); - case 3: - return snapshotId(); - case 4: - return addedFilesCount(); - case 5: - return existingFilesCount(); - case 6: - return deletedFilesCount(); - case 7: - return partitions(); - case 8: - return addedRowsCount(); - case 9: - return existingRowsCount(); - case 10: - return deletedRowsCount(); - case 11: - return keyMetadata(); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + return switch (pos) { + case 0 -> path(); + case 1 -> length(); + case 2 -> partitionSpecId(); + case 3 -> snapshotId(); + case 4 -> addedFilesCount(); + case 5 -> existingFilesCount(); + case 6 -> deletedFilesCount(); + case 7 -> partitions(); + case 8 -> addedRowsCount(); + case 9 -> existingRowsCount(); + case 10 -> deletedRowsCount(); + case 11 -> keyMetadata(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -266,20 +253,18 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return wrapped.status().id(); - case 1: - return wrapped.snapshotId(); - case 2: + return switch (pos) { + case 0 -> wrapped.status().id(); + case 1 -> wrapped.snapshotId(); + case 2 -> { DataFile file = wrapped.file(); if (file != null) { - return fileWrapper.wrap(file); + yield fileWrapper.wrap(file); } - return null; - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + yield null; + } + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -364,39 +349,24 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return wrapped.location(); - case 1: - return wrapped.format() != null ? wrapped.format().toString() : null; - case 2: - return wrapped.partition(); - case 3: - return wrapped.recordCount(); - case 4: - return wrapped.fileSizeInBytes(); - case 5: - return DEFAULT_BLOCK_SIZE; - case 6: - return wrapped.columnSizes(); - case 7: - return wrapped.valueCounts(); - case 8: - return wrapped.nullValueCounts(); - case 9: - return wrapped.nanValueCounts(); - case 10: - return wrapped.lowerBounds(); - case 11: - return wrapped.upperBounds(); - case 12: - return wrapped.keyMetadata(); - case 13: - return wrapped.splitOffsets(); - case 14: - return wrapped.sortOrderId(); - } - throw new IllegalArgumentException("Unknown field ordinal: " + pos); + return switch (pos) { + case 0 -> wrapped.location(); + case 1 -> wrapped.format() != null ? wrapped.format().toString() : null; + case 2 -> wrapped.partition(); + case 3 -> wrapped.recordCount(); + case 4 -> wrapped.fileSizeInBytes(); + case 5 -> DEFAULT_BLOCK_SIZE; + case 6 -> wrapped.columnSizes(); + case 7 -> wrapped.valueCounts(); + case 8 -> wrapped.nullValueCounts(); + case 9 -> wrapped.nanValueCounts(); + case 10 -> wrapped.lowerBounds(); + case 11 -> wrapped.upperBounds(); + case 12 -> wrapped.keyMetadata(); + case 13 -> wrapped.splitOffsets(); + case 14 -> wrapped.sortOrderId(); + default -> throw new IllegalArgumentException("Unknown field ordinal: " + pos); + }; } @Override diff --git a/core/src/main/java/org/apache/iceberg/V2Metadata.java b/core/src/main/java/org/apache/iceberg/V2Metadata.java index 803905f6b42e..b22297c64f1c 100644 --- a/core/src/main/java/org/apache/iceberg/V2Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V2Metadata.java @@ -85,17 +85,15 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return wrapped.path(); - case 1: - return wrapped.length(); - case 2: - return wrapped.partitionSpecId(); - case 3: + return switch (pos) { + case 0 -> wrapped.path(); + case 1 -> wrapped.length(); + case 2 -> wrapped.partitionSpecId(); + case 3 -> { checkContentType(wrapped.content()); - return wrapped.content().id(); - case 4: + yield wrapped.content().id(); + } + case 4 -> { if (wrapped.sequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // if the sequence number is being assigned here, then the manifest must be created by // the current @@ -104,11 +102,12 @@ private Object get(int pos) { commitSnapshotId == wrapped.snapshotId(), "Found unassigned sequence number for a manifest from snapshot: %s", wrapped.snapshotId()); - return sequenceNumber; + yield sequenceNumber; } else { - return wrapped.sequenceNumber(); + yield wrapped.sequenceNumber(); } - case 5: + } + case 5 -> { if (wrapped.minSequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // same sanity check as above Preconditions.checkState( @@ -119,31 +118,22 @@ private Object get(int pos) { // number for any file // written to the wrapped manifest. replace the unassigned sequence number with the one // for this commit - return sequenceNumber; + yield sequenceNumber; } else { - return wrapped.minSequenceNumber(); + yield wrapped.minSequenceNumber(); } - case 6: - return wrapped.snapshotId(); - case 7: - return wrapped.addedFilesCount(); - case 8: - return wrapped.existingFilesCount(); - case 9: - return wrapped.deletedFilesCount(); - case 10: - return wrapped.addedRowsCount(); - case 11: - return wrapped.existingRowsCount(); - case 12: - return wrapped.deletedRowsCount(); - case 13: - return wrapped.partitions(); - case 14: - return wrapped.keyMetadata(); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + } + case 6 -> wrapped.snapshotId(); + case 7 -> wrapped.addedFilesCount(); + case 8 -> wrapped.existingFilesCount(); + case 9 -> wrapped.deletedFilesCount(); + case 10 -> wrapped.addedRowsCount(); + case 11 -> wrapped.existingRowsCount(); + case 12 -> wrapped.deletedRowsCount(); + case 13 -> wrapped.partitions(); + case 14 -> wrapped.keyMetadata(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -312,12 +302,10 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return wrapped.status().id(); - case 1: - return wrapped.snapshotId(); - case 2: + return switch (pos) { + case 0 -> wrapped.status().id(); + case 1 -> wrapped.snapshotId(); + case 2 -> { if (wrapped.dataSequenceNumber() == null) { // if the entry's data sequence number is null, // then it will inherit the sequence number of the current commit. @@ -333,16 +321,14 @@ private Object get(int pos) { wrapped.status() == Status.ADDED, "Only entries with status ADDED can have null sequence number"); - return null; + yield null; } - return wrapped.dataSequenceNumber(); - case 3: - return wrapped.fileSequenceNumber(); - case 4: - return fileWrapper.wrap(wrapped.file()); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + yield wrapped.dataSequenceNumber(); + } + case 3 -> wrapped.fileSequenceNumber(); + case 4 -> fileWrapper.wrap(wrapped.file()); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -427,48 +413,35 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: + return switch (pos) { + case 0 -> { checkContentType(wrapped.content()); - return wrapped.content().id(); - case 1: - return wrapped.location(); - case 2: - return wrapped.format() != null ? wrapped.format().toString() : null; - case 3: - return wrapped.partition(); - case 4: - return wrapped.recordCount(); - case 5: - return wrapped.fileSizeInBytes(); - case 6: - return wrapped.columnSizes(); - case 7: - return wrapped.valueCounts(); - case 8: - return wrapped.nullValueCounts(); - case 9: - return wrapped.nanValueCounts(); - case 10: - return wrapped.lowerBounds(); - case 11: - return wrapped.upperBounds(); - case 12: - return wrapped.keyMetadata(); - case 13: - return wrapped.splitOffsets(); - case 14: - return wrapped.equalityFieldIds(); - case 15: - return wrapped.sortOrderId(); - case 16: + yield wrapped.content().id(); + } + case 1 -> wrapped.location(); + case 2 -> wrapped.format() != null ? wrapped.format().toString() : null; + case 3 -> wrapped.partition(); + case 4 -> wrapped.recordCount(); + case 5 -> wrapped.fileSizeInBytes(); + case 6 -> wrapped.columnSizes(); + case 7 -> wrapped.valueCounts(); + case 8 -> wrapped.nullValueCounts(); + case 9 -> wrapped.nanValueCounts(); + case 10 -> wrapped.lowerBounds(); + case 11 -> wrapped.upperBounds(); + case 12 -> wrapped.keyMetadata(); + case 13 -> wrapped.splitOffsets(); + case 14 -> wrapped.equalityFieldIds(); + case 15 -> wrapped.sortOrderId(); + case 16 -> { if (wrapped instanceof DeleteFile) { - return ((DeleteFile) wrapped).referencedDataFile(); + yield ((DeleteFile) wrapped).referencedDataFile(); } else { - return null; + yield null; } - } - throw new IllegalArgumentException("Unknown field ordinal: " + pos); + } + default -> throw new IllegalArgumentException("Unknown field ordinal: " + pos); + }; } @Override diff --git a/core/src/main/java/org/apache/iceberg/V3Metadata.java b/core/src/main/java/org/apache/iceberg/V3Metadata.java index 4e67d9977e64..ea75b2952b02 100644 --- a/core/src/main/java/org/apache/iceberg/V3Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V3Metadata.java @@ -86,17 +86,15 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return wrapped.path(); - case 1: - return wrapped.length(); - case 2: - return wrapped.partitionSpecId(); - case 3: + return switch (pos) { + case 0 -> wrapped.path(); + case 1 -> wrapped.length(); + case 2 -> wrapped.partitionSpecId(); + case 3 -> { checkContentType(wrapped.content()); - return wrapped.content().id(); - case 4: + yield wrapped.content().id(); + } + case 4 -> { if (wrapped.sequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // if the sequence number is being assigned here, then the manifest must be created by // the current @@ -105,11 +103,12 @@ private Object get(int pos) { commitSnapshotId == wrapped.snapshotId(), "Found unassigned sequence number for a manifest from snapshot: %s", wrapped.snapshotId()); - return sequenceNumber; + yield sequenceNumber; } else { - return wrapped.sequenceNumber(); + yield wrapped.sequenceNumber(); } - case 5: + } + case 5 -> { if (wrapped.minSequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // same sanity check as above Preconditions.checkState( @@ -120,47 +119,39 @@ private Object get(int pos) { // number for any file // written to the wrapped manifest. replace the unassigned sequence number with the one // for this commit - return sequenceNumber; + yield sequenceNumber; } else { - return wrapped.minSequenceNumber(); + yield wrapped.minSequenceNumber(); } - case 6: - return wrapped.snapshotId(); - case 7: - return wrapped.addedFilesCount(); - case 8: - return wrapped.existingFilesCount(); - case 9: - return wrapped.deletedFilesCount(); - case 10: - return wrapped.addedRowsCount(); - case 11: - return wrapped.existingRowsCount(); - case 12: - return wrapped.deletedRowsCount(); - case 13: - return wrapped.partitions(); - case 14: - return wrapped.keyMetadata(); - case 15: + } + case 6 -> wrapped.snapshotId(); + case 7 -> wrapped.addedFilesCount(); + case 8 -> wrapped.existingFilesCount(); + case 9 -> wrapped.deletedFilesCount(); + case 10 -> wrapped.addedRowsCount(); + case 11 -> wrapped.existingRowsCount(); + case 12 -> wrapped.deletedRowsCount(); + case 13 -> wrapped.partitions(); + case 14 -> wrapped.keyMetadata(); + case 15 -> { if (wrappedFirstRowId != null) { // if first-row-id is assigned, ensure that it is valid Preconditions.checkState( wrapped.content() == ManifestContent.DATA && wrapped.firstRowId() == null, "Found invalid first-row-id assignment: %s", wrapped); - return wrappedFirstRowId; + yield wrappedFirstRowId; } else if (wrapped.content() != ManifestContent.DATA) { - return null; + yield null; } else { Preconditions.checkState( wrapped.firstRowId() != null, "Found unassigned first-row-id for file: " + wrapped.path()); - return wrapped.firstRowId(); + yield wrapped.firstRowId(); } - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + } + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -337,12 +328,10 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return wrapped.status().id(); - case 1: - return wrapped.snapshotId(); - case 2: + return switch (pos) { + case 0 -> wrapped.status().id(); + case 1 -> wrapped.snapshotId(); + case 2 -> { if (wrapped.dataSequenceNumber() == null) { // if the entry's data sequence number is null, // then it will inherit the sequence number of the current commit. @@ -358,16 +347,14 @@ private Object get(int pos) { wrapped.status() == Status.ADDED, "Only entries with status ADDED can have null sequence number"); - return null; + yield null; } - return wrapped.dataSequenceNumber(); - case 3: - return wrapped.fileSequenceNumber(); - case 4: - return fileWrapper.wrap(wrapped.file()); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + yield wrapped.dataSequenceNumber(); + } + case 3 -> wrapped.fileSequenceNumber(); + case 4 -> fileWrapper.wrap(wrapped.file()); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -453,66 +440,56 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: + return switch (pos) { + case 0 -> { checkContentType(wrapped.content()); - return wrapped.content().id(); - case 1: - return wrapped.location(); - case 2: - return wrapped.format() != null ? wrapped.format().toString() : null; - case 3: - return wrapped.partition(); - case 4: - return wrapped.recordCount(); - case 5: - return wrapped.fileSizeInBytes(); - case 6: - return wrapped.columnSizes(); - case 7: - return wrapped.valueCounts(); - case 8: - return wrapped.nullValueCounts(); - case 9: - return wrapped.nanValueCounts(); - case 10: - return wrapped.lowerBounds(); - case 11: - return wrapped.upperBounds(); - case 12: - return wrapped.keyMetadata(); - case 13: - return wrapped.splitOffsets(); - case 14: - return wrapped.equalityFieldIds(); - case 15: - return wrapped.sortOrderId(); - case 16: + yield wrapped.content().id(); + } + case 1 -> wrapped.location(); + case 2 -> wrapped.format() != null ? wrapped.format().toString() : null; + case 3 -> wrapped.partition(); + case 4 -> wrapped.recordCount(); + case 5 -> wrapped.fileSizeInBytes(); + case 6 -> wrapped.columnSizes(); + case 7 -> wrapped.valueCounts(); + case 8 -> wrapped.nullValueCounts(); + case 9 -> wrapped.nanValueCounts(); + case 10 -> wrapped.lowerBounds(); + case 11 -> wrapped.upperBounds(); + case 12 -> wrapped.keyMetadata(); + case 13 -> wrapped.splitOffsets(); + case 14 -> wrapped.equalityFieldIds(); + case 15 -> wrapped.sortOrderId(); + case 16 -> { if (wrapped.content() == FileContent.DATA) { - return wrapped.firstRowId(); + yield wrapped.firstRowId(); } else { - return null; + yield null; } - case 17: + } + case 17 -> { if (wrapped.content() == FileContent.POSITION_DELETES) { - return ((DeleteFile) wrapped).referencedDataFile(); + yield ((DeleteFile) wrapped).referencedDataFile(); } else { - return null; + yield null; } - case 18: + } + case 18 -> { if (wrapped.content() == FileContent.POSITION_DELETES) { - return ((DeleteFile) wrapped).contentOffset(); + yield ((DeleteFile) wrapped).contentOffset(); } else { - return null; + yield null; } - case 19: + } + case 19 -> { if (wrapped.content() == FileContent.POSITION_DELETES) { - return ((DeleteFile) wrapped).contentSizeInBytes(); + yield ((DeleteFile) wrapped).contentSizeInBytes(); } else { - return null; + yield null; } - } - throw new IllegalArgumentException("Unknown field ordinal: " + pos); + } + default -> throw new IllegalArgumentException("Unknown field ordinal: " + pos); + }; } @Override diff --git a/core/src/main/java/org/apache/iceberg/V4Metadata.java b/core/src/main/java/org/apache/iceberg/V4Metadata.java index 06fc75213df0..c7a121ef735d 100644 --- a/core/src/main/java/org/apache/iceberg/V4Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V4Metadata.java @@ -87,16 +87,12 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return wrapped.path(); - case 1: - return wrapped.length(); - case 2: - return wrapped.partitionSpecId(); - case 3: - return wrapped.content().id(); - case 4: + return switch (pos) { + case 0 -> wrapped.path(); + case 1 -> wrapped.length(); + case 2 -> wrapped.partitionSpecId(); + case 3 -> wrapped.content().id(); + case 4 -> { if (wrapped.sequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // if the sequence number is being assigned here, then the manifest must be created by // the current @@ -105,11 +101,12 @@ private Object get(int pos) { commitSnapshotId == wrapped.snapshotId(), "Found unassigned sequence number for a manifest from snapshot: %s", wrapped.snapshotId()); - return sequenceNumber; + yield sequenceNumber; } else { - return wrapped.sequenceNumber(); + yield wrapped.sequenceNumber(); } - case 5: + } + case 5 -> { if (wrapped.minSequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // same sanity check as above Preconditions.checkState( @@ -120,47 +117,39 @@ private Object get(int pos) { // number for any file // written to the wrapped manifest. replace the unassigned sequence number with the one // for this commit - return sequenceNumber; + yield sequenceNumber; } else { - return wrapped.minSequenceNumber(); + yield wrapped.minSequenceNumber(); } - case 6: - return wrapped.snapshotId(); - case 7: - return wrapped.addedFilesCount(); - case 8: - return wrapped.existingFilesCount(); - case 9: - return wrapped.deletedFilesCount(); - case 10: - return wrapped.addedRowsCount(); - case 11: - return wrapped.existingRowsCount(); - case 12: - return wrapped.deletedRowsCount(); - case 13: - return wrapped.partitions(); - case 14: - return wrapped.keyMetadata(); - case 15: + } + case 6 -> wrapped.snapshotId(); + case 7 -> wrapped.addedFilesCount(); + case 8 -> wrapped.existingFilesCount(); + case 9 -> wrapped.deletedFilesCount(); + case 10 -> wrapped.addedRowsCount(); + case 11 -> wrapped.existingRowsCount(); + case 12 -> wrapped.deletedRowsCount(); + case 13 -> wrapped.partitions(); + case 14 -> wrapped.keyMetadata(); + case 15 -> { if (wrappedFirstRowId != null) { // if first-row-id is assigned, ensure that it is valid Preconditions.checkState( wrapped.content() == ManifestContent.DATA && wrapped.firstRowId() == null, "Found invalid first-row-id assignment: %s", wrapped); - return wrappedFirstRowId; + yield wrappedFirstRowId; } else if (wrapped.content() != ManifestContent.DATA) { - return null; + yield null; } else { Preconditions.checkState( wrapped.firstRowId() != null, "Found unassigned first-row-id for file: " + wrapped.path()); - return wrapped.firstRowId(); + yield wrapped.firstRowId(); } - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + } + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -347,12 +336,10 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - switch (pos) { - case 0: - return wrapped.status().id(); - case 1: - return wrapped.snapshotId(); - case 2: + return switch (pos) { + case 0 -> wrapped.status().id(); + case 1 -> wrapped.snapshotId(); + case 2 -> { if (wrapped.dataSequenceNumber() == null) { // if the entry's data sequence number is null, // then it will inherit the sequence number of the current commit. @@ -368,16 +355,14 @@ private Object get(int pos) { wrapped.status() == Status.ADDED, "Only entries with status ADDED can have null sequence number"); - return null; + yield null; } - return wrapped.dataSequenceNumber(); - case 3: - return wrapped.fileSequenceNumber(); - case 4: - return fileWrapper.wrap(wrapped.file()); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + yield wrapped.dataSequenceNumber(); + } + case 3 -> wrapped.fileSequenceNumber(); + case 4 -> fileWrapper.wrap(wrapped.file()); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override @@ -470,65 +455,53 @@ private Object get(int pos) { // when the partition field is omitted, positions at or after where it would appear // shift down by 1, so adjust back to the canonical field ordering int adjusted = hasPartition ? pos : (pos >= PARTITION_POSITION ? pos + 1 : pos); - switch (adjusted) { - case 0: - return wrapped.content().id(); - case 1: - return wrapped.location(); - case 2: - return wrapped.format() != null ? wrapped.format().toString() : null; - case 3: - return wrapped.partition(); - case 4: - return wrapped.recordCount(); - case 5: - return wrapped.fileSizeInBytes(); - case 6: - return wrapped.columnSizes(); - case 7: - return wrapped.valueCounts(); - case 8: - return wrapped.nullValueCounts(); - case 9: - return wrapped.nanValueCounts(); - case 10: - return wrapped.lowerBounds(); - case 11: - return wrapped.upperBounds(); - case 12: - return wrapped.keyMetadata(); - case 13: - return wrapped.splitOffsets(); - case 14: - return wrapped.equalityFieldIds(); - case 15: - return wrapped.sortOrderId(); - case 16: + return switch (adjusted) { + case 0 -> wrapped.content().id(); + case 1 -> wrapped.location(); + case 2 -> wrapped.format() != null ? wrapped.format().toString() : null; + case 3 -> wrapped.partition(); + case 4 -> wrapped.recordCount(); + case 5 -> wrapped.fileSizeInBytes(); + case 6 -> wrapped.columnSizes(); + case 7 -> wrapped.valueCounts(); + case 8 -> wrapped.nullValueCounts(); + case 9 -> wrapped.nanValueCounts(); + case 10 -> wrapped.lowerBounds(); + case 11 -> wrapped.upperBounds(); + case 12 -> wrapped.keyMetadata(); + case 13 -> wrapped.splitOffsets(); + case 14 -> wrapped.equalityFieldIds(); + case 15 -> wrapped.sortOrderId(); + case 16 -> { if (wrapped.content() == FileContent.DATA) { - return wrapped.firstRowId(); + yield wrapped.firstRowId(); } else { - return null; + yield null; } - case 17: + } + case 17 -> { if (wrapped.content() == FileContent.POSITION_DELETES) { - return ((DeleteFile) wrapped).referencedDataFile(); + yield ((DeleteFile) wrapped).referencedDataFile(); } else { - return null; + yield null; } - case 18: + } + case 18 -> { if (wrapped.content() == FileContent.POSITION_DELETES) { - return ((DeleteFile) wrapped).contentOffset(); + yield ((DeleteFile) wrapped).contentOffset(); } else { - return null; + yield null; } - case 19: + } + case 19 -> { if (wrapped.content() == FileContent.POSITION_DELETES) { - return ((DeleteFile) wrapped).contentSizeInBytes(); + yield ((DeleteFile) wrapped).contentSizeInBytes(); } else { - return null; + yield null; } - } - throw new IllegalArgumentException("Unknown field ordinal: " + pos); + } + default -> throw new IllegalArgumentException("Unknown field ordinal: " + pos); + }; } @Override diff --git a/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java b/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java index 72d7f4f65bb0..5f3d70de5cb1 100644 --- a/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java +++ b/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java @@ -104,18 +104,14 @@ public int outputSpecId() { } public static Comparator comparator(RewriteJobOrder rewriteJobOrder) { - switch (rewriteJobOrder) { - case BYTES_ASC: - return Comparator.comparing(RewriteFileGroup::inputFilesSizeInBytes); - case BYTES_DESC: - return Comparator.comparing( - RewriteFileGroup::inputFilesSizeInBytes, Comparator.reverseOrder()); - case FILES_ASC: - return Comparator.comparing(RewriteFileGroup::inputFileNum); - case FILES_DESC: - return Comparator.comparing(RewriteFileGroup::inputFileNum, Comparator.reverseOrder()); - default: - return (unused, unused2) -> 0; - } + return switch (rewriteJobOrder) { + case BYTES_ASC -> Comparator.comparing(RewriteFileGroup::inputFilesSizeInBytes); + case BYTES_DESC -> + Comparator.comparing(RewriteFileGroup::inputFilesSizeInBytes, Comparator.reverseOrder()); + case FILES_ASC -> Comparator.comparing(RewriteFileGroup::inputFileNum); + case FILES_DESC -> + Comparator.comparing(RewriteFileGroup::inputFileNum, Comparator.reverseOrder()); + default -> (unused, unused2) -> 0; + }; } } diff --git a/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java b/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java index d5ffe7f1e01b..342cb1d4350a 100644 --- a/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java +++ b/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java @@ -108,19 +108,16 @@ public long addedBytes() { } public static Comparator comparator(RewriteJobOrder order) { - switch (order) { - case BYTES_ASC: - return Comparator.comparing(RewritePositionDeletesGroup::inputFilesSizeInBytes); - case BYTES_DESC: - return Comparator.comparing( - RewritePositionDeletesGroup::inputFilesSizeInBytes, Comparator.reverseOrder()); - case FILES_ASC: - return Comparator.comparing(RewritePositionDeletesGroup::inputFileNum); - case FILES_DESC: - return Comparator.comparing( - RewritePositionDeletesGroup::inputFileNum, Comparator.reverseOrder()); - default: - return (unused, unused2) -> 0; - } + return switch (order) { + case BYTES_ASC -> Comparator.comparing(RewritePositionDeletesGroup::inputFilesSizeInBytes); + case BYTES_DESC -> + Comparator.comparing( + RewritePositionDeletesGroup::inputFilesSizeInBytes, Comparator.reverseOrder()); + case FILES_ASC -> Comparator.comparing(RewritePositionDeletesGroup::inputFileNum); + case FILES_DESC -> + Comparator.comparing( + RewritePositionDeletesGroup::inputFileNum, Comparator.reverseOrder()); + default -> (unused, unused2) -> 0; + }; } } diff --git a/core/src/main/java/org/apache/iceberg/avro/Avro.java b/core/src/main/java/org/apache/iceberg/avro/Avro.java index 4a5136f58e71..1b1c234ef6d3 100644 --- a/core/src/main/java/org/apache/iceberg/avro/Avro.java +++ b/core/src/main/java/org/apache/iceberg/avro/Avro.java @@ -248,26 +248,17 @@ static Context deleteContext(Map config) { private static CodecFactory toCodec(String codecAsString, String compressionLevel) { CodecFactory codecFactory; try { - switch (Codec.valueOf(codecAsString.toUpperCase(Locale.ENGLISH))) { - case UNCOMPRESSED: - codecFactory = CodecFactory.nullCodec(); - break; - case SNAPPY: - codecFactory = CodecFactory.snappyCodec(); - break; - case ZSTD: - codecFactory = - CodecFactory.zstandardCodec( - compressionLevelAsInt(compressionLevel, ZSTD_COMPRESSION_LEVEL_DEFAULT)); - break; - case GZIP: - codecFactory = - CodecFactory.deflateCodec( - compressionLevelAsInt(compressionLevel, GZIP_COMPRESSION_LEVEL_DEFAULT)); - break; - default: - throw new IllegalArgumentException("Unsupported compression codec: " + codecAsString); - } + codecFactory = + switch (Codec.valueOf(codecAsString.toUpperCase(Locale.ENGLISH))) { + case UNCOMPRESSED -> CodecFactory.nullCodec(); + case SNAPPY -> CodecFactory.snappyCodec(); + case ZSTD -> + CodecFactory.zstandardCodec( + compressionLevelAsInt(compressionLevel, ZSTD_COMPRESSION_LEVEL_DEFAULT)); + case GZIP -> + CodecFactory.deflateCodec( + compressionLevelAsInt(compressionLevel, GZIP_COMPRESSION_LEVEL_DEFAULT)); + }; } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Unsupported compression codec: " + codecAsString); } diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroCustomOrderSchemaVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroCustomOrderSchemaVisitor.java index f579381ac0c8..52c56cf66c96 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroCustomOrderSchemaVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroCustomOrderSchemaVisitor.java @@ -31,8 +31,8 @@ abstract class AvroCustomOrderSchemaVisitor { private static final String VALUE = "value"; public static T visit(Schema schema, AvroCustomOrderSchemaVisitor visitor) { - switch (schema.getType()) { - case RECORD: + return switch (schema.getType()) { + case RECORD -> { // check to make sure this hasn't been visited before String name = schema.getFullName(); Preconditions.checkState( @@ -42,7 +42,7 @@ public static T visit(Schema schema, AvroCustomOrderSchemaVisitor v Preconditions.checkArgument( AvroSchemaUtil.isVariantSchema(schema), "Invalid variant record: %s", schema); - return visitor.variant( + yield visitor.variant( schema, new VisitFuture<>(schema.getField(METADATA).schema(), visitor), new VisitFuture<>(schema.getField(VALUE).schema(), visitor)); @@ -58,26 +58,21 @@ public static T visit(Schema schema, AvroCustomOrderSchemaVisitor v } visitor.recordLevels.pop(); - return visitor.record(schema, names, Iterables.transform(results, Supplier::get)); + yield visitor.record(schema, names, Iterables.transform(results, Supplier::get)); } - - case UNION: + } + case UNION -> { List types = schema.getTypes(); List> options = Lists.newArrayListWithExpectedSize(types.size()); for (Schema type : types) { options.add(new VisitFuture<>(type, visitor)); } - return visitor.union(schema, Iterables.transform(options, Supplier::get)); - - case ARRAY: - return visitor.array(schema, new VisitFuture<>(schema.getElementType(), visitor)); - - case MAP: - return visitor.map(schema, new VisitFuture<>(schema.getValueType(), visitor)); - - default: - return visitor.primitive(schema); - } + yield visitor.union(schema, Iterables.transform(options, Supplier::get)); + } + case ARRAY -> visitor.array(schema, new VisitFuture<>(schema.getElementType(), visitor)); + case MAP -> visitor.map(schema, new VisitFuture<>(schema.getValueType(), visitor)); + default -> visitor.primitive(schema); + }; } private final Deque recordLevels = Lists.newLinkedList(); diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroFormatModel.java b/core/src/main/java/org/apache/iceberg/avro/AvroFormatModel.java index e0fcf8952604..441e923f0ac0 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroFormatModel.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroFormatModel.java @@ -159,17 +159,17 @@ public ModelWriteBuilder withAADPrefix(ByteBuffer aadPrefix) { @Override public FileAppender build() throws IOException { switch (content) { - case DATA: + case DATA -> { internal.createContextFunc(Avro.WriteBuilder.Context::dataContext); internal.createWriterFunc( avroSchema -> writerFunction.write(schema, avroSchema, engineSchema)); - break; - case EQUALITY_DELETES: + } + case EQUALITY_DELETES -> { internal.createContextFunc(Avro.WriteBuilder.Context::deleteContext); internal.createWriterFunc( avroSchema -> writerFunction.write(schema, avroSchema, engineSchema)); - break; - case POSITION_DELETES: + } + case POSITION_DELETES -> { Preconditions.checkState( schema == null, "Invalid schema: %s. Position deletes with schema are not supported by the API.", @@ -182,9 +182,8 @@ public FileAppender build() throws IOException { internal.createContextFunc(Avro.WriteBuilder.Context::deleteContext); internal.createWriterFunc(unused -> new Avro.PositionDatumWriter()); internal.schema(DeleteSchemaUtil.pathPosSchema()); - break; - default: - throw new IllegalArgumentException("Unknown file content: " + content); + } + default -> throw new IllegalArgumentException("Unknown file content: " + content); } return internal.build(); diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java index c67a3089a6bf..a5ca34314815 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java @@ -189,16 +189,15 @@ public static boolean isOptionSchema(Schema schema) { } static Schema toOption(Schema schema) { - switch (schema.getType()) { - case UNION: + return switch (schema.getType()) { + case UNION -> { Preconditions.checkArgument( isOptionSchema(schema), "Union schemas are not supported: %s", schema); - return schema; - case NULL: - return schema; - default: - return Schema.createUnion(NULL, schema); - } + yield schema; + } + case NULL -> schema; + default -> Schema.createUnion(NULL, schema); + }; } static Schema fromOption(Schema schema) { diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaVisitor.java index df3ca72fa09f..ac31ca9ccb5d 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaVisitor.java @@ -29,8 +29,8 @@ public abstract class AvroSchemaVisitor { private static final String VALUE = "value"; public static T visit(Schema schema, AvroSchemaVisitor visitor) { - switch (schema.getType()) { - case RECORD: + return switch (schema.getType()) { + case RECORD -> { // check to make sure this hasn't been visited before String name = schema.getFullName(); Preconditions.checkState( @@ -40,7 +40,7 @@ public static T visit(Schema schema, AvroSchemaVisitor visitor) { Preconditions.checkArgument( AvroSchemaUtil.isVariantSchema(schema), "Invalid variant record: %s", schema); - return visitor.variant( + yield visitor.variant( schema, visit(schema.getField(METADATA).schema(), visitor), visit(schema.getField(VALUE).schema(), visitor)); @@ -57,30 +57,27 @@ public static T visit(Schema schema, AvroSchemaVisitor visitor) { } visitor.recordLevels.pop(); - return visitor.record(schema, names, results); + yield visitor.record(schema, names, results); } - - case UNION: + } + case UNION -> { List types = schema.getTypes(); List options = Lists.newArrayListWithExpectedSize(types.size()); for (Schema type : types) { options.add(visit(type, visitor)); } - return visitor.union(schema, options); - - case ARRAY: + yield visitor.union(schema, options); + } + case ARRAY -> { if (schema.getLogicalType() instanceof LogicalMap) { - return visitor.array(schema, visit(schema.getElementType(), visitor)); + yield visitor.array(schema, visit(schema.getElementType(), visitor)); } else { - return visitor.array(schema, visitWithName("element", schema.getElementType(), visitor)); + yield visitor.array(schema, visitWithName("element", schema.getElementType(), visitor)); } - - case MAP: - return visitor.map(schema, visitWithName("value", schema.getValueType(), visitor)); - - default: - return visitor.primitive(schema); - } + } + case MAP -> visitor.map(schema, visitWithName("value", schema.getValueType(), visitor)); + default -> visitor.primitive(schema); + }; } private final Deque recordLevels = Lists.newLinkedList(); diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaWithTypeVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaWithTypeVisitor.java index 45892d3de151..c6542c2555fe 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaWithTypeVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaWithTypeVisitor.java @@ -33,26 +33,19 @@ public static T visit( } public static T visit(Type iType, Schema schema, AvroSchemaWithTypeVisitor visitor) { - switch (schema.getType()) { - case RECORD: - return visitRecord(iType != null ? iType.asStructType() : null, schema, visitor); - - case UNION: - return visitUnion(iType, schema, visitor); - - case ARRAY: - return visitArray(iType, schema, visitor); - - case MAP: + return switch (schema.getType()) { + case RECORD -> visitRecord(iType != null ? iType.asStructType() : null, schema, visitor); + case UNION -> visitUnion(iType, schema, visitor); + case ARRAY -> visitArray(iType, schema, visitor); + case MAP -> { Types.MapType map = iType != null ? iType.asMapType() : null; - return visitor.map( + yield visitor.map( map, schema, visit(map != null ? map.valueType() : null, schema.getValueType(), visitor)); - - default: - return visitor.primitive(iType != null ? iType.asPrimitiveType() : null, schema); - } + } + default -> visitor.primitive(iType != null ? iType.asPrimitiveType() : null, schema); + }; } private static T visitRecord( diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerByStructureVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerByStructureVisitor.java index 991ae12664c4..5df3b65e62dd 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerByStructureVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerByStructureVisitor.java @@ -37,31 +37,26 @@ public abstract class AvroWithPartnerByStructureVisitor { public static T visit( P partner, Schema schema, AvroWithPartnerByStructureVisitor visitor) { - switch (schema.getType()) { - case RECORD: + return switch (schema.getType()) { + case RECORD -> { if (schema.getLogicalType() instanceof VariantLogicalType || visitor.isVariantType(partner)) { - return visitVariant(partner, schema, visitor); + yield visitVariant(partner, schema, visitor); } else { - return visitRecord(partner, schema, visitor); + yield visitRecord(partner, schema, visitor); } - - case UNION: - return visitUnion(partner, schema, visitor); - - case ARRAY: - return visitArray(partner, schema, visitor); - - case MAP: + } + case UNION -> visitUnion(partner, schema, visitor); + case ARRAY -> visitArray(partner, schema, visitor); + case MAP -> { P keyType = visitor.mapKeyType(partner); Preconditions.checkArgument( visitor.isStringType(keyType), "Invalid map: %s is not a string", keyType); - return visitor.map( + yield visitor.map( partner, schema, visit(visitor.mapValueType(partner), schema.getValueType(), visitor)); - - default: - return visitor.primitive(partner, schema); - } + } + default -> visitor.primitive(partner, schema); + }; } // ---------------------------------- Static helpers --------------------------------------------- diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerVisitor.java index 83ddc9be5e29..6bc09d635ac8 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerVisitor.java @@ -102,33 +102,27 @@ public static R visit( Schema schema, AvroWithPartnerVisitor visitor, PartnerAccessors

accessors) { - switch (schema.getType()) { - case RECORD: + return switch (schema.getType()) { + case RECORD -> { if (schema.getLogicalType() instanceof VariantLogicalType) { - return visitVariant(partner, schema, visitor, accessors); + yield visitVariant(partner, schema, visitor, accessors); } else { - return visitRecord(partner, schema, visitor, accessors); + yield visitRecord(partner, schema, visitor, accessors); } - - case UNION: - return visitUnion(partner, schema, visitor, accessors); - - case ARRAY: - return visitArray(partner, schema, visitor, accessors); - - case MAP: - return visitor.map( - partner, - schema, - visit( - partner != null ? accessors.mapValuePartner(partner) : null, - schema.getValueType(), - visitor, - accessors)); - - default: - return visitor.primitive(partner, schema); - } + } + case UNION -> visitUnion(partner, schema, visitor, accessors); + case ARRAY -> visitArray(partner, schema, visitor, accessors); + case MAP -> + visitor.map( + partner, + schema, + visit( + partner != null ? accessors.mapValuePartner(partner) : null, + schema.getValueType(), + visitor, + accessors)); + default -> visitor.primitive(partner, schema); + }; } private static R visitVariant( diff --git a/core/src/main/java/org/apache/iceberg/avro/BaseWriteBuilder.java b/core/src/main/java/org/apache/iceberg/avro/BaseWriteBuilder.java index 51e02eaade82..6ddcb6bc9b89 100644 --- a/core/src/main/java/org/apache/iceberg/avro/BaseWriteBuilder.java +++ b/core/src/main/java/org/apache/iceberg/avro/BaseWriteBuilder.java @@ -74,52 +74,31 @@ public ValueWriter variant( public ValueWriter primitive(Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - switch (logicalType.getName()) { - case "date": - return ValueWriters.ints(); - - case "time-micros": - return ValueWriters.longs(); - - case "timestamp-micros": - return ValueWriters.longs(); - - case "timestamp-nanos": - return ValueWriters.longs(); - - case "decimal": + return switch (logicalType.getName()) { + case "date" -> ValueWriters.ints(); + case "time-micros" -> ValueWriters.longs(); + case "timestamp-micros" -> ValueWriters.longs(); + case "timestamp-nanos" -> ValueWriters.longs(); + case "decimal" -> { LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; - return ValueWriters.decimal(decimal.getPrecision(), decimal.getScale()); - - case "uuid": - return ValueWriters.uuids(); - - default: - throw new IllegalArgumentException("Unsupported logical type: " + logicalType); - } + yield ValueWriters.decimal(decimal.getPrecision(), decimal.getScale()); + } + case "uuid" -> ValueWriters.uuids(); + default -> throw new IllegalArgumentException("Unsupported logical type: " + logicalType); + }; } - switch (primitive.getType()) { - case NULL: - return ValueWriters.nulls(); - case BOOLEAN: - return ValueWriters.booleans(); - case INT: - return ValueWriters.ints(); - case LONG: - return ValueWriters.longs(); - case FLOAT: - return ValueWriters.floats(); - case DOUBLE: - return ValueWriters.doubles(); - case STRING: - return ValueWriters.strings(); - case FIXED: - return fixedWriter(primitive.getFixedSize()); - case BYTES: - return ValueWriters.byteBuffers(); - default: - throw new IllegalArgumentException("Unsupported type: " + primitive); - } + return switch (primitive.getType()) { + case NULL -> ValueWriters.nulls(); + case BOOLEAN -> ValueWriters.booleans(); + case INT -> ValueWriters.ints(); + case LONG -> ValueWriters.longs(); + case FLOAT -> ValueWriters.floats(); + case DOUBLE -> ValueWriters.doubles(); + case STRING -> ValueWriters.strings(); + case FIXED -> fixedWriter(primitive.getFixedSize()); + case BYTES -> ValueWriters.byteBuffers(); + default -> throw new IllegalArgumentException("Unsupported type: " + primitive); + }; } } diff --git a/core/src/main/java/org/apache/iceberg/avro/BuildAvroProjection.java b/core/src/main/java/org/apache/iceberg/avro/BuildAvroProjection.java index f4dd2f41302d..5a16db9d9619 100644 --- a/core/src/main/java/org/apache/iceberg/avro/BuildAvroProjection.java +++ b/core/src/main/java/org/apache/iceberg/avro/BuildAvroProjection.java @@ -273,21 +273,20 @@ public Schema variant(Schema variant, Supplier metadata, Supplier { if (current.typeId() == Type.TypeID.LONG) { - return Schema.create(Schema.Type.LONG); + yield Schema.create(Schema.Type.LONG); } - return primitive; - - case FLOAT: + yield primitive; + } + case FLOAT -> { if (current.typeId() == Type.TypeID.DOUBLE) { - return Schema.create(Schema.Type.DOUBLE); + yield Schema.create(Schema.Type.DOUBLE); } - return primitive; - - default: - return primitive; - } + yield primitive; + } + default -> primitive; + }; } } diff --git a/core/src/main/java/org/apache/iceberg/avro/GenericAvroReader.java b/core/src/main/java/org/apache/iceberg/avro/GenericAvroReader.java index fc2d44f47060..47b3ab91518c 100644 --- a/core/src/main/java/org/apache/iceberg/avro/GenericAvroReader.java +++ b/core/src/main/java/org/apache/iceberg/avro/GenericAvroReader.java @@ -173,67 +173,47 @@ public ValueReader variant( public ValueReader primitive(Type partner, Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - switch (logicalType.getName()) { - case "date": + return switch (logicalType.getName()) { // Spark uses the same representation - return ValueReaders.ints(); - - case "time-micros": - return ValueReaders.longs(); - - case "timestamp-millis": - // adjust to microseconds - ValueReader longs = ValueReaders.longs(); - return (ValueReader) (decoder, ignored) -> longs.read(decoder, null) * 1000L; - - case "timestamp-micros": - case "timestamp-nanos": - // both are handled in memory as long values, using the type to track units - return ValueReaders.longs(); - - case "decimal": - return ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - - case "uuid": - return ValueReaders.uuids(); - - default: - throw new IllegalArgumentException("Unknown logical type: " + logicalType); - } + case "date" -> ValueReaders.ints(); + case "time-micros" -> ValueReaders.longs(); + case "timestamp-millis" -> // adjust to microseconds + (decoder, ignored) -> ValueReaders.longs().read(decoder, null) * 1000L; + // both timestamp-micros and timestamp-nanos are handled in memory as long values, + // using the type to track units + case "timestamp-micros", "timestamp-nanos" -> ValueReaders.longs(); + case "decimal" -> + ValueReaders.decimal( + ValueReaders.decimalBytesReader(primitive), + ((LogicalTypes.Decimal) logicalType).getScale()); + case "uuid" -> ValueReaders.uuids(); + default -> throw new IllegalArgumentException("Unknown logical type: " + logicalType); + }; } - switch (primitive.getType()) { - case NULL: - return ValueReaders.nulls(); - case BOOLEAN: - return ValueReaders.booleans(); - case INT: + return switch (primitive.getType()) { + case NULL -> ValueReaders.nulls(); + case BOOLEAN -> ValueReaders.booleans(); + case INT -> { if (partner != null && partner.typeId() == Type.TypeID.LONG) { - return ValueReaders.intsAsLongs(); + yield ValueReaders.intsAsLongs(); } - return ValueReaders.ints(); - case LONG: - return ValueReaders.longs(); - case FLOAT: + yield ValueReaders.ints(); + } + case LONG -> ValueReaders.longs(); + case FLOAT -> { if (partner != null && partner.typeId() == Type.TypeID.DOUBLE) { - return ValueReaders.floatsAsDoubles(); + yield ValueReaders.floatsAsDoubles(); } - return ValueReaders.floats(); - case DOUBLE: - return ValueReaders.doubles(); - case STRING: - return ValueReaders.utf8s(); - case FIXED: - return ValueReaders.fixed(primitive); - case BYTES: - return ValueReaders.byteBuffers(); - case ENUM: - return ValueReaders.enums(primitive.getEnumSymbols()); - default: - throw new IllegalArgumentException("Unsupported type: " + primitive); - } + yield ValueReaders.floats(); + } + case DOUBLE -> ValueReaders.doubles(); + case STRING -> ValueReaders.utf8s(); + case FIXED -> ValueReaders.fixed(primitive); + case BYTES -> ValueReaders.byteBuffers(); + case ENUM -> ValueReaders.enums(primitive.getEnumSymbols()); + default -> throw new IllegalArgumentException("Unsupported type: " + primitive); + }; } } } diff --git a/core/src/main/java/org/apache/iceberg/avro/InternalReader.java b/core/src/main/java/org/apache/iceberg/avro/InternalReader.java index af3c4f1a822b..fe7fec7f6e5f 100644 --- a/core/src/main/java/org/apache/iceberg/avro/InternalReader.java +++ b/core/src/main/java/org/apache/iceberg/avro/InternalReader.java @@ -170,65 +170,45 @@ public ValueReader variant( public ValueReader primitive(Pair partner, Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - switch (logicalType.getName()) { - case "date": - return ValueReaders.ints(); - - case "time-micros": - return ValueReaders.longs(); - - case "timestamp-millis": - // adjust to microseconds - ValueReader longs = ValueReaders.longs(); - return (ValueReader) (decoder, ignored) -> longs.read(decoder, null) * 1000L; - - case "timestamp-micros": - case "timestamp-nanos": - // both are handled in memory as long values, using the type to track units - return ValueReaders.longs(); - - case "decimal": - return ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - - case "uuid": - return ValueReaders.uuids(); - - default: - throw new IllegalArgumentException("Unknown logical type: " + logicalType); - } + return switch (logicalType.getName()) { + case "date" -> ValueReaders.ints(); + case "time-micros" -> ValueReaders.longs(); + case "timestamp-millis" -> // adjust to microseconds + (decoder, ignored) -> ValueReaders.longs().read(decoder, null) * 1000L; + // both timestamp-micros and timestamp-nanos are handled in memory as long values, + // using the type to track units + case "timestamp-micros", "timestamp-nanos" -> ValueReaders.longs(); + case "decimal" -> + ValueReaders.decimal( + ValueReaders.decimalBytesReader(primitive), + ((LogicalTypes.Decimal) logicalType).getScale()); + case "uuid" -> ValueReaders.uuids(); + default -> throw new IllegalArgumentException("Unknown logical type: " + logicalType); + }; } - switch (primitive.getType()) { - case NULL: - return ValueReaders.nulls(); - case BOOLEAN: - return ValueReaders.booleans(); - case INT: + return switch (primitive.getType()) { + case NULL -> ValueReaders.nulls(); + case BOOLEAN -> ValueReaders.booleans(); + case INT -> { if (partner != null && partner.second().typeId() == Type.TypeID.LONG) { - return ValueReaders.intsAsLongs(); + yield ValueReaders.intsAsLongs(); } - return ValueReaders.ints(); - case LONG: - return ValueReaders.longs(); - case FLOAT: + yield ValueReaders.ints(); + } + case LONG -> ValueReaders.longs(); + case FLOAT -> { if (partner != null && partner.second().typeId() == Type.TypeID.DOUBLE) { - return ValueReaders.floatsAsDoubles(); + yield ValueReaders.floatsAsDoubles(); } - return ValueReaders.floats(); - case DOUBLE: - return ValueReaders.doubles(); - case STRING: - return ValueReaders.strings(); - case FIXED: - case BYTES: - return ValueReaders.byteBuffers(); - case ENUM: - return ValueReaders.enums(primitive.getEnumSymbols()); - default: - throw new IllegalArgumentException("Unsupported type: " + primitive); - } + yield ValueReaders.floats(); + } + case DOUBLE -> ValueReaders.doubles(); + case STRING -> ValueReaders.strings(); + case FIXED, BYTES -> ValueReaders.byteBuffers(); + case ENUM -> ValueReaders.enums(primitive.getEnumSymbols()); + default -> throw new IllegalArgumentException("Unsupported type: " + primitive); + }; } } diff --git a/core/src/main/java/org/apache/iceberg/avro/SchemaToType.java b/core/src/main/java/org/apache/iceberg/avro/SchemaToType.java index 2ffd658f447e..02c9c5f1e1b2 100644 --- a/core/src/main/java/org/apache/iceberg/avro/SchemaToType.java +++ b/core/src/main/java/org/apache/iceberg/avro/SchemaToType.java @@ -224,28 +224,18 @@ public Type primitive(Schema primitive) { } } - switch (primitive.getType()) { - case BOOLEAN: - return Types.BooleanType.get(); - case INT: - return Types.IntegerType.get(); - case LONG: - return Types.LongType.get(); - case FLOAT: - return Types.FloatType.get(); - case DOUBLE: - return Types.DoubleType.get(); - case STRING: - case ENUM: - return Types.StringType.get(); - case FIXED: - return Types.FixedType.ofLength(primitive.getFixedSize()); - case BYTES: - return Types.BinaryType.get(); - case NULL: - return Types.UnknownType.get(); - } - - throw new UnsupportedOperationException("Unsupported primitive type: " + primitive); + return switch (primitive.getType()) { + case BOOLEAN -> Types.BooleanType.get(); + case INT -> Types.IntegerType.get(); + case LONG -> Types.LongType.get(); + case FLOAT -> Types.FloatType.get(); + case DOUBLE -> Types.DoubleType.get(); + case STRING, ENUM -> Types.StringType.get(); + case FIXED -> Types.FixedType.ofLength(primitive.getFixedSize()); + case BYTES -> Types.BinaryType.get(); + case NULL -> Types.UnknownType.get(); + default -> + throw new UnsupportedOperationException("Unsupported primitive type: " + primitive); + }; } } diff --git a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java index 4442695b7a44..b62700dba5f1 100644 --- a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java +++ b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java @@ -213,58 +213,36 @@ public Schema variant(Types.VariantType variant) { public Schema primitive(Type.PrimitiveType primitive) { Schema primitiveSchema; switch (primitive.typeId()) { - case UNKNOWN: - primitiveSchema = NULL_SCHEMA; - break; - case BOOLEAN: - primitiveSchema = BOOLEAN_SCHEMA; - break; - case INTEGER: - primitiveSchema = INTEGER_SCHEMA; - break; - case LONG: - primitiveSchema = LONG_SCHEMA; - break; - case FLOAT: - primitiveSchema = FLOAT_SCHEMA; - break; - case DOUBLE: - primitiveSchema = DOUBLE_SCHEMA; - break; - case DATE: - primitiveSchema = DATE_SCHEMA; - break; - case TIME: - primitiveSchema = TIME_SCHEMA; - break; - case TIMESTAMP: + case UNKNOWN -> primitiveSchema = NULL_SCHEMA; + case BOOLEAN -> primitiveSchema = BOOLEAN_SCHEMA; + case INTEGER -> primitiveSchema = INTEGER_SCHEMA; + case LONG -> primitiveSchema = LONG_SCHEMA; + case FLOAT -> primitiveSchema = FLOAT_SCHEMA; + case DOUBLE -> primitiveSchema = DOUBLE_SCHEMA; + case DATE -> primitiveSchema = DATE_SCHEMA; + case TIME -> primitiveSchema = TIME_SCHEMA; + case TIMESTAMP -> { if (((Types.TimestampType) primitive).shouldAdjustToUTC()) { primitiveSchema = TIMESTAMPTZ_SCHEMA; } else { primitiveSchema = TIMESTAMP_SCHEMA; } - break; - case TIMESTAMP_NANO: + } + case TIMESTAMP_NANO -> { if (((Types.TimestampNanoType) primitive).shouldAdjustToUTC()) { primitiveSchema = TIMESTAMPTZ_NANO_SCHEMA; } else { primitiveSchema = TIMESTAMP_NANO_SCHEMA; } - break; - case STRING: - primitiveSchema = STRING_SCHEMA; - break; - case UUID: - primitiveSchema = UUID_SCHEMA; - break; - case FIXED: + } + case STRING -> primitiveSchema = STRING_SCHEMA; + case UUID -> primitiveSchema = UUID_SCHEMA; + case FIXED -> { Types.FixedType fixed = (Types.FixedType) primitive; primitiveSchema = Schema.createFixed("fixed_" + fixed.length(), null, null, fixed.length()); - break; - case BINARY: - primitiveSchema = BINARY_SCHEMA; - break; - case DECIMAL: + } + case BINARY -> primitiveSchema = BINARY_SCHEMA; + case DECIMAL -> { Types.DecimalType decimal = (Types.DecimalType) primitive; primitiveSchema = LogicalTypes.decimal(decimal.precision(), decimal.scale()) @@ -274,9 +252,9 @@ public Schema primitive(Type.PrimitiveType primitive) { null, null, TypeUtil.decimalRequiredBytes(decimal.precision()))); - break; - default: - throw new UnsupportedOperationException("Unsupported type ID: " + primitive.typeId()); + } + default -> + throw new UnsupportedOperationException("Unsupported type ID: " + primitive.typeId()); } cacheSchema(primitive, primitiveSchema); diff --git a/core/src/main/java/org/apache/iceberg/avro/ValueReaders.java b/core/src/main/java/org/apache/iceberg/avro/ValueReaders.java index ec46c56c72c3..ce5227f90220 100644 --- a/core/src/main/java/org/apache/iceberg/avro/ValueReaders.java +++ b/core/src/main/java/org/apache/iceberg/avro/ValueReaders.java @@ -133,15 +133,13 @@ public static ValueReader decimal(ValueReader unscaledReader } public static ValueReader decimalBytesReader(Schema schema) { - switch (schema.getType()) { - case FIXED: - return ValueReaders.fixed(schema.getFixedSize()); - case BYTES: - return ValueReaders.bytes(); - default: - throw new IllegalArgumentException( - "Invalid primitive type for decimal: " + schema.getType()); - } + return switch (schema.getType()) { + case FIXED -> ValueReaders.fixed(schema.getFixedSize()); + case BYTES -> ValueReaders.bytes(); + default -> + throw new IllegalArgumentException( + "Invalid primitive type for decimal: " + schema.getType()); + }; } public static ValueReader variants() { diff --git a/core/src/main/java/org/apache/iceberg/data/GenericDataUtil.java b/core/src/main/java/org/apache/iceberg/data/GenericDataUtil.java index 9b720c1f865a..01ca179fd9ec 100644 --- a/core/src/main/java/org/apache/iceberg/data/GenericDataUtil.java +++ b/core/src/main/java/org/apache/iceberg/data/GenericDataUtil.java @@ -40,27 +40,25 @@ public static Object internalToGeneric(Type type, Object value) { return null; } - switch (type.typeId()) { - case DATE: - return DateTimeUtil.dateFromDays((Integer) value); - case TIME: - return DateTimeUtil.timeFromMicros((Long) value); - case TIMESTAMP: + return switch (type.typeId()) { + case DATE -> DateTimeUtil.dateFromDays((Integer) value); + case TIME -> DateTimeUtil.timeFromMicros((Long) value); + case TIMESTAMP -> { if (((Types.TimestampType) type).shouldAdjustToUTC()) { - return DateTimeUtil.timestamptzFromMicros((Long) value); + yield DateTimeUtil.timestamptzFromMicros((Long) value); } else { - return DateTimeUtil.timestampFromMicros((Long) value); + yield DateTimeUtil.timestampFromMicros((Long) value); } - case TIMESTAMP_NANO: + } + case TIMESTAMP_NANO -> { if (((Types.TimestampNanoType) type).shouldAdjustToUTC()) { - return DateTimeUtil.timestamptzFromNanos((Long) value); + yield DateTimeUtil.timestamptzFromNanos((Long) value); } else { - return DateTimeUtil.timestampFromNanos((Long) value); + yield DateTimeUtil.timestampFromNanos((Long) value); } - case FIXED: - return ByteBuffers.toByteArray((ByteBuffer) value); - } - - return value; + } + case FIXED -> ByteBuffers.toByteArray((ByteBuffer) value); + default -> value; + }; } } diff --git a/core/src/main/java/org/apache/iceberg/data/IdentityPartitionConverters.java b/core/src/main/java/org/apache/iceberg/data/IdentityPartitionConverters.java index 4cb41263152d..382493ac70e7 100644 --- a/core/src/main/java/org/apache/iceberg/data/IdentityPartitionConverters.java +++ b/core/src/main/java/org/apache/iceberg/data/IdentityPartitionConverters.java @@ -32,26 +32,24 @@ public static Object convertConstant(Type type, Object value) { return null; } - switch (type.typeId()) { - case STRING: - return value.toString(); - case TIME: - return DateTimeUtil.timeFromMicros((Long) value); - case DATE: - return DateTimeUtil.dateFromDays((Integer) value); - case TIMESTAMP: + return switch (type.typeId()) { + case STRING -> value.toString(); + case TIME -> DateTimeUtil.timeFromMicros((Long) value); + case DATE -> DateTimeUtil.dateFromDays((Integer) value); + case TIMESTAMP -> { if (((Types.TimestampType) type).shouldAdjustToUTC()) { - return DateTimeUtil.timestamptzFromMicros((Long) value); + yield DateTimeUtil.timestamptzFromMicros((Long) value); } else { - return DateTimeUtil.timestampFromMicros((Long) value); + yield DateTimeUtil.timestampFromMicros((Long) value); } - case FIXED: + } + case FIXED -> { if (value instanceof GenericData.Fixed) { - return ((GenericData.Fixed) value).bytes(); + yield ((GenericData.Fixed) value).bytes(); } - return value; - default: - } - return value; + yield value; + } + default -> value; + }; } } diff --git a/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java b/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java index 5f813f8db576..6fe260239426 100644 --- a/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java +++ b/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java @@ -127,67 +127,50 @@ public ValueReader map(Types.MapType ignored, Schema map, ValueReader valu public ValueReader primitive(Type.PrimitiveType ignored, Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - switch (logicalType.getName()) { - case "date": - return GenericReaders.dates(); - - case "time-micros": - return GenericReaders.times(); - - case "timestamp-micros": + return switch (logicalType.getName()) { + case "date" -> GenericReaders.dates(); + case "time-micros" -> GenericReaders.times(); + case "timestamp-micros" -> { if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptz(); + yield GenericReaders.timestamptz(); } - return GenericReaders.timestamps(); - - case "timestamp-nanos": + yield GenericReaders.timestamps(); + } + case "timestamp-nanos" -> { if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptzNanos(); + yield GenericReaders.timestamptzNanos(); } - return GenericReaders.timestampNanos(); - - case "timestamp-millis": + yield GenericReaders.timestampNanos(); + } + case "timestamp-millis" -> { if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptzMillis(); + yield GenericReaders.timestamptzMillis(); } - return GenericReaders.timestampMillis(); - - case "decimal": - return ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - - case "uuid": - return ValueReaders.uuids(); - - default: - throw new IllegalArgumentException("Unknown logical type: " + logicalType); - } + yield GenericReaders.timestampMillis(); + } + case "decimal" -> + ValueReaders.decimal( + ValueReaders.decimalBytesReader(primitive), + ((LogicalTypes.Decimal) logicalType).getScale()); + case "uuid" -> ValueReaders.uuids(); + default -> throw new IllegalArgumentException("Unknown logical type: " + logicalType); + }; } - switch (primitive.getType()) { - case NULL: - return ValueReaders.nulls(); - case BOOLEAN: - return ValueReaders.booleans(); - case INT: - return ValueReaders.ints(); - case LONG: - return ValueReaders.longs(); - case FLOAT: - return ValueReaders.floats(); - case DOUBLE: - return ValueReaders.doubles(); - case STRING: - // might want to use a binary-backed container like Utf8 - return ValueReaders.strings(); - case FIXED: - return ValueReaders.fixed(primitive.getFixedSize()); - case BYTES: - return ValueReaders.byteBuffers(); - default: - throw new IllegalArgumentException("Unsupported type: " + primitive); - } + return switch (primitive.getType()) { + case NULL -> ValueReaders.nulls(); + case BOOLEAN -> ValueReaders.booleans(); + case INT -> ValueReaders.ints(); + case LONG -> ValueReaders.longs(); + case FLOAT -> ValueReaders.floats(); + case DOUBLE -> ValueReaders.doubles(); + case STRING -> + // might want to use a binary-backed container like Utf8 + ValueReaders.strings(); + case FIXED -> ValueReaders.fixed(primitive.getFixedSize()); + case BYTES -> ValueReaders.byteBuffers(); + default -> throw new IllegalArgumentException("Unsupported type: " + primitive); + }; } } } diff --git a/core/src/main/java/org/apache/iceberg/data/avro/DataWriter.java b/core/src/main/java/org/apache/iceberg/data/avro/DataWriter.java index 397b38643b15..ffc8c10f92d1 100644 --- a/core/src/main/java/org/apache/iceberg/data/avro/DataWriter.java +++ b/core/src/main/java/org/apache/iceberg/data/avro/DataWriter.java @@ -111,59 +111,42 @@ public ValueWriter variant( public ValueWriter primitive(Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - switch (logicalType.getName()) { - case "date": - return GenericWriters.dates(); - - case "time-micros": - return GenericWriters.times(); - - case "timestamp-micros": + return switch (logicalType.getName()) { + case "date" -> GenericWriters.dates(); + case "time-micros" -> GenericWriters.times(); + case "timestamp-micros" -> { if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericWriters.timestamptz(); + yield GenericWriters.timestamptz(); } - return GenericWriters.timestamps(); - - case "timestamp-nanos": + yield GenericWriters.timestamps(); + } + case "timestamp-nanos" -> { if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericWriters.timestamptzNanos(); + yield GenericWriters.timestamptzNanos(); } - return GenericWriters.timestampNanos(); - - case "decimal": + yield GenericWriters.timestampNanos(); + } + case "decimal" -> { LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; - return ValueWriters.decimal(decimal.getPrecision(), decimal.getScale()); - - case "uuid": - return ValueWriters.uuids(); - - default: - throw new IllegalArgumentException("Unsupported logical type: " + logicalType); - } + yield ValueWriters.decimal(decimal.getPrecision(), decimal.getScale()); + } + case "uuid" -> ValueWriters.uuids(); + default -> throw new IllegalArgumentException("Unsupported logical type: " + logicalType); + }; } - switch (primitive.getType()) { - case NULL: - return ValueWriters.nulls(); - case BOOLEAN: - return ValueWriters.booleans(); - case INT: - return ValueWriters.ints(); - case LONG: - return ValueWriters.longs(); - case FLOAT: - return ValueWriters.floats(); - case DOUBLE: - return ValueWriters.doubles(); - case STRING: - return ValueWriters.strings(); - case FIXED: - return ValueWriters.fixed(primitive.getFixedSize()); - case BYTES: - return ValueWriters.byteBuffers(); - default: - throw new IllegalArgumentException("Unsupported type: " + primitive); - } + return switch (primitive.getType()) { + case NULL -> ValueWriters.nulls(); + case BOOLEAN -> ValueWriters.booleans(); + case INT -> ValueWriters.ints(); + case LONG -> ValueWriters.longs(); + case FLOAT -> ValueWriters.floats(); + case DOUBLE -> ValueWriters.doubles(); + case STRING -> ValueWriters.strings(); + case FIXED -> ValueWriters.fixed(primitive.getFixedSize()); + case BYTES -> ValueWriters.byteBuffers(); + default -> throw new IllegalArgumentException("Unsupported type: " + primitive); + }; } } } diff --git a/core/src/main/java/org/apache/iceberg/data/avro/PlannedDataReader.java b/core/src/main/java/org/apache/iceberg/data/avro/PlannedDataReader.java index 747907a2fb97..54e3cee4d4a9 100644 --- a/core/src/main/java/org/apache/iceberg/data/avro/PlannedDataReader.java +++ b/core/src/main/java/org/apache/iceberg/data/avro/PlannedDataReader.java @@ -135,73 +135,60 @@ public ValueReader variant( public ValueReader primitive(Type partner, Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - switch (logicalType.getName()) { - case "date": - return GenericReaders.dates(); - - case "time-micros": - return GenericReaders.times(); - - case "timestamp-micros": + return switch (logicalType.getName()) { + case "date" -> GenericReaders.dates(); + case "time-micros" -> GenericReaders.times(); + case "timestamp-micros" -> { if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptz(); + yield GenericReaders.timestamptz(); } - return GenericReaders.timestamps(); - - case "timestamp-nanos": + yield GenericReaders.timestamps(); + } + case "timestamp-nanos" -> { if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptzNanos(); + yield GenericReaders.timestamptzNanos(); } - return GenericReaders.timestampNanos(); - - case "timestamp-millis": + yield GenericReaders.timestampNanos(); + } + case "timestamp-millis" -> { if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptzMillis(); + yield GenericReaders.timestamptzMillis(); } - return GenericReaders.timestampMillis(); - - case "decimal": - return ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - - case "uuid": - return ValueReaders.uuids(); - - default: - throw new IllegalArgumentException("Unknown logical type: " + logicalType); - } + yield GenericReaders.timestampMillis(); + } + case "decimal" -> + ValueReaders.decimal( + ValueReaders.decimalBytesReader(primitive), + ((LogicalTypes.Decimal) logicalType).getScale()); + case "uuid" -> ValueReaders.uuids(); + default -> throw new IllegalArgumentException("Unknown logical type: " + logicalType); + }; } - switch (primitive.getType()) { - case NULL: - return ValueReaders.nulls(); - case BOOLEAN: - return ValueReaders.booleans(); - case INT: + return switch (primitive.getType()) { + case NULL -> ValueReaders.nulls(); + case BOOLEAN -> ValueReaders.booleans(); + case INT -> { if (partner != null && partner.typeId() == Type.TypeID.LONG) { - return ValueReaders.intsAsLongs(); + yield ValueReaders.intsAsLongs(); } - return ValueReaders.ints(); - case LONG: - return ValueReaders.longs(); - case FLOAT: + yield ValueReaders.ints(); + } + case LONG -> ValueReaders.longs(); + case FLOAT -> { if (partner != null && partner.typeId() == Type.TypeID.DOUBLE) { - return ValueReaders.floatsAsDoubles(); + yield ValueReaders.floatsAsDoubles(); } - return ValueReaders.floats(); - case DOUBLE: - return ValueReaders.doubles(); - case STRING: - // might want to use a binary-backed container like Utf8 - return ValueReaders.strings(); - case FIXED: - return ValueReaders.fixed(primitive.getFixedSize()); - case BYTES: - return ValueReaders.byteBuffers(); - default: - throw new IllegalArgumentException("Unsupported type: " + primitive); - } + yield ValueReaders.floats(); + } + case DOUBLE -> ValueReaders.doubles(); + case STRING -> + // might want to use a binary-backed container like Utf8 + ValueReaders.strings(); + case FIXED -> ValueReaders.fixed(primitive.getFixedSize()); + case BYTES -> ValueReaders.byteBuffers(); + default -> throw new IllegalArgumentException("Unsupported type: " + primitive); + }; } } } diff --git a/core/src/main/java/org/apache/iceberg/deletes/DeleteGranularity.java b/core/src/main/java/org/apache/iceberg/deletes/DeleteGranularity.java index c225192fa121..4ca4e5ae67c8 100644 --- a/core/src/main/java/org/apache/iceberg/deletes/DeleteGranularity.java +++ b/core/src/main/java/org/apache/iceberg/deletes/DeleteGranularity.java @@ -48,14 +48,10 @@ public enum DeleteGranularity { @Override public String toString() { - switch (this) { - case FILE: - return "file"; - case PARTITION: - return "partition"; - default: - throw new IllegalArgumentException("Unknown delete granularity: " + this); - } + return switch (this) { + case FILE -> "file"; + case PARTITION -> "partition"; + }; } public static DeleteGranularity fromString(String valueAsString) { diff --git a/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java b/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java index c3b6cbaa9bff..8bb006390fb9 100644 --- a/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java +++ b/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java @@ -81,32 +81,21 @@ public R row() { @Override @SuppressWarnings("unchecked") public T get(int colPos, Class javaClass) { - switch (colPos) { - case 0: - return (T) path; - case 1: - return (T) (Long) pos; - case 2: - return (T) row; - default: - throw new IllegalArgumentException("No column at position " + colPos); - } + return switch (colPos) { + case 0 -> (T) path; + case 1 -> (T) (Long) pos; + case 2 -> (T) row; + default -> throw new IllegalArgumentException("No column at position " + colPos); + }; } @Override public void set(int colPos, T value) { switch (colPos) { - case 0: - this.path = (CharSequence) value; - break; - case 1: - this.pos = (Long) value; - break; - case 2: - this.row = (R) value; - break; - default: - throw new IllegalArgumentException("No column at position " + colPos); + case 0 -> this.path = (CharSequence) value; + case 1 -> this.pos = (Long) value; + case 2 -> this.row = (R) value; + default -> throw new IllegalArgumentException("No column at position " + colPos); } } } diff --git a/core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java b/core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java index ba954577aa74..50b6c15e78b0 100644 --- a/core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java +++ b/core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java @@ -101,14 +101,11 @@ public DeleteWriteResult result() { public void close() throws IOException { if (result == null) { switch (granularity) { - case FILE: - this.result = writeFileDeletes(); - return; - case PARTITION: - this.result = writePartitionDeletes(); - return; - default: - throw new UnsupportedOperationException("Unsupported delete granularity: " + granularity); + case FILE -> this.result = writeFileDeletes(); + case PARTITION -> this.result = writePartitionDeletes(); + default -> + throw new UnsupportedOperationException( + "Unsupported delete granularity: " + granularity); } } } diff --git a/core/src/main/java/org/apache/iceberg/encryption/StandardKeyMetadata.java b/core/src/main/java/org/apache/iceberg/encryption/StandardKeyMetadata.java index 6ddea184d8c4..062328a080f7 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/StandardKeyMetadata.java +++ b/core/src/main/java/org/apache/iceberg/encryption/StandardKeyMetadata.java @@ -144,32 +144,23 @@ public NativeEncryptionKeyMetadata copyWithLength(long length) { @Override public void put(int i, Object v) { switch (i) { - case 0: - this.encryptionKey = (ByteBuffer) v; - return; - case 1: - this.aadPrefix = (ByteBuffer) v; - return; - case 2: - this.fileLength = (Long) v; - return; - default: + case 0 -> this.encryptionKey = (ByteBuffer) v; + case 1 -> this.aadPrefix = (ByteBuffer) v; + case 2 -> this.fileLength = (Long) v; + default -> { // ignore the object, it must be from a newer version of the format + } } } @Override public Object get(int i) { - switch (i) { - case 0: - return encryptionKey; - case 1: - return aadPrefix; - case 2: - return fileLength; - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + i); - } + return switch (i) { + case 0 -> encryptionKey; + case 1 -> aadPrefix; + case 2 -> fileLength; + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + i); + }; } @Override diff --git a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java index 9bb5b7d05f0b..16e0cdacd022 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java +++ b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java @@ -295,20 +295,18 @@ static Expression fromJson(JsonNode json, Schema schema) { } Expression.Operation op = fromType(type); - switch (op) { - case NOT: - return Expressions.not(fromJson(JsonUtil.get(CHILD, json), schema)); - case AND: - return Expressions.and( - fromJson(JsonUtil.get(LEFT, json), schema), - fromJson(JsonUtil.get(RIGHT, json), schema)); - case OR: - return Expressions.or( - fromJson(JsonUtil.get(LEFT, json), schema), - fromJson(JsonUtil.get(RIGHT, json), schema)); - } - - return predicateFromJson(op, json, schema); + return switch (op) { + case NOT -> Expressions.not(fromJson(JsonUtil.get(CHILD, json), schema)); + case AND -> + Expressions.and( + fromJson(JsonUtil.get(LEFT, json), schema), + fromJson(JsonUtil.get(RIGHT, json), schema)); + case OR -> + Expressions.or( + fromJson(JsonUtil.get(LEFT, json), schema), + fromJson(JsonUtil.get(RIGHT, json), schema)); + default -> predicateFromJson(op, json, schema); + }; } private static Expression.Operation fromType(String type) { @@ -328,34 +326,25 @@ private static UnboundPredicate predicateFromJson( convertValue = valueNode -> (T) ExpressionParser.asObject(valueNode); } - switch (op) { - case IS_NULL: - case NOT_NULL: - case IS_NAN: - case NOT_NAN: + return switch (op) { + case IS_NULL, NOT_NULL, IS_NAN, NOT_NAN -> { // unary predicates Preconditions.checkArgument( !node.has(VALUE), "Cannot parse %s predicate: has invalid value field", op); Preconditions.checkArgument( !node.has(VALUES), "Cannot parse %s predicate: has invalid values field", op); - return Expressions.predicate(op, term); - case LT: - case LT_EQ: - case GT: - case GT_EQ: - case EQ: - case NOT_EQ: - case STARTS_WITH: - case NOT_STARTS_WITH: + yield Expressions.predicate(op, term); + } + case LT, LT_EQ, GT, GT_EQ, EQ, NOT_EQ, STARTS_WITH, NOT_STARTS_WITH -> { // literal predicates Preconditions.checkArgument( node.has(VALUE), "Cannot parse %s predicate: missing value", op); Preconditions.checkArgument( !node.has(VALUES), "Cannot parse %s predicate: has invalid values field", op); T value = literal(JsonUtil.get(VALUE, node), convertValue); - return Expressions.predicate(op, term, ImmutableList.of(value)); - case IN: - case NOT_IN: + yield Expressions.predicate(op, term, ImmutableList.of(value)); + } + case IN, NOT_IN -> { // literal set predicates Preconditions.checkArgument( node.has(VALUES), "Cannot parse %s predicate: missing values", op); @@ -364,14 +353,14 @@ private static UnboundPredicate predicateFromJson( JsonNode valuesNode = JsonUtil.get(VALUES, node); Preconditions.checkArgument( valuesNode.isArray(), "Cannot parse literals from non-array: %s", valuesNode); - return Expressions.predicate( + yield Expressions.predicate( op, term, Iterables.transform( ((ArrayNode) valuesNode)::elements, valueNode -> literal(valueNode, convertValue))); - default: - throw new UnsupportedOperationException("Unsupported operation: " + op); - } + } + default -> throw new UnsupportedOperationException("Unsupported operation: " + op); + }; } private static T literal(JsonNode valueNode, Function toValue) { @@ -406,17 +395,16 @@ private static UnboundTerm term(JsonNode node) { return Expressions.ref(node.asText()); } else if (node.isObject()) { String type = JsonUtil.getString(TYPE, node); - switch (type) { - case REFERENCE: - return Expressions.ref(JsonUtil.getString(TERM, node)); - case TRANSFORM: + return switch (type) { + case REFERENCE -> Expressions.ref(JsonUtil.getString(TERM, node)); + case TRANSFORM -> { UnboundTerm child = term(JsonUtil.get(TERM, node)); String transform = JsonUtil.getString(TRANSFORM, node); - return (UnboundTerm) + yield (UnboundTerm) Expressions.transform(child.ref().name(), Transforms.fromString(transform)); - default: - throw new IllegalArgumentException("Cannot parse type as a reference: " + type); - } + } + default -> throw new IllegalArgumentException("Cannot parse type as a reference: " + type); + }; } throw new IllegalArgumentException( diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopMetricsContext.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopMetricsContext.java index 4e677c9b739c..de39fe106825 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopMetricsContext.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopMetricsContext.java @@ -66,24 +66,28 @@ public void initialize(Map properties) { @Override @SuppressWarnings("unchecked") public Counter counter(String name, Class type, Unit unit) { - switch (name) { - case READ_BYTES: + return switch (name) { + case READ_BYTES -> { ValidationException.check(type == Long.class, "'%s' requires Long type", READ_BYTES); - return (Counter) longCounter(statistics()::incrementBytesRead); - case READ_OPERATIONS: + yield (Counter) longCounter(statistics()::incrementBytesRead); + } + case READ_OPERATIONS -> { ValidationException.check( type == Integer.class, "'%s' requires Integer type", READ_OPERATIONS); - return (Counter) integerCounter(statistics()::incrementReadOps); - case WRITE_BYTES: + yield (Counter) integerCounter(statistics()::incrementReadOps); + } + case WRITE_BYTES -> { ValidationException.check(type == Long.class, "'%s' requires Long type", WRITE_BYTES); - return (Counter) longCounter(statistics()::incrementBytesWritten); - case WRITE_OPERATIONS: + yield (Counter) longCounter(statistics()::incrementBytesWritten); + } + case WRITE_OPERATIONS -> { ValidationException.check( type == Integer.class, "'%s' requires Integer type", WRITE_OPERATIONS); - return (Counter) integerCounter(statistics()::incrementWriteOps); - default: - throw new IllegalArgumentException(String.format("Unsupported counter: '%s'", name)); - } + yield (Counter) integerCounter(statistics()::incrementWriteOps); + } + default -> + throw new IllegalArgumentException(String.format("Unsupported counter: '%s'", name)); + }; } private Counter longCounter(Consumer consumer) { @@ -125,19 +129,17 @@ public void increment(Integer amount) { */ @Override public org.apache.iceberg.metrics.Counter counter(String name, Unit unit) { - switch (name) { - case READ_BYTES: - return counter(statistics()::incrementBytesRead, statistics()::getBytesRead); - case READ_OPERATIONS: - return counter((long x) -> statistics.incrementReadOps((int) x), statistics()::getReadOps); - case WRITE_BYTES: - return counter(statistics()::incrementBytesWritten, statistics()::getBytesWritten); - case WRITE_OPERATIONS: - return counter( - (long x) -> statistics.incrementWriteOps((int) x), statistics()::getWriteOps); - default: - throw new IllegalArgumentException(String.format("Unsupported counter: '%s'", name)); - } + return switch (name) { + case READ_BYTES -> counter(statistics()::incrementBytesRead, statistics()::getBytesRead); + case READ_OPERATIONS -> + counter((long x) -> statistics.incrementReadOps((int) x), statistics()::getReadOps); + case WRITE_BYTES -> + counter(statistics()::incrementBytesWritten, statistics()::getBytesWritten); + case WRITE_OPERATIONS -> + counter((long x) -> statistics.incrementWriteOps((int) x), statistics()::getWriteOps); + default -> + throw new IllegalArgumentException(String.format("Unsupported counter: '%s'", name)); + }; } private org.apache.iceberg.metrics.Counter counter(LongConsumer consumer, LongSupplier supplier) { diff --git a/core/src/main/java/org/apache/iceberg/io/ClusteredPositionDeleteWriter.java b/core/src/main/java/org/apache/iceberg/io/ClusteredPositionDeleteWriter.java index c9d911894c77..ba67a8a2e5c8 100644 --- a/core/src/main/java/org/apache/iceberg/io/ClusteredPositionDeleteWriter.java +++ b/core/src/main/java/org/apache/iceberg/io/ClusteredPositionDeleteWriter.java @@ -70,14 +70,10 @@ public ClusteredPositionDeleteWriter( @Override protected FileWriter, DeleteWriteResult> newWriter( PartitionSpec spec, StructLike partition) { - switch (granularity) { - case FILE: - return new FileScopedPositionDeleteWriter<>(() -> newRollingWriter(spec, partition)); - case PARTITION: - return newRollingWriter(spec, partition); - default: - throw new UnsupportedOperationException("Unsupported delete granularity: " + granularity); - } + return switch (granularity) { + case FILE -> new FileScopedPositionDeleteWriter<>(() -> newRollingWriter(spec, partition)); + case PARTITION -> newRollingWriter(spec, partition); + }; } private RollingPositionDeleteWriter newRollingWriter( diff --git a/core/src/main/java/org/apache/iceberg/metrics/TimerResultParser.java b/core/src/main/java/org/apache/iceberg/metrics/TimerResultParser.java index 52235d030744..d76432bd1f67 100644 --- a/core/src/main/java/org/apache/iceberg/metrics/TimerResultParser.java +++ b/core/src/main/java/org/apache/iceberg/metrics/TimerResultParser.java @@ -119,23 +119,17 @@ private static TimeUnit toTimeUnit(String timeUnit) { } private static ChronoUnit toChronoUnit(TimeUnit unit) { - switch (unit) { - case NANOSECONDS: - return ChronoUnit.NANOS; - case MICROSECONDS: - return ChronoUnit.MICROS; - case MILLISECONDS: - return ChronoUnit.MILLIS; - case SECONDS: - return ChronoUnit.SECONDS; - case MINUTES: - return ChronoUnit.MINUTES; - case HOURS: - return ChronoUnit.HOURS; - case DAYS: - return ChronoUnit.DAYS; - default: - throw new IllegalArgumentException("Cannot determine chrono unit from time unit: " + unit); - } + return switch (unit) { + case NANOSECONDS -> ChronoUnit.NANOS; + case MICROSECONDS -> ChronoUnit.MICROS; + case MILLISECONDS -> ChronoUnit.MILLIS; + case SECONDS -> ChronoUnit.SECONDS; + case MINUTES -> ChronoUnit.MINUTES; + case HOURS -> ChronoUnit.HOURS; + case DAYS -> ChronoUnit.DAYS; + default -> + throw new IllegalArgumentException( + "Cannot determine chrono unit from time unit: " + unit); + }; } } diff --git a/core/src/main/java/org/apache/iceberg/puffin/PuffinFormat.java b/core/src/main/java/org/apache/iceberg/puffin/PuffinFormat.java index 7a2ee61612a9..f461d7b6fd8a 100644 --- a/core/src/main/java/org/apache/iceberg/puffin/PuffinFormat.java +++ b/core/src/main/java/org/apache/iceberg/puffin/PuffinFormat.java @@ -104,17 +104,14 @@ static int readIntegerLittleEndian(byte[] data, int offset) { } static ByteBuffer compress(PuffinCompressionCodec codec, ByteBuffer input) { - switch (codec) { - case NONE: - return input.duplicate(); - case LZ4: - // TODO requires LZ4 frame compressor, e.g. - // https://github.com/airlift/aircompressor/pull/142 - break; - case ZSTD: - return compress(new ZstdCompressor(), input); - } - throw new UnsupportedOperationException("Unsupported codec: " + codec); + return switch (codec) { + case NONE -> input.duplicate(); + case LZ4 -> + // TODO requires LZ4 frame compressor, e.g. + // https://github.com/airlift/aircompressor/pull/142 + throw new UnsupportedOperationException("Unsupported codec: " + codec); + case ZSTD -> compress(new ZstdCompressor(), input); + }; } private static ByteBuffer compress(Compressor compressor, ByteBuffer input) { @@ -125,20 +122,14 @@ private static ByteBuffer compress(Compressor compressor, ByteBuffer input) { } static ByteBuffer decompress(PuffinCompressionCodec codec, ByteBuffer input) { - switch (codec) { - case NONE: - return input.duplicate(); - - case LZ4: - // TODO requires LZ4 frame decompressor, e.g. - // https://github.com/airlift/aircompressor/pull/142 - break; - - case ZSTD: - return decompressZstd(input); - } - - throw new UnsupportedOperationException("Unsupported codec: " + codec); + return switch (codec) { + case NONE -> input.duplicate(); + case LZ4 -> + // TODO requires LZ4 frame decompressor, e.g. + // https://github.com/airlift/aircompressor/pull/142 + throw new UnsupportedOperationException("Unsupported codec: " + codec); + case ZSTD -> decompressZstd(input); + }; } private static ByteBuffer decompressZstd(ByteBuffer input) { diff --git a/core/src/main/java/org/apache/iceberg/puffin/PuffinReader.java b/core/src/main/java/org/apache/iceberg/puffin/PuffinReader.java index e30b6e1ee6ef..1f9ac98fd9bc 100644 --- a/core/src/main/java/org/apache/iceberg/puffin/PuffinReader.java +++ b/core/src/main/java/org/apache/iceberg/puffin/PuffinReader.java @@ -71,13 +71,11 @@ public FileMetadata fileMetadata() throws IOException { PuffinCompressionCodec footerCompression = PuffinCompressionCodec.NONE; for (Flag flag : decodeFlags(footer, footerStructOffset)) { - switch (flag) { - case FOOTER_PAYLOAD_COMPRESSED: - footerCompression = PuffinFormat.FOOTER_COMPRESSION_CODEC; - break; - default: - throw new IllegalStateException("Unsupported flag: " + flag); - } + footerCompression = + switch (flag) { + case FOOTER_PAYLOAD_COMPRESSED -> PuffinFormat.FOOTER_COMPRESSION_CODEC; + default -> throw new IllegalStateException("Unsupported flag: " + flag); + }; } int footerPayloadSize = diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index 3a1e62260aae..b3e82b00ecfe 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -518,21 +518,18 @@ public static LoadTableResponse loadTable( if (table instanceof BaseTable) { TableMetadata loadedMetadata = ((BaseTable) table).operations().current(); - TableMetadata metadata; - switch (mode) { - case ALL: - metadata = loadedMetadata; - break; - case REFS: - metadata = - TableMetadata.buildFrom(loadedMetadata) - .withMetadataLocation(loadedMetadata.metadataFileLocation()) - .suppressHistoricalSnapshots() - .build(); - break; - default: - throw new IllegalArgumentException(String.format("Invalid snapshot mode: %s", mode)); - } + TableMetadata metadata = + switch (mode) { + case ALL -> loadedMetadata; + case REFS -> + TableMetadata.buildFrom(loadedMetadata) + .withMetadataLocation(loadedMetadata.metadataFileLocation()) + .suppressHistoricalSnapshots() + .build(); + default -> + throw new IllegalArgumentException( + String.format("Invalid snapshot mode: %s", mode)); + }; return LoadTableResponse.builder().withTableMetadata(metadata).build(); } else if (table instanceof BaseMetadataTable) { diff --git a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java index 334bfde8abfc..7a3fcf808d47 100644 --- a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java @@ -122,16 +122,12 @@ private static class CommitErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404: - throw new NoSuchTableException("%s", error.message()); - case 409: - throw new CommitFailedException("Commit failed: %s", error.message()); - case 500: - case 502: - case 503: - case 504: - throw new CommitStateUnknownException( - new ServiceFailureException("Service failed: %s: %s", error.code(), error.message())); + case 404 -> throw new NoSuchTableException("%s", error.message()); + case 409 -> throw new CommitFailedException("Commit failed: %s", error.message()); + case 500, 502, 503, 504 -> + throw new CommitStateUnknownException( + new ServiceFailureException( + "Service failed: %s: %s", error.code(), error.message())); } super.accept(error); @@ -145,7 +141,7 @@ private static class TableErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404: + case 404 -> { if (NoSuchNamespaceException.class.getSimpleName().equals(error.type())) { throw new NoSuchNamespaceException("%s", error.message()); } else if (NotFoundException.class.getSimpleName().equals(error.type())) { @@ -153,8 +149,8 @@ public void accept(ErrorResponse error) { } else { throw new NoSuchTableException("%s", error.message()); } - case 409: - throw new AlreadyExistsException("%s", error.message()); + } + case 409 -> throw new AlreadyExistsException("%s", error.message()); } super.accept(error); @@ -168,10 +164,8 @@ private static class CreateTableErrorHandler extends CommitErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404: - throw new NoSuchNamespaceException("%s", error.message()); - case 409: - throw new AlreadyExistsException("%s", error.message()); + case 404 -> throw new NoSuchNamespaceException("%s", error.message()); + case 409 -> throw new AlreadyExistsException("%s", error.message()); } super.accept(error); @@ -225,16 +219,12 @@ private static class ViewCommitErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404: - throw new NoSuchViewException("%s", error.message()); - case 409: - throw new CommitFailedException("Commit failed: %s", error.message()); - case 500: - case 502: - case 503: - case 504: - throw new CommitStateUnknownException( - new ServiceFailureException("Service failed: %s: %s", error.code(), error.message())); + case 404 -> throw new NoSuchViewException("%s", error.message()); + case 409 -> throw new CommitFailedException("Commit failed: %s", error.message()); + case 500, 502, 503, 504 -> + throw new CommitStateUnknownException( + new ServiceFailureException( + "Service failed: %s: %s", error.code(), error.message())); } super.accept(error); @@ -248,14 +238,14 @@ private static class ViewErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404: + case 404 -> { if (NoSuchNamespaceException.class.getSimpleName().equals(error.type())) { throw new NoSuchNamespaceException("%s", error.message()); } else { throw new NoSuchViewException("%s", error.message()); } - case 409: - throw new AlreadyExistsException("%s", error.message()); + } + case 409 -> throw new AlreadyExistsException("%s", error.message()); } super.accept(error); @@ -269,17 +259,15 @@ private static class NamespaceErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 400: + case 400 -> { if (NamespaceNotEmptyException.class.getSimpleName().equals(error.type())) { throw new NamespaceNotEmptyException("%s", error.message()); } throw new BadRequestException("Malformed request: %s", error.message()); - case 404: - throw new NoSuchNamespaceException("%s", error.message()); - case 409: - throw new AlreadyExistsException("%s", error.message()); - case 422: - throw createRESTException(error); + } + case 404 -> throw new NoSuchNamespaceException("%s", error.message()); + case 409 -> throw new AlreadyExistsException("%s", error.message()); + case 422 -> throw createRESTException(error); } super.accept(error); @@ -334,24 +322,21 @@ public ErrorResponse parseResponse(int code, String json) { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 400: + case 400 -> { if (IllegalArgumentException.class.getSimpleName().equals(error.type())) { throw new IllegalArgumentException(error.message()); } throw new BadRequestException("Malformed request: %s", error.message()); - case 401: - throw new NotAuthorizedException("Not authorized: %s", error.message()); - case 403: - throw new ForbiddenException("Forbidden: %s", error.message()); - case 405: - case 406: - break; - case 500: - throw new ServiceFailureException("Server error: %s: %s", error.type(), error.message()); - case 501: - throw new UnsupportedOperationException(error.message()); - case 503: - throw new ServiceUnavailableException("Service unavailable: %s", error.message()); + } + case 401 -> throw new NotAuthorizedException("Not authorized: %s", error.message()); + case 403 -> throw new ForbiddenException("Forbidden: %s", error.message()); + case 405, 406 -> {} + case 500 -> + throw new ServiceFailureException( + "Server error: %s: %s", error.type(), error.message()); + case 501 -> throw new UnsupportedOperationException(error.message()); + case 503 -> + throw new ServiceUnavailableException("Service unavailable: %s", error.message()); } throw createRESTException(error); @@ -375,16 +360,16 @@ public ErrorResponse parseResponse(int code, String json) { public void accept(ErrorResponse error) { if (error.type() != null) { switch (error.type()) { - case OAuth2Properties.INVALID_CLIENT_ERROR: - throw new NotAuthorizedException( - "Not authorized: %s: %s", error.type(), error.message()); - case OAuth2Properties.INVALID_REQUEST_ERROR: - case OAuth2Properties.INVALID_GRANT_ERROR: - case OAuth2Properties.UNAUTHORIZED_CLIENT_ERROR: - case OAuth2Properties.UNSUPPORTED_GRANT_TYPE_ERROR: - case OAuth2Properties.INVALID_SCOPE_ERROR: - throw new BadRequestException( - "Malformed request: %s: %s", error.type(), error.message()); + case OAuth2Properties.INVALID_CLIENT_ERROR -> + throw new NotAuthorizedException( + "Not authorized: %s: %s", error.type(), error.message()); + case OAuth2Properties.INVALID_REQUEST_ERROR, + OAuth2Properties.INVALID_GRANT_ERROR, + OAuth2Properties.UNAUTHORIZED_CLIENT_ERROR, + OAuth2Properties.UNSUPPORTED_GRANT_TYPE_ERROR, + OAuth2Properties.INVALID_SCOPE_ERROR -> + throw new BadRequestException( + "Malformed request: %s: %s", error.type(), error.message()); } } throw createRESTException(error); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java b/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java index be763d30fef1..f228c3ab733e 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java @@ -160,7 +160,7 @@ public void commit(TableMetadata base, TableMetadata metadata) { List requirements; List updates; switch (updateType) { - case CREATE: + case CREATE -> { Preconditions.checkState( base == null, "Invalid base metadata for create transaction, expected null: %s", base); updates = @@ -170,9 +170,8 @@ public void commit(TableMetadata base, TableMetadata metadata) { .build(); requirements = UpdateRequirements.forCreateTable(updates); errorHandler = ErrorHandlers.createTableErrorHandler(); - break; - - case REPLACE: + } + case REPLACE -> { Preconditions.checkState(base != null, "Invalid base metadata: null"); updates = ImmutableList.builder() @@ -182,18 +181,16 @@ public void commit(TableMetadata base, TableMetadata metadata) { // use the original replace base metadata because the transaction will refresh requirements = UpdateRequirements.forReplaceTable(replaceBase, updates); errorHandler = ErrorHandlers.tableCommitHandler(); - break; - - case SIMPLE: + } + case SIMPLE -> { Preconditions.checkState(base != null, "Invalid base metadata: null"); updates = metadata.changes(); requirements = UpdateRequirements.forUpdateTable(base, updates); errorHandler = ErrorHandlers.tableCommitHandler(); - break; - - default: - throw new UnsupportedOperationException( - String.format("Update type %s is not supported", updateType)); + } + default -> + throw new UnsupportedOperationException( + String.format("Update type %s is not supported", updateType)); } UpdateTableRequest request = new UpdateTableRequest(requirements, updates); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java b/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java index 9fa273ca169f..0256a0083798 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java @@ -212,18 +212,18 @@ private CloseableIterable planTableScan(PlanTableScanRequest planT this.scanFileIO = !response.credentials().isEmpty() ? scanFileIO(response.credentials()) : table().io(); - switch (planStatus) { - case COMPLETED: - return scanTasksIterable(response.planTasks(), response.fileScanTasks()); - case SUBMITTED: + return switch (planStatus) { + case COMPLETED -> scanTasksIterable(response.planTasks(), response.fileScanTasks()); + case SUBMITTED -> { Endpoint.check(supportedEndpoints, Endpoint.V1_FETCH_TABLE_SCAN_PLAN); - return fetchPlanningResult(); - case FAILED: - throw new IllegalStateException(failureMessage(planId, response.errorResponse())); - default: - throw new IllegalStateException( - String.format("Invalid planStatus: %s for planId: %s", planStatus, planId)); - } + yield fetchPlanningResult(); + } + case FAILED -> + throw new IllegalStateException(failureMessage(planId, response.errorResponse())); + default -> + throw new IllegalStateException( + String.format("Invalid planStatus: %s for planId: %s", planStatus, planId)); + }; } private FileIO scanFileIO(List storageCredentials) { @@ -282,24 +282,21 @@ private CloseableIterable fetchPlanningResult() { parserContext); switch (response.planStatus()) { - case COMPLETED: - result.set(response); - break; - case SUBMITTED: - throw new NotCompleteException(); - case FAILED: - throw new IllegalStateException(failureMessage(id, response.errorResponse())); - case CANCELLED: - throw new IllegalStateException( - String.format( - Locale.ROOT, "Remote scan planning cancelled for planId: %s", id)); - default: - throw new IllegalStateException( - String.format( - Locale.ROOT, - "Invalid planStatus: %s for planId: %s", - response.planStatus(), - id)); + case COMPLETED -> result.set(response); + case SUBMITTED -> throw new NotCompleteException(); + case FAILED -> + throw new IllegalStateException(failureMessage(id, response.errorResponse())); + case CANCELLED -> + throw new IllegalStateException( + String.format( + Locale.ROOT, "Remote scan planning cancelled for planId: %s", id)); + default -> + throw new IllegalStateException( + String.format( + Locale.ROOT, + "Invalid planStatus: %s for planId: %s", + response.planStatus(), + id)); } }); } catch (NotCompleteException e) { diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java b/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java index 4e48118561f7..f8de625d69bc 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java @@ -84,26 +84,15 @@ public static AuthManager loadAuthManager(String name, Map prope delegate = loadAuthManager(name, newProperties); } - String impl; - switch (authType.toLowerCase(Locale.ROOT)) { - case AuthProperties.AUTH_TYPE_NONE: - impl = AuthProperties.AUTH_MANAGER_IMPL_NONE; - break; - case AuthProperties.AUTH_TYPE_BASIC: - impl = AuthProperties.AUTH_MANAGER_IMPL_BASIC; - break; - case AuthProperties.AUTH_TYPE_SIGV4: - impl = AuthProperties.AUTH_MANAGER_IMPL_SIGV4; - break; - case AuthProperties.AUTH_TYPE_GOOGLE: - impl = AuthProperties.AUTH_MANAGER_IMPL_GOOGLE; - break; - case AuthProperties.AUTH_TYPE_OAUTH2: - impl = AuthProperties.AUTH_MANAGER_IMPL_OAUTH2; - break; - default: - impl = authType; - } + String impl = + switch (authType.toLowerCase(Locale.ROOT)) { + case AuthProperties.AUTH_TYPE_NONE -> AuthProperties.AUTH_MANAGER_IMPL_NONE; + case AuthProperties.AUTH_TYPE_BASIC -> AuthProperties.AUTH_MANAGER_IMPL_BASIC; + case AuthProperties.AUTH_TYPE_SIGV4 -> AuthProperties.AUTH_MANAGER_IMPL_SIGV4; + case AuthProperties.AUTH_TYPE_GOOGLE -> AuthProperties.AUTH_MANAGER_IMPL_GOOGLE; + case AuthProperties.AUTH_TYPE_OAUTH2 -> AuthProperties.AUTH_MANAGER_IMPL_OAUTH2; + default -> authType; + }; LOG.info("Loading AuthManager implementation: {}", impl); DynConstructors.Ctor ctor; diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java index a9504a7a4e99..511782fe5a8f 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java @@ -290,17 +290,17 @@ private static Map tokenExchangeRequest( private static Pair parseCredential(String credential) { Preconditions.checkNotNull(credential, "Invalid credential: null"); List parts = CREDENTIAL_SPLITTER.splitToList(credential); - switch (parts.size()) { - case 2: - // client ID and client secret - return Pair.of(parts.get(0), parts.get(1)); - case 1: - // client secret - return Pair.of(null, parts.get(0)); - default: - // this should never happen because the credential splitter is limited to 2 - throw new IllegalArgumentException("Invalid credential: " + credential); - } + return switch (parts.size()) { + case 2 -> + // client ID and client secret + Pair.of(parts.get(0), parts.get(1)); + case 1 -> + // client secret + Pair.of(null, parts.get(0)); + default -> + // this should never happen because the credential splitter is limited to 2 + throw new IllegalArgumentException("Invalid credential: " + credential); + }; } private static Map clientCredentialsRequest( diff --git a/core/src/main/java/org/apache/iceberg/schema/SchemaWithPartnerVisitor.java b/core/src/main/java/org/apache/iceberg/schema/SchemaWithPartnerVisitor.java index 694bfb2f6242..861658f1a872 100644 --- a/core/src/main/java/org/apache/iceberg/schema/SchemaWithPartnerVisitor.java +++ b/core/src/main/java/org/apache/iceberg/schema/SchemaWithPartnerVisitor.java @@ -47,8 +47,8 @@ public static T visit( public static T visit( Type type, P partner, SchemaWithPartnerVisitor visitor, PartnerAccessors

accessors) { - switch (type.typeId()) { - case STRUCT: + return switch (type.typeId()) { + case STRUCT -> { Types.StructType struct = type.asNestedType().asStructType(); List results = Lists.newArrayListWithExpectedSize(struct.fields().size()); for (Types.NestedField field : struct.fields()) { @@ -65,9 +65,9 @@ public static T visit( } results.add(visitor.field(field, fieldPartner, result)); } - return visitor.struct(struct, partner, results); - - case LIST: + yield visitor.struct(struct, partner, results); + } + case LIST -> { Types.ListType list = type.asNestedType().asListType(); T elementResult; @@ -80,9 +80,9 @@ public static T visit( visitor.afterListElement(elementField, partnerElement); } - return visitor.list(list, partner, elementResult); - - case MAP: + yield visitor.list(list, partner, elementResult); + } + case MAP -> { Types.MapType map = type.asNestedType().asMapType(); T keyResult; T valueResult; @@ -105,14 +105,11 @@ public static T visit( visitor.afterMapValue(valueField, valuePartner); } - return visitor.map(map, partner, keyResult, valueResult); - - case VARIANT: - return visitor.variant(type.asVariantType(), partner); - - default: - return visitor.primitive(type.asPrimitiveType(), partner); - } + yield visitor.map(map, partner, keyResult, valueResult); + } + case VARIANT -> visitor.variant(type.asVariantType(), partner); + default -> visitor.primitive(type.asPrimitiveType(), partner); + }; } public void beforeField(Types.NestedField field, P partnerField) {} diff --git a/core/src/main/java/org/apache/iceberg/variants/PrimitiveWrapper.java b/core/src/main/java/org/apache/iceberg/variants/PrimitiveWrapper.java index 6fd211156b00..3ae9a3dba04b 100644 --- a/core/src/main/java/org/apache/iceberg/variants/PrimitiveWrapper.java +++ b/core/src/main/java/org/apache/iceberg/variants/PrimitiveWrapper.java @@ -82,113 +82,108 @@ public T get() { @Override public int sizeInBytes() { - switch (type()) { - case NULL: - case BOOLEAN_TRUE: - case BOOLEAN_FALSE: - return 1; // 1 header only - case INT8: - return 2; // 1 header + 1 value - case INT16: - return 3; // 1 header + 2 value - case INT32: - case DATE: - case FLOAT: - return 5; // 1 header + 4 value - case INT64: - case DOUBLE: - case TIMESTAMPTZ: - case TIMESTAMPNTZ: - case TIMESTAMPTZ_NANOS: - case TIMESTAMPNTZ_NANOS: - case TIME: - return 9; // 1 header + 8 value - case DECIMAL4: - return 6; // 1 header + 1 scale + 4 unscaled value - case DECIMAL8: - return 10; // 1 header + 1 scale + 8 unscaled value - case DECIMAL16: - return 18; // 1 header + 1 scale + 16 unscaled value - case BINARY: - return 5 + ((ByteBuffer) value).remaining(); // 1 header + 4 length + value length - case STRING: + return switch (type()) { + case NULL, BOOLEAN_TRUE, BOOLEAN_FALSE -> 1; // 1 header only + case INT8 -> 2; // 1 header + 1 value + case INT16 -> 3; // 1 header + 2 value + case INT32, DATE, FLOAT -> 5; // 1 header + 4 value + case INT64, DOUBLE, TIMESTAMPTZ, TIMESTAMPNTZ, TIMESTAMPTZ_NANOS, TIMESTAMPNTZ_NANOS, TIME -> + 9; // 1 header + 8 value + case DECIMAL4 -> 6; // 1 header + 1 scale + 4 unscaled value + case DECIMAL8 -> 10; // 1 header + 1 scale + 8 unscaled value + case DECIMAL16 -> 18; // 1 header + 1 scale + 16 unscaled value + case BINARY -> 5 + ((ByteBuffer) value).remaining(); // 1 header + 4 length + value length + case STRING -> { if (null == buffer) { this.buffer = ByteBuffer.wrap(((String) value).getBytes(StandardCharsets.UTF_8)); } if (buffer.remaining() <= MAX_SHORT_STRING_LENGTH) { - return 1 + buffer.remaining(); // 1 header + value length + yield 1 + buffer.remaining(); // 1 header + value length } - return 5 + buffer.remaining(); // 1 header + 4 length + value length - case UUID: - return 1 + 16; // 1 header + 16 length - } - - throw new UnsupportedOperationException("Unsupported primitive type: " + type()); + yield 5 + buffer.remaining(); // 1 header + 4 length + value length + } + case UUID -> 1 + 16; // 1 header + 16 length + default -> throw new UnsupportedOperationException("Unsupported primitive type: " + type()); + }; } @Override public int writeTo(ByteBuffer outBuffer, int offset) { Preconditions.checkArgument( outBuffer.order() == ByteOrder.LITTLE_ENDIAN, "Invalid byte order: big endian"); - switch (type()) { - case NULL: + return switch (type()) { + case NULL -> { outBuffer.put(offset, NULL_HEADER); - return 1; - case BOOLEAN_TRUE: + yield 1; + } + case BOOLEAN_TRUE -> { outBuffer.put(offset, TRUE_HEADER); - return 1; - case BOOLEAN_FALSE: + yield 1; + } + case BOOLEAN_FALSE -> { outBuffer.put(offset, FALSE_HEADER); - return 1; - case INT8: + yield 1; + } + case INT8 -> { outBuffer.put(offset, INT8_HEADER); outBuffer.put(offset + 1, (Byte) value); - return 2; - case INT16: + yield 2; + } + case INT16 -> { outBuffer.put(offset, INT16_HEADER); outBuffer.putShort(offset + 1, (Short) value); - return 3; - case INT32: + yield 3; + } + case INT32 -> { outBuffer.put(offset, INT32_HEADER); outBuffer.putInt(offset + 1, (Integer) value); - return 5; - case INT64: + yield 5; + } + case INT64 -> { outBuffer.put(offset, INT64_HEADER); outBuffer.putLong(offset + 1, (Long) value); - return 9; - case FLOAT: + yield 9; + } + case FLOAT -> { outBuffer.put(offset, FLOAT_HEADER); outBuffer.putFloat(offset + 1, (Float) value); - return 5; - case DOUBLE: + yield 5; + } + case DOUBLE -> { outBuffer.put(offset, DOUBLE_HEADER); outBuffer.putDouble(offset + 1, (Double) value); - return 9; - case DATE: + yield 9; + } + case DATE -> { outBuffer.put(offset, DATE_HEADER); outBuffer.putInt(offset + 1, (Integer) value); - return 5; - case TIMESTAMPTZ: + yield 5; + } + case TIMESTAMPTZ -> { outBuffer.put(offset, TIMESTAMPTZ_HEADER); outBuffer.putLong(offset + 1, (Long) value); - return 9; - case TIMESTAMPNTZ: + yield 9; + } + case TIMESTAMPNTZ -> { outBuffer.put(offset, TIMESTAMPNTZ_HEADER); outBuffer.putLong(offset + 1, (Long) value); - return 9; - case DECIMAL4: + yield 9; + } + case DECIMAL4 -> { BigDecimal decimal4 = (BigDecimal) value; outBuffer.put(offset, DECIMAL4_HEADER); outBuffer.put(offset + 1, (byte) decimal4.scale()); outBuffer.putInt(offset + 2, decimal4.unscaledValue().intValueExact()); - return 6; - case DECIMAL8: + yield 6; + } + case DECIMAL8 -> { BigDecimal decimal8 = (BigDecimal) value; outBuffer.put(offset, DECIMAL8_HEADER); outBuffer.put(offset + 1, (byte) decimal8.scale()); outBuffer.putLong(offset + 2, decimal8.unscaledValue().longValueExact()); - return 10; - case DECIMAL16: + yield 10; + } + case DECIMAL16 -> { BigDecimal decimal16 = (BigDecimal) value; byte padding = (byte) (decimal16.signum() < 0 ? 0xFF : 0x00); byte[] bytes = decimal16.unscaledValue().toByteArray(); @@ -203,47 +198,53 @@ public int writeTo(ByteBuffer outBuffer, int offset) { outBuffer.put(offset + 2 + i, padding); } } - return 18; - case BINARY: + yield 18; + } + case BINARY -> { ByteBuffer binary = (ByteBuffer) value; outBuffer.put(offset, BINARY_HEADER); outBuffer.putInt(offset + 1, binary.remaining()); outBuffer.put(offset + 5, binary, binary.position(), binary.remaining()); - return 5 + binary.remaining(); - case STRING: + yield 5 + binary.remaining(); + } + case STRING -> { if (null == buffer) { this.buffer = ByteBuffer.wrap(((String) value).getBytes(StandardCharsets.UTF_8)); } if (buffer.remaining() <= MAX_SHORT_STRING_LENGTH) { outBuffer.put(offset, VariantUtil.shortStringHeader(buffer.remaining())); outBuffer.put(offset + 1, buffer, buffer.position(), buffer.remaining()); - return 1 + buffer.remaining(); + yield 1 + buffer.remaining(); } else { outBuffer.put(offset, STRING_HEADER); outBuffer.putInt(offset + 1, buffer.remaining()); outBuffer.put(offset + 5, buffer, buffer.position(), buffer.remaining()); - return 5 + buffer.remaining(); + yield 5 + buffer.remaining(); } - case TIME: + } + case TIME -> { outBuffer.put(offset, TIME_HEADER); outBuffer.putLong(offset + 1, (Long) value); - return 9; - case TIMESTAMPTZ_NANOS: + yield 9; + } + case TIMESTAMPTZ_NANOS -> { outBuffer.put(offset, TIMESTAMPTZ_NANOS_HEADER); outBuffer.putLong(offset + 1, (Long) value); - return 9; - case TIMESTAMPNTZ_NANOS: + yield 9; + } + case TIMESTAMPNTZ_NANOS -> { outBuffer.put(offset, TIMESTAMPNTZ_NANOS_HEADER); outBuffer.putLong(offset + 1, (Long) value); - return 9; - case UUID: + yield 9; + } + case UUID -> { outBuffer.put(offset, UUID_HEADER); ByteBuffer uuidBuffer = UUIDUtil.convertToByteBuffer((UUID) value); outBuffer.put(offset + 1, uuidBuffer, uuidBuffer.position(), uuidBuffer.remaining()); - return 17; - } - - throw new UnsupportedOperationException("Unsupported primitive type: " + type()); + yield 17; + } + default -> throw new UnsupportedOperationException("Unsupported primitive type: " + type()); + }; } @Override diff --git a/core/src/main/java/org/apache/iceberg/variants/VariantVisitor.java b/core/src/main/java/org/apache/iceberg/variants/VariantVisitor.java index 3bee300f20a9..c89e7d03849c 100644 --- a/core/src/main/java/org/apache/iceberg/variants/VariantVisitor.java +++ b/core/src/main/java/org/apache/iceberg/variants/VariantVisitor.java @@ -47,8 +47,8 @@ public static R visit(Variant variant, VariantVisitor visitor) { } public static R visit(VariantValue value, VariantVisitor visitor) { - switch (value.type()) { - case ARRAY: + return switch (value.type()) { + case ARRAY -> { VariantArray array = value.asArray(); List elementResults = Lists.newArrayList(); for (int index = 0; index < array.numElements(); index += 1) { @@ -60,9 +60,9 @@ public static R visit(VariantValue value, VariantVisitor visitor) { } } - return visitor.array(array, elementResults); - - case OBJECT: + yield visitor.array(array, elementResults); + } + case OBJECT -> { VariantObject object = value.asObject(); List fieldNames = Lists.newArrayList(); List fieldResults = Lists.newArrayList(); @@ -76,10 +76,9 @@ public static R visit(VariantValue value, VariantVisitor visitor) { } } - return visitor.object(object, fieldNames, fieldResults); - - default: - return visitor.primitive(value.asPrimitive()); - } + yield visitor.object(object, fieldNames, fieldResults); + } + default -> visitor.primitive(value.asPrimitive()); + }; } } diff --git a/core/src/main/java/org/apache/iceberg/view/ViewRepresentationParser.java b/core/src/main/java/org/apache/iceberg/view/ViewRepresentationParser.java index 79d50701ea51..d02dbc6ad01f 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewRepresentationParser.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewRepresentationParser.java @@ -34,14 +34,12 @@ static void toJson(ViewRepresentation representation, JsonGenerator generator) throws IOException { Preconditions.checkArgument(representation != null, "Invalid view representation: null"); switch (representation.type().toLowerCase(Locale.ENGLISH)) { - case ViewRepresentation.Type.SQL: - SQLViewRepresentationParser.toJson((SQLViewRepresentation) representation, generator); - break; - - default: - throw new UnsupportedOperationException( - String.format( - "Cannot serialize unsupported view representation: %s", representation.type())); + case ViewRepresentation.Type.SQL -> + SQLViewRepresentationParser.toJson((SQLViewRepresentation) representation, generator); + default -> + throw new UnsupportedOperationException( + String.format( + "Cannot serialize unsupported view representation: %s", representation.type())); } } @@ -58,12 +56,9 @@ static ViewRepresentation fromJson(JsonNode node) { Preconditions.checkArgument( node.isObject(), "Cannot parse view representation from non-object: %s", node); String type = JsonUtil.getString(TYPE, node).toLowerCase(Locale.ENGLISH); - switch (type) { - case ViewRepresentation.Type.SQL: - return SQLViewRepresentationParser.fromJson(node); - - default: - return ImmutableUnknownViewRepresentation.builder().type(type).build(); - } + return switch (type) { + case ViewRepresentation.Type.SQL -> SQLViewRepresentationParser.fromJson(node); + default -> ImmutableUnknownViewRepresentation.builder().type(type).build(); + }; } } From 0474d660bf5594a7f87e4e8745f56c83a985ad03 Mon Sep 17 00:00:00 2001 From: Yujiang Zhong <42907416+zhongyujiang@users.noreply.github.com> Date: Wed, 24 Jun 2026 19:12:24 +0800 Subject: [PATCH 33/73] Flink: Fix file offset mismatch in DataIterator.seek() when files are skipped (#16929) --- .../iceberg/flink/source/DataIterator.java | 4 +- .../flink/source/reader/ReaderUtil.java | 16 +- ...stArrayPoolDataIteratorBatcherRowData.java | 184 ++++++++++++++++++ 3 files changed, 200 insertions(+), 4 deletions(-) diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java index 3beda960cec8..2cb7667a489a 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java @@ -86,6 +86,7 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask); for (long i = 0L; i < startingFileOffset; ++i) { tasks.next(); + fileOffset += 1; } updateCurrentIterator(); @@ -103,9 +104,6 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask)); } } - - fileOffset = startingFileOffset; - recordOffset = startingRecordOffset; } @Override diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java index 3b094ba02298..5a6787001170 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java @@ -57,6 +57,21 @@ private ReaderUtil() {} public static FileScanTask createFileTask( List records, File file, FileFormat fileFormat, Schema schema) throws IOException { + return createFileTask( + records, + file, + fileFormat, + schema, + ResidualEvaluator.unpartitioned(Expressions.alwaysTrue())); + } + + public static FileScanTask createFileTask( + List records, + File file, + FileFormat fileFormat, + Schema schema, + ResidualEvaluator residuals) + throws IOException { DataWriter writer = new GenericFileWriterFactory.Builder() .dataSchema(schema) @@ -69,7 +84,6 @@ public static FileScanTask createFileTask( DataFile dataFile = writer.toDataFile(); - ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.alwaysTrue()); return new BaseFileScanTask( dataFile, null, diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java index 4f6d182bb3c1..67f049dbf0b5 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -32,13 +33,18 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.flink.FlinkConfigOptions; import org.apache.iceberg.flink.TestFixtures; import org.apache.iceberg.flink.TestHelpers; import org.apache.iceberg.flink.source.DataIterator; import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -354,4 +360,182 @@ public void testMultipleFilesWithSeekPosition() throws Exception { assertThat(recordBatchIterator).isExhausted(); } + + @Test + public void testDataIteratorWithResidualFilter() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "file-1"); + record0.setField("id", 0L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "file-2"); + record1.setField("id", 1L); + record1.setField("dt", "-"); + + List fileRecords0 = ImmutableList.of(record0); + List fileRecords1 = ImmutableList.of(record1); + + validateFileOffsetWithResidualFilter(fileRecords0, fileRecords1, Expressions.alwaysTrue()); + + validateFileOffsetWithResidualFilter( + fileRecords0, fileRecords1, Expressions.greaterThan("id", 0)); + } + + private void validateFileOffsetWithResidualFilter( + List fileRecords0, List fileRecords1, Expression residualFilter) + throws IOException { + ResidualEvaluator residualEvaluator = ResidualEvaluator.unpartitioned(residualFilter); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + fileRecords0, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + fileRecords1, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + while (dataIterator.hasNext()) { + assertThat(dataIterator.fileOffset()).isEqualTo(dataIterator.next().getLong(1)); + } + } + + @Test + public void testInitializationWithHeadFilesSkipped() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "a"); + record0.setField("id", 1L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "a"); + record1.setField("id", 10L); + record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + assertThat(dataIterator.fileOffset()) + .as("File offset should be 1 because file 0 should be skipped") + .isEqualTo(1); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, record1, dataIterator.next()); + } + + @Test + void testSeekResumesCorrectlyAfterHeadFilesSkipped() throws IOException { + // Setup: [F0(filtered), F1(1 record), F2(2 records)] + // F1 has only 1 record so that after reading F1, the checkpoint position (fileOffset=1, + // recordOffset=1) falls at a file boundary. Without the fix, fileOffset is incorrectly + // reported as 0 instead of 1 after seek(0,0) skips F0. This causes F2's first record to + // be reported at (fileOffset=1, recordOffset=1). On restore with seek(1, 1), the iterator + // skips only F0 + 1 record from F1, landing at the start of F2 instead of after F2 record0, + // resulting in F2 record0 being read again (duplicate). + GenericRecord filteredRecord = GenericRecord.create(TestFixtures.SCHEMA); + filteredRecord.setField("data", "a"); + filteredRecord.setField("id", 1L); + filteredRecord.setField("dt", "-"); + + GenericRecord f1Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f1Record0.setField("data", "b"); + f1Record0.setField("id", 10L); + f1Record0.setField("dt", "-"); + + GenericRecord f2Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record0.setField("data", "d"); + f2Record0.setField("id", 20L); + f2Record0.setField("dt", "-"); + + GenericRecord f2Record1 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record1.setField("data", "e"); + f2Record1.setField("id", 21L); + f2Record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(filteredRecord), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(f1Record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask2 = + ReaderUtil.createFileTask( + ImmutableList.of(f2Record0, f2Record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = + new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1, fileTask2)); + + // First read: consume file 1's record and file 2's first record + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + dataIterator.seek(0, 0); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, f1Record0, dataIterator.next()); + // F1 exhausted, updateCurrentIterator moves to F2 + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record0, dataIterator.next()); + + // Capture checkpoint position after reading F2 record0. + // With the fix: fileOffset=2, recordOffset=1 (correctly points into F2) + // Without fix: fileOffset=1, recordOffset=1 (incorrectly points to "F1 record 1") + int checkpointFileOffset = dataIterator.fileOffset(); + long checkpointRecordOffset = dataIterator.recordOffset(); + + // Simulate restore: seek to the checkpoint position + DataIterator restoredIterator = ReaderUtil.createDataIterator(combinedTask); + restoredIterator.seek(checkpointFileOffset, checkpointRecordOffset); + + // After restore, should read only F2 record1 (the unread remainder) + assertThat(restoredIterator.hasNext()).isTrue(); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record1, restoredIterator.next()); + assertThat(restoredIterator.hasNext()).isFalse(); + } } From 729aead0f6b92f7432abfb5af1e2ad51de3d829a Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Wed, 24 Jun 2026 18:49:25 +0700 Subject: [PATCH 34/73] ORC: Remove unused conf field from OrcFileAppender (#16345) --- .../apache/iceberg/orc/OrcFileAppender.java | 4 - .../iceberg/orc/TestTableProperties.java | 96 +++++++------------ 2 files changed, 34 insertions(+), 66 deletions(-) diff --git a/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java b/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java index 48b0e837f1a9..aef43c053142 100644 --- a/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java +++ b/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java @@ -54,9 +54,6 @@ class OrcFileAppender implements FileAppender { private final OrcRowWriter valueWriter; private boolean isClosed = false; - @SuppressWarnings("unused") // Currently used in tests TODO remove this redundant field - private final Configuration conf; - private final MetricsConfig metricsConfig; OrcFileAppender( @@ -67,7 +64,6 @@ class OrcFileAppender implements FileAppender { Map metadata, int batchSize, MetricsConfig metricsConfig) { - this.conf = conf; this.file = file; this.batchSize = batchSize; this.metricsConfig = metricsConfig; diff --git a/orc/src/test/java/org/apache/iceberg/orc/TestTableProperties.java b/orc/src/test/java/org/apache/iceberg/orc/TestTableProperties.java index ce3985597ed0..f045dafeafee 100644 --- a/orc/src/test/java/org/apache/iceberg/orc/TestTableProperties.java +++ b/orc/src/test/java/org/apache/iceberg/orc/TestTableProperties.java @@ -21,15 +21,15 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; -import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.common.DynFields; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.orc.GenericOrcWriter; import org.apache.iceberg.deletes.EqualityDeleteWriter; @@ -38,8 +38,8 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Types; import org.apache.orc.CompressionKind; -import org.apache.orc.OrcConf; -import org.apache.orc.OrcFile.CompressionStrategy; +import org.apache.orc.OrcFile; +import org.apache.orc.Reader; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -54,23 +54,15 @@ public class TestTableProperties { @TempDir private File testFile; @Test - public void testOrcTableProperties() throws Exception { - Random random = new Random(); - int numOfCodecs = CompressionKind.values().length; - int numOfStrategies = CompressionStrategy.values().length; - - long stripeSizeBytes = 32L * 1024 * 1024; - long blockSizeBytes = 128L * 1024 * 1024; - String codecAsString = CompressionKind.values()[random.nextInt(numOfCodecs)].name(); - String strategyAsString = CompressionStrategy.values()[random.nextInt(numOfStrategies)].name(); + public void testOrcTablePropertiesForDataFile() throws Exception { + String codecAsString = CompressionKind.SNAPPY.name(); ImmutableMap properties = ImmutableMap.of( - TableProperties.ORC_STRIPE_SIZE_BYTES, String.valueOf(stripeSizeBytes), - TableProperties.ORC_BLOCK_SIZE_BYTES, String.valueOf(blockSizeBytes), - TableProperties.ORC_COMPRESSION, codecAsString, - TableProperties.ORC_COMPRESSION_STRATEGY, strategyAsString, - TableProperties.DEFAULT_FILE_FORMAT, FileFormat.ORC.name()); + TableProperties.ORC_COMPRESSION, + codecAsString, + TableProperties.DEFAULT_FILE_FORMAT, + FileFormat.ORC.name()); String warehouse = folder.getAbsolutePath(); String tablePath = warehouse.concat("/test"); @@ -81,42 +73,30 @@ public void testOrcTableProperties() throws Exception { assertThat(testFile.delete()).isTrue(); - FileAppender writer = + try (FileAppender writer = ORC.write(Files.localOutput(testFile)) .forTable(table) .createWriterFunc(GenericOrcWriter::buildWriter) - .build(); - - DynFields.BoundField confField = - DynFields.builder().hiddenImpl(writer.getClass(), "conf").build(writer); - - Configuration configuration = confField.get(); - assertThat(OrcConf.BLOCK_SIZE.getLong(configuration)).isEqualTo(blockSizeBytes); - assertThat(OrcConf.STRIPE_SIZE.getLong(configuration)).isEqualTo(stripeSizeBytes); - assertThat(OrcConf.COMPRESS.getString(configuration)).isEqualTo(codecAsString); - assertThat(OrcConf.COMPRESSION_STRATEGY.getString(configuration)).isEqualTo(strategyAsString); - assertThat(configuration.get(TableProperties.DEFAULT_FILE_FORMAT)) - .isEqualTo(FileFormat.ORC.name()); + .build()) { + writer.add(GenericRecord.create(SCHEMA).copy(ImmutableMap.of("id", 1, "data", "a"))); + } + + Reader reader = + OrcFile.createReader( + new Path(testFile.toURI()), OrcFile.readerOptions(new Configuration())); + assertThat(reader.getCompressionKind()).isEqualTo(CompressionKind.SNAPPY); } @Test - public void testOrcTableDeleteProperties() throws Exception { - Random random = new Random(); - int numOfCodecs = CompressionKind.values().length; - int numOfStrategies = CompressionStrategy.values().length; - - long stripeSizeBytes = 32L * 1024 * 1024; - long blockSizeBytes = 128L * 1024 * 1024; - String codecAsString = CompressionKind.values()[random.nextInt(numOfCodecs)].name(); - String strategyAsString = CompressionStrategy.values()[random.nextInt(numOfStrategies)].name(); + public void testOrcTablePropertiesForDeleteFile() throws Exception { + String codecAsString = CompressionKind.SNAPPY.name(); ImmutableMap properties = ImmutableMap.of( - TableProperties.DELETE_ORC_STRIPE_SIZE_BYTES, String.valueOf(stripeSizeBytes), - TableProperties.DELETE_ORC_BLOCK_SIZE_BYTES, String.valueOf(blockSizeBytes), - TableProperties.DELETE_ORC_COMPRESSION, codecAsString, - TableProperties.DELETE_ORC_COMPRESSION_STRATEGY, strategyAsString, - TableProperties.DEFAULT_FILE_FORMAT, FileFormat.ORC.name()); + TableProperties.DELETE_ORC_COMPRESSION, + codecAsString, + TableProperties.DEFAULT_FILE_FORMAT, + FileFormat.ORC.name()); String warehouse = folder.getAbsolutePath(); String tablePath = warehouse.concat("/test"); @@ -127,26 +107,18 @@ public void testOrcTableDeleteProperties() throws Exception { assertThat(testFile.delete()).isTrue(); - EqualityDeleteWriter deleteWriter = + try (EqualityDeleteWriter deleteWriter = ORC.writeDeletes(Files.localOutput(testFile)) .forTable(table) .equalityFieldIds(1) .createWriterFunc(GenericOrcWriter::buildWriter) - .buildEqualityWriter(); - - DynFields.BoundField> writer = - DynFields.builder().hiddenImpl(deleteWriter.getClass(), "appender").build(deleteWriter); - - OrcFileAppender orcFileAppender = writer.get(); - DynFields.BoundField confField = - DynFields.builder().hiddenImpl(orcFileAppender.getClass(), "conf").build(orcFileAppender); - - Configuration configuration = confField.get(); - assertThat(OrcConf.BLOCK_SIZE.getLong(configuration)).isEqualTo(blockSizeBytes); - assertThat(OrcConf.STRIPE_SIZE.getLong(configuration)).isEqualTo(stripeSizeBytes); - assertThat(OrcConf.COMPRESS.getString(configuration)).isEqualTo(codecAsString); - assertThat(OrcConf.COMPRESSION_STRATEGY.getString(configuration)).isEqualTo(strategyAsString); - assertThat(configuration.get(TableProperties.DEFAULT_FILE_FORMAT)) - .isEqualTo(FileFormat.ORC.name()); + .buildEqualityWriter()) { + deleteWriter.write(GenericRecord.create(SCHEMA).copy(ImmutableMap.of("id", 1, "data", "a"))); + } + + Reader reader = + OrcFile.createReader( + new Path(testFile.toURI()), OrcFile.readerOptions(new Configuration())); + assertThat(reader.getCompressionKind()).isEqualTo(CompressionKind.SNAPPY); } } From d5c427b93b5713a7650c793cc393d3d4c932f10f Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Wed, 24 Jun 2026 17:21:01 +0200 Subject: [PATCH 35/73] Flink: Backport: Add equality delete conversion planner (#16889) (#16944) * Flink: Backport: Add equality delete conversion planner (#16889) * fixup! Convert scan-task wrapper in Flink 1.20 to plain Java class --- .../operator/EqualityConvertPlanner.java | 762 +++++++++++ .../operator/EqualityDeleteFileScanTask.java | 24 +- .../operator/FlinkAddedRowsScanTask.java | 33 +- .../operator/TestEqualityConvertPlanner.java | 1115 +++++++++++++++++ .../operator/EqualityConvertPlanner.java | 762 +++++++++++ .../operator/TestEqualityConvertPlanner.java | 1115 +++++++++++++++++ 6 files changed, 3804 insertions(+), 7 deletions(-) create mode 100644 flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java create mode 100644 flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java create mode 100644 flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java create mode 100644 flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java new file mode 100644 index 000000000000..c3c1785290cf --- /dev/null +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -0,0 +1,762 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.OutputTag; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotChanges; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Planner for the equality delete conversion pipeline. For each trigger, it picks the oldest + * staging snapshot that hasn't been converted yet and emits {@link ReadCommand}s describing the + * files its downstream readers and workers must process. + * + *

Each trigger runs two steps in order: + * + *

    + *
  1. {@link #ensureIndexCurrent}: updates {@link #lastStagingSnapshotId} from main's history, + * bootstraps the worker index from main on first run, and reindexes when external commits + * (e.g. compaction) have advanced main past the currently-indexed snapshot. + *
  2. {@link #processStagingSnapshot}: resolve the chosen staging snapshot's eq deletes against + * the (now-current) index, pass through any DV files, and index the snapshot's new data files + * for the next cycle. + *
+ * + * Watermarks separate phases that gate the worker's keyed state. The contract is documented on + * {@link #advancePhase()}. + * + *

An {@link EqualityConvertPlan} with the current cycle's metadata is emitted via the {@link + * #METADATA_STREAM} side output after the read commands. + * + *

Assumes a single equality-field set supplied via the builder; staging eq-deletes with a + * different {@code equalityFieldIds} fail fast in {@link #retrieveStagingFiles}. Concurrent writes + * on the target branch are handled by {@link #ensureIndexCurrent} reindexing from the new main + * snapshot; commit-time conflicts are caught by {@code RowDelta.validateFromSnapshot}. + */ +@Internal +public class EqualityConvertPlanner extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertPlanner.class); + + public static final OutputTag METADATA_STREAM = + new OutputTag<>("metadata-stream") {}; + + public static final OutputTag CLEAR_BROADCAST_STREAM = + new OutputTag<>("clear-broadcast-stream") {}; + + private static final String PROCESSED_EQ_DELETE_FILE_NUM_METRIC = "processedEqDeleteFileNum"; + private static final String PROCESSED_STAGING_SNAPSHOT_NUM_METRIC = "processedStagingSnapshotNum"; + private static final String SKIPPED_NO_OP_CYCLES_METRIC = "skippedNoOpCycles"; + private static final String REINDEX_COUNT_METRIC = "reindexCount"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String stagingBranch; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + // Equality-field-id set the worker keys on. Supplied via the builder; every staging + // eq-delete's equalityFieldIds() must match exactly. + private final Set eqFieldIds; + + // Main snapshot id the worker's index reflects. + private transient ListState indexSnapshotState; + // Main sequence number the worker's index reflects. + private transient ListState indexedSequenceNumberState; + // Equality field IDs the index was built with, allows to detect reconfiguration. + private transient ListState eqFieldIdsState; + + private transient Table table; + + private transient Long lastMainSnapshotId; + private transient Long lastStagingSnapshotId; + private transient Long indexSnapshotId; + private transient Long indexedSequenceNumber; + + private transient long nextPhaseTs; + + private transient Counter processedEqDeleteFileNumCounter; + private transient Counter processedStagingSnapshotNumCounter; + private transient Counter skippedNoOpCyclesCounter; + private transient Counter reindexCounter; + private transient Counter errorCounter; + + public EqualityConvertPlanner( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch, + Set eqFieldIds) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.stagingBranch = stagingBranch; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + Preconditions.checkArgument( + eqFieldIds != null && !eqFieldIds.isEmpty(), "eqFieldIds must not be null or empty"); + this.eqFieldIds = ImmutableSet.copyOf(eqFieldIds); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.processedEqDeleteFileNumCounter = + taskMetricGroup.counter(PROCESSED_EQ_DELETE_FILE_NUM_METRIC); + this.processedStagingSnapshotNumCounter = + taskMetricGroup.counter(PROCESSED_STAGING_SNAPSHOT_NUM_METRIC); + this.skippedNoOpCyclesCounter = taskMetricGroup.counter(SKIPPED_NO_OP_CYCLES_METRIC); + this.reindexCounter = taskMetricGroup.counter(REINDEX_COUNT_METRIC); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + indexSnapshotState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexSnapshotId", Types.LONG)); + + indexSnapshotId = null; + for (Long stateValue : indexSnapshotState.get()) { + Preconditions.checkState( + indexSnapshotId == null, "indexSnapshotId state should hold at most one value"); + indexSnapshotId = stateValue; + } + + indexedSequenceNumberState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexedSequenceNumber", Types.LONG)); + + indexedSequenceNumber = null; + for (Long stateValue : indexedSequenceNumberState.get()) { + Preconditions.checkState( + indexedSequenceNumber == null, + "indexedSequenceNumber state should hold at most one value"); + indexedSequenceNumber = stateValue; + } + + eqFieldIdsState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("eqFieldIds", Types.INT)); + Set restoredEqFieldIds = Sets.newHashSet(eqFieldIdsState.get()); + Preconditions.checkState( + restoredEqFieldIds.isEmpty() || restoredEqFieldIds.equals(eqFieldIds), + "Equality field IDs changed across restart: restored=%s, configured=%s. " + + "Reconfiguring equality-field columns is not supported; " + + "restart from a clean state (no savepoint).", + restoredEqFieldIds, + eqFieldIds); + } + + @Override + public void snapshotState(StateSnapshotContext context) throws Exception { + super.snapshotState(context); + indexSnapshotState.clear(); + if (indexSnapshotId != null) { + indexSnapshotState.add(indexSnapshotId); + } + + indexedSequenceNumberState.clear(); + if (indexedSequenceNumber != null) { + indexedSequenceNumberState.add(indexedSequenceNumber); + } + + eqFieldIdsState.clear(); + for (int id : eqFieldIds) { + eqFieldIdsState.add(id); + } + } + + @Override + public void processElement(StreamRecord element) throws Exception { + long triggerTs = element.getTimestamp(); + nextPhaseTs = Math.max(triggerTs, nextPhaseTs + 1); + + Long currentMainSnapshotId = lastMainSnapshotId; + try { + table.refresh(); + Snapshot mainSnapshot = table.snapshot(targetBranch); + currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + ensureIndexCurrent(mainSnapshot); + + Snapshot nextToProcess = + nextUnprocessedStagingSnapshot(table.snapshot(stagingBranch), mainSnapshot); + + if (nextToProcess == null) { + LOG.info("Nothing new to convert on staging branch '{}'.", stagingBranch); + emitNoOpResult(triggerTs, currentMainSnapshotId); + return; + } + + processStagingSnapshot(nextToProcess, triggerTs, currentMainSnapshotId); + } catch (Exception e) { + LOG.error("Error processing equality deletes for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + emitDrainResult(triggerTs, currentMainSnapshotId); + } + } + + /** + * Brings the worker's index up to date with the current state of the target branch: + * + *

    + *
  • Updates {@link #lastStagingSnapshotId} from the most recent committer marker on main. + *
  • Bootstraps the index from main on the first trigger with a non-null main snapshot. + *
  • Reindexes from main when external commits (e.g. compaction or direct writes) have + * advanced main past the currently-indexed snapshot. + *
+ * + *

No-op when main hasn't moved since the last trigger. Otherwise the history walk is bounded + * to commits added since {@link #lastMainSnapshotId}. + */ + private void ensureIndexCurrent(Snapshot mainSnapshot) { + Long currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + if (Objects.equals(lastMainSnapshotId, currentMainSnapshotId)) { + return; + } + + LastCommittedWork info = discoverLastCommittedWork(mainSnapshot); + updateLastStagingSnapshotId(info); + + boolean bootstrap = mainSnapshot != null && indexSnapshotId == null; + boolean reindex = indexSnapshotId != null && info.externalCommitCount() > 0; + if (bootstrap || reindex) { + LOG.info( + "{} worker index from main snapshot {} for field IDs {}.", + bootstrap ? "Bootstrapping" : "Reindexing", + currentMainSnapshotId, + eqFieldIds); + if (reindex) { + // Evict keyed entries the reindex will not re-add (e.g. data file removed by CoW). + output.collect( + CLEAR_BROADCAST_STREAM, + new StreamRecord<>( + IndexCommand.clearBeforeReindex( + currentMainSnapshotId, mainSnapshot.sequenceNumber()))); + reindexCounter.inc(); + } + + indexSnapshotId = currentMainSnapshotId; + indexedSequenceNumber = mainSnapshot.sequenceNumber(); + emitMainDataReadCommands(mainSnapshot); + } + + lastMainSnapshotId = currentMainSnapshotId; + } + + private void updateLastStagingSnapshotId(LastCommittedWork info) { + if (info.lastCommittedStaging() != null) { + lastStagingSnapshotId = info.lastCommittedStaging(); + return; + } + + Preconditions.checkState( + lastMainSnapshotId == null || lastStagingSnapshotId == null || info.reachedLastInspected(), + "No COMMITTED_STAGING_SNAPSHOT marker reachable on target branch '%s' for table %s, " + + "but a prior marker was seen (lastStagingSnapshotId=%s). Target may have been " + + "rewritten (rollback, replace_main, or snapshot expiration). " + + "Manual intervention required.", + targetBranch, + tableName, + lastStagingSnapshotId); + } + + /** + * Walks main back from head looking for the most recent snapshot tagged with {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}. Returns the staging snapshot id + * recorded there (or {@code null} if not reached) and the count of intervening external commits. + * + *

The walk stops at whichever comes first: + * + *

    + *
  • The first snapshot carrying the committer marker. + *
  • {@link #lastMainSnapshotId} — anything older was inspected on a previous trigger. + *
+ */ + private LastCommittedWork discoverLastCommittedWork(Snapshot mainSnapshot) { + Long lastCommittedStaging = null; + int externalCount = 0; + boolean reachedLastInspected = false; + Snapshot current = mainSnapshot; + while (current != null) { + if (lastMainSnapshotId != null && current.snapshotId() == lastMainSnapshotId) { + reachedLastInspected = true; + break; + } + + String prop = + current.summary().get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY); + if (prop != null) { + lastCommittedStaging = Long.parseLong(prop); + break; + } + + externalCount++; + current = parentOf(current); + } + + return new LastCommittedWork(lastCommittedStaging, externalCount, reachedLastInspected); + } + + private record LastCommittedWork( + Long lastCommittedStaging, int externalCommitCount, boolean reachedLastInspected) {} + + /** + * Walks staging history from head back to the stop point and returns the oldest unprocessed + * snapshot to convert this cycle, or {@code null} if there's nothing new. + * + *

The stop point is: + * + *

    + *
  • the last-processed staging snapshot, if known; + *
  • otherwise, on cold start with {@code stagingBranch != targetBranch}, the common ancestor + * with target; + *
  • otherwise, on cold start with {@code stagingBranch == targetBranch}, the root of history + * ({@code null}). {@link #shouldSkip} filters already-converted snapshots in O(1) via the + * {@code COMMITTED_STAGING_SNAPSHOT_PROPERTY} marker, and pure-insert snapshots via the + * same-branch eq-delete predicate. + *
+ * + *

When {@code stagingBranch == targetBranch}, the writer commits eq-deletes and data files + * directly to target; the converter performs in-place eq-delete-to-DV compaction. On cold start + * we walk the full history so eq-deletes committed before the converter started are still picked + * up. + */ + private Snapshot nextUnprocessedStagingSnapshot(Snapshot stagingHead, Snapshot mainSnapshot) { + if (stagingHead == null) { + return null; + } + + Long stopAt; + if (lastStagingSnapshotId != null) { + stopAt = lastStagingSnapshotId; + } else if (stagingOnTargetBranch) { + stopAt = null; + } else { + stopAt = findCommonAncestor(stagingHead, mainSnapshot); + } + + Snapshot current = stagingHead; + Snapshot oldestUnprocessed = null; + while (current != null) { + if (stopAt != null && current.snapshotId() == stopAt) { + break; + } + + if (!shouldSkip(current)) { + oldestUnprocessed = current; + } + + current = parentOf(current); + } + + return oldestUnprocessed; + } + + /** + * Resolves the eq deletes in {@code stagingSnapshot} against the current index and emits the + * cycle's metadata. Phase ordering (separated by watermarks): + * + *

    + *
  1. Eq delete read commands. Eq deletes resolve in the worker. + *
  2. Staging data files. Indexed for the NEXT cycle's eq-delete resolution. + *
+ * + * Cold-start bootstrap of the index from main data is handled separately in {@link + * #processElement}, which runs once before the first cycle. + */ + private void processStagingSnapshot( + Snapshot stagingSnapshot, long triggerTs, Long currentMainSnapshotId) { + + StagingInputs inputs = retrieveStagingFiles(stagingSnapshot); + Preconditions.checkState( + !inputs.isEmpty(), + "Staging snapshot %s has no convertible inputs; shouldSkip should have filtered it.", + stagingSnapshot.snapshotId()); + + emitDeletePhase(inputs.eqDeleteFiles()); + emitSnapshotDataPhase(inputs.newDataFiles()); + + LOG.info( + "Emitted read commands for {} new data files from staging branch '{}'.", + inputs.newDataFiles().size(), + stagingBranch); + + processedStagingSnapshotNumCounter.inc(); + + output.collect( + METADATA_STREAM, + new StreamRecord<>( + new EqualityConvertPlan( + inputs.newDataFiles(), + inputs.stagingDVFiles(), + stagingSnapshot.snapshotId(), + currentMainSnapshotId, + triggerTs, + nextPhaseTs))); + + advancePhase(); + } + + /** + * Classifies the files added by {@code stagingSnapshot} into data files, eq delete files, and DV + * files. Throws if the snapshot: + * + *
    + *
  • Removes data files (rewrites on the staging branch aren't supported). + *
  • Contains V2 positional delete files (the converter expects a V3 staging branch written by + * Flink, which produces only deletion vectors for deletes). + *
  • Contains an eq-delete file whose {@code equalityFieldIds()} doesn't match the + * builder-configured set (silent wrong-key serialization otherwise). + *
+ */ + private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { + SnapshotChanges changes = SnapshotChanges.builderFor(table).snapshot(stagingSnapshot).build(); + + // Rewrites on the staging branch would require rewriting the corresponding DVs against new + // data files on target. Not implemented; fail fast instead of silently dropping work. + if (changes.removedDataFiles().iterator().hasNext()) { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' removes data files; " + + "equality delete conversion does not support rewrites on the staging branch. " + + "Run compaction on the target branch instead.", + stagingSnapshot.snapshotId(), stagingBranch)); + } + + List newDataFiles = Lists.newArrayList(); + List stagingDVFiles = Lists.newArrayList(); + List eqDeleteFiles = Lists.newArrayList(); + + for (DataFile dataFile : changes.addedDataFiles()) { + newDataFiles.add(dataFile); + } + + for (DeleteFile deleteFile : changes.addedDeleteFiles()) { + if (deleteFile.content() == FileContent.EQUALITY_DELETES) { + Set deleteFieldIds = Sets.newHashSet(deleteFile.equalityFieldIds()); + Preconditions.checkState( + deleteFieldIds.equals(eqFieldIds), + "Staging snapshot %s on branch '%s' contains an equality delete file %s with " + + "equalityFieldIds=%s, which does not match the configured eqFieldIds=%s. " + + "The writer must use the same equality field IDs as the converter.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + deleteFieldIds, + eqFieldIds); + eqDeleteFiles.add(deleteFile); + } else if (ContentFileUtil.isDV(deleteFile)) { + stagingDVFiles.add(deleteFile); + } else { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' contains a V2 positional delete file (%s); " + + "equality delete conversion expects a V3 staging branch written by Flink, " + + "which produces only deletion vectors for deletes.", + stagingSnapshot.snapshotId(), stagingBranch, deleteFile.location())); + } + } + + return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); + } + + /** Files added by one staging snapshot, classified for cycle emission. */ + private record StagingInputs( + List newDataFiles, + List stagingDVFiles, + List eqDeleteFiles) { + + boolean isEmpty() { + return newDataFiles.isEmpty() && eqDeleteFiles.isEmpty() && stagingDVFiles.isEmpty(); + } + } + + private void emitDeletePhase(List eqDeleteFiles) { + for (DeleteFile deleteFile : eqDeleteFiles) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.eqDeleteFile( + deleteFile, + spec, + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(deleteFile)), + nextPhaseTs)); + processedEqDeleteFileNumCounter.inc(); + } + + advancePhase(); + } + + private void emitSnapshotDataPhase(List snapshotDataFiles) { + // Shared-branch skip: when stagingBranch == targetBranch, these files are already on target + // and were indexed by bootstrap/reindex. Re-emitting would duplicate entries in + // dataRowPositions. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : snapshotDataFiles) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.stagingDataFile( + new FlinkAddedRowsScanTask(dataFile, spec), + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(dataFile)), + nextPhaseTs)); + } + } + + advancePhase(); + } + + private void emitNoOpResult(long triggerTimestamp, Long currentMainSnapshotId) { + skippedNoOpCyclesCounter.inc(); + emitDrainResult(triggerTimestamp, currentMainSnapshotId); + } + + /** Emits an empty plan result and advances the phase so the pipeline drains. */ + private void emitDrainResult(long triggerTimestamp, Long currentMainSnapshotId) { + output.collect( + METADATA_STREAM, + new StreamRecord<>( + EqualityConvertPlan.noOp(currentMainSnapshotId, triggerTimestamp, nextPhaseTs))); + advancePhase(); + } + + /** + * Emits {@link ReadCommand}s for every data file on {@code mainSnapshot} so the worker indexes + * them for the configured equality-field set. Existing DVs attached to a data file are loaded by + * the reader and their positions are skipped. V2 positional deletes are not expected on main; the + * reader throws if it encounters one. Equality deletes attached to the scan task are skipped + * during indexing (they are processed via the planner's eq-delete read commands). + */ + private void emitMainDataReadCommands(Snapshot mainSnapshot) { + long commitSnapshotId = mainSnapshot.snapshotId(); + + try (CloseableIterable tasks = + table.newScan().useSnapshot(commitSnapshotId).planFiles()) { + for (FileScanTask task : tasks) { + output.collect( + new StreamRecord<>( + ReadCommand.dataFile( + task, indexSnapshotId, indexedSequenceNumber, dataSequenceNumber(task.file())), + nextPhaseTs)); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to plan files for main index", e); + } + + LOG.info( + "Emitted main data read commands for field IDs {} from snapshot {}.", + eqFieldIds, + commitSnapshotId); + + advancePhase(); + } + + /** + * Emits a phase-end watermark and bumps the phase timestamp. Every phase-emitting method must + * call this exactly once after its records; the worker uses these watermarks to gate keyed-state + * transitions. Missing or extra calls silently break ordering. + */ + private void advancePhase() { + output.emitWatermark(new Watermark(nextPhaseTs)); + nextPhaseTs++; + } + + /** + * Returns {@code true} if {@code snapshot} can be skipped: + * + *
    + *
  • It was already committed by us (carries {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}), OR + *
  • it adds no data or delete files (e.g. delete-file-only removals), OR + *
  • {@code stagingBranch == targetBranch} and the snapshot adds no equality-delete files. + * Pure-insert (and data-file-only) commits on the shared branch don't need a conversion + * cycle: their data is already on target, and the worker's index stays fresh via {@link + * #ensureIndexCurrent} when the next eq-delete arrives. + *
+ * + *

Filter checks read per-snapshot counts from {@link Snapshot#summary()}; we don't parse + * manifests here. Manifest parsing happens later in {@link #retrieveStagingFiles} only for the + * chosen snapshot. + */ + private boolean shouldSkip(Snapshot snapshot) { + Map summary = snapshot.summary(); + if (summary.containsKey(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) { + return true; + } + + long addedDataFiles = summaryCount(summary, SnapshotSummary.ADDED_FILES_PROP); + long addedDeleteFiles = summaryCount(summary, SnapshotSummary.ADDED_DELETE_FILES_PROP); + if (addedDataFiles == 0 && addedDeleteFiles == 0) { + LOG.info( + "Skipping staging snapshot {}: no added data or delete files.", snapshot.snapshotId()); + return true; + } + + if (stagingOnTargetBranch + && summaryCount(summary, SnapshotSummary.ADD_EQ_DELETE_FILES_PROP) == 0) { + return true; + } + + return false; + } + + private static long summaryCount(Map summary, String key) { + String value = summary.get(key); + return value == null ? 0L : Long.parseLong(value); + } + + /** + * Returns the id of the newest snapshot that is reachable from both the staging and target branch + * heads (i.e. the most recent common ancestor where the two branches last matched), or {@code + * null} if they share no history. Used on cold start to know where staging diverged from target + * so we can skip converting snapshots that already exist on target. + */ + private Long findCommonAncestor(Snapshot stagingHead, Snapshot mainHead) { + if (mainHead == null) { + return null; + } + + Set stagingSeen = Sets.newHashSet(); + Set mainSeen = Sets.newHashSet(); + Snapshot stagingCurrent = stagingHead; + Snapshot mainCurrent = mainHead; + + while (stagingCurrent != null || mainCurrent != null) { + if (stagingCurrent != null) { + long id = stagingCurrent.snapshotId(); + if (mainSeen.contains(id)) { + return id; + } + + stagingSeen.add(id); + stagingCurrent = parentOf(stagingCurrent); + } + + if (mainCurrent != null) { + long id = mainCurrent.snapshotId(); + if (stagingSeen.contains(id)) { + return id; + } + + mainSeen.add(id); + mainCurrent = parentOf(mainCurrent); + } + } + + return null; + } + + /** Returns the parent snapshot, or {@code null} if {@code snapshot} has no parent. */ + private Snapshot parentOf(Snapshot snapshot) { + Long parentId = snapshot.parentId(); + return parentId != null ? table.snapshot(parentId) : null; + } + + private static long dataSequenceNumber(ContentFile file) { + Long sequenceNumber = file.dataSequenceNumber(); + Preconditions.checkNotNull( + sequenceNumber, "Missing data sequence number for committed file %s", file.location()); + return sequenceNumber; + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + @VisibleForTesting + long reindexCount() { + return reindexCounter.getCount(); + } + + @VisibleForTesting + long skippedNoOpCycles() { + return skippedNoOpCyclesCounter.getCount(); + } + + @VisibleForTesting + long processedStagingSnapshotNum() { + return processedStagingSnapshotNumCounter.getCount(); + } + + @VisibleForTesting + long processedEqDeleteFileNum() { + return processedEqDeleteFileNumCounter.getCount(); + } +} diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityDeleteFileScanTask.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityDeleteFileScanTask.java index 2b1fc4096609..649344e35cab 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityDeleteFileScanTask.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityDeleteFileScanTask.java @@ -29,10 +29,30 @@ * {@link ContentScanTask} for reading a single equality delete file standalone. The {@link * DeleteFile#equalityFieldIds()} on {@link #file()} carries the PK field IDs the worker resolves * against. + * + *

A plain class rather than a record so it round-trips through Flink's Kryo fallback on Flink + * 1.20, whose bundled Kryo 2.x cannot build a serializer for record classes. */ @Internal -record EqualityDeleteFileScanTask(DeleteFile file, PartitionSpec spec) - implements ContentScanTask { +class EqualityDeleteFileScanTask implements ContentScanTask { + + private final DeleteFile file; + private final PartitionSpec spec; + + EqualityDeleteFileScanTask(DeleteFile file, PartitionSpec spec) { + this.file = file; + this.spec = spec; + } + + @Override + public DeleteFile file() { + return file; + } + + @Override + public PartitionSpec spec() { + return spec; + } @Override public long start() { diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/FlinkAddedRowsScanTask.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/FlinkAddedRowsScanTask.java index 5b1bb016316a..d5787ba9fe9c 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/FlinkAddedRowsScanTask.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/FlinkAddedRowsScanTask.java @@ -34,22 +34,45 @@ * staging data file from {@code SnapshotChanges#addedDataFiles}) and need to feed it into the * reader. For data files that already come from a planned scan, we pass the native {@link * FileScanTask} through directly. + * + *

A plain class rather than a record to please Kryo 2.x in Flink 1.20, both of which cannot + * build a serializer for record classes. */ @Internal -record FlinkAddedRowsScanTask(DataFile file, PartitionSpec spec, List deletes) - implements FileScanTask { +class FlinkAddedRowsScanTask implements FileScanTask { + + private final DataFile file; + private final PartitionSpec spec; + private final List deletes; - FlinkAddedRowsScanTask { + FlinkAddedRowsScanTask(DataFile file, PartitionSpec spec, List deletes) { + this.file = file; + this.spec = spec; // Iceberg's scan APIs return Guava ImmutableList (see BaseAddedRowsScanTask#deletes), which - // does not round-trip through Flink's Kryo fallback. Copy into a plain ArrayList so the record + // does not round-trip through Flink's Kryo fallback. Copy into a plain ArrayList so the task // serializes cleanly regardless of what the caller passes. - deletes = Lists.newArrayList(deletes); + this.deletes = Lists.newArrayList(deletes); } FlinkAddedRowsScanTask(DataFile file, PartitionSpec spec) { this(file, spec, Collections.emptyList()); } + @Override + public DataFile file() { + return file; + } + + @Override + public PartitionSpec spec() { + return spec; + } + + @Override + public List deletes() { + return deletes; + } + @Override public long start() { return 0L; diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java new file mode 100644 index 000000000000..82860a0de881 --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -0,0 +1,1115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestEqualityConvertPlanner extends OperatorTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void bootstrapsIndexFromBuilderEqualityFieldIds() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Staging branch exists but contains no eq-delete files yet. The planner still populates the + // worker index from main using the builder-configured equality field set so the index is ready + // when the first delete arrives. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isZero(); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Builder configured for [1, 2], but writer produces an eq delete with [1] only. + DeleteFile mismatched = writeIdOnlyEqualityDelete(table, 1); + table.newRowDelta().addDeletes(mismatched).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { + // When stagingBranch == targetBranch, the writer commits new data files directly to main. + // Bootstrap scans the main snapshot (which already includes those files) and indexes them. + // The planner must NOT also emit the staging-data phase for the same files — that would + // duplicate ADD_DATA_ROW commands for the same (PK, file, position) in the worker's index. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Writer commits (new-data id=3) + (eq-delete id=1) in one RowDelta directly to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(SnapshotRef.MAIN_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + + // Bootstrap from main emits one data-file command per file (id=1, id=2, id=3) = 3. + // Without the same-branch guard, emitSnapshotDataPhase would also emit newDataFile → 4. + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + long newDataFileCount = + commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .filter(c -> c.task().file().location().equals(newDataFile.location())) + .count(); + assertThat(newDataFileCount).isEqualTo(1); + } + } + + @Test + void emitsReadCommandsForEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 3 DATA_FILE (from main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).isEmpty(); + } + } + + @Test + void emitsDataFileFromInsertOnlyStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: eq delete targeting main data + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s1SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // S2: insert-only (no eq deletes), but its data must still be indexed for the configured + // equality fields + DataFile insertS2 = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(insertS2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: processes S1 (eq delete) + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + // Record S1's conversion on main, so the planner walks past S1 before processing S2. + simulateConvertCommit(table, s1SnapshotId); + + // Trigger 2: processes S2 (insert-only). Must emit its data file so the configured equality + // fields stay indexed. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + + Set dataFilePaths = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .map(TestEqualityConvertPlanner::filePath) + .collect(Collectors.toSet()); + + assertThat(dataFilePaths).contains(insertS2.location()); + } + } + + @Test + void includesNewDataFilesFromStaging() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 1 main data file + 1 eq delete + 1 staging data file + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).hasSize(1); + } + } + + @Test + void findsIntersectionWhenMainAdvancedAfterStagingFork() throws Exception { + Table table = createTableWithDelete(3); + // Pre-fork main: two snapshots. + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Fork staging from current main head. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Main advances with two more snapshots after the fork. These are on main's + // lineage but not on staging's. + insert(table, 3, "c"); + insert(table, 4, "d"); + + // Staging gets one new snapshot containing an equality delete. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Planner identifies the fork point and processes only the staging-only + // commit (the eq delete), and emits all four current main data files + // for the index refresh. + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(countDataFileTasks(commands)).isEqualTo(4); + } + } + + @Test + void skipsAlreadyProcessedStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + assertThat(firstTriggerCount).isGreaterThan(0); + assertThat(planner(harness).skippedNoOpCycles()).isZero(); + + // Simulate the committer committing to main with the staging snapshot property + // (the planner promotes pending only after confirming the commit landed on main). + simulateConvertCommit(table, stagingSnapshotId); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(firstTriggerCount); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + } + } + + @Test + void noOutputWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Bootstrap from main for the configured field set: 1 main DATA_FILE; no-op metadata still + // emitted because there's nothing on staging to convert. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(1); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isZero(); + } + } + + @Test + void propagatesStagingOnlyPositionalDeletes() throws Exception { + Table table = createTableWithDelete(3); + DataFile mainData = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(mainData).commit(); + table.refresh(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot has only a DV (no eq deletes, no new data files). Without the + // planner forwarding stagingDVFiles in this case, the committer would never + // see the DV and it would be lost. + DeleteFile stagingDV = writeStagingDV(table, mainData.location(), 0L); + table.newRowDelta().addDeletes(stagingDV).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(commands)).isZero(); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + EqualityConvertPlan result = metadata.get(0).getValue(); + assertThat(result.dataFiles()).isEmpty(); + assertThat(result.stagingDVFiles()).hasSize(1); + assertThat(result.stagingDVFiles().get(0).location()).isEqualTo(stagingDV.location()); + } + } + + @Test + void phaseTimestampsAreMonotonicallyIncreasing() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging: eq delete + new data file (triggers 3 phases: main data, eq delete, staging data) + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + long triggerTs = 100L; + sendTrigger(harness, triggerTs); + + List output = Lists.newArrayList(harness.getOutput()); + assertThat(((StreamRecord) output.get(0)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(1)).isEqualTo(new Watermark(triggerTs)); + + assertThat(((StreamRecord) output.get(2)).getValue().task()) + .isInstanceOf(EqualityDeleteFileScanTask.class); + assertThat(output.get(3)).isEqualTo(new Watermark(triggerTs + 1)); + + assertThat(((StreamRecord) output.get(4)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(5)).isEqualTo(new Watermark(triggerTs + 2)); + } + } + + @Test + void routesExceptionToErrorStream() throws Exception { + createTableWithDelete(3); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + dropTable(); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + sendTrigger(harness); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsOnRemovedDataFilesOnStagingBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + // Rewrite on the staging branch (not main). Equality delete conversion does not support + // rewrites on staging; the planner must fail the cycle instead of silently dropping the + // removed files. + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + // Bootstrap runs before processCycle's failure, so the main data commands are already on + // the wire. Only the cycle itself fails. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + } + } + + @Test + void reEmitsMainDataAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + + // Compact data files on main: rewrite 2 files into 1 + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // 1 DATA_FILE (compacted main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + // DATA_FILE should reference the compacted file + List dataCmds = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .collect(Collectors.toList()); + for (ReadCommand cmd : dataCmds) { + assertThat(cmd.task().file().location()).isEqualTo(compactedFile.location()); + } + } + } + + @Test + void refreshesIndexBeforeFirstCycleProcessesEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Stage MULTIPLE eq-delete snapshots before the first trigger arrives. The first trigger + // bootstraps the index from main for the configured equality fields before processing any eq + // deletes, then processes one snapshot per trigger (oldest first). + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s2SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + + // First trigger processes the OLDER staging snapshot (eqDelete1). Bootstrap built the index + // from main once: 2 main DATA_FILE + 1 EQ_DELETE_FILE = 3 commands. + assertThat(afterFirst).isEqualTo(3); + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + simulateConvertCommit(table, table.snapshot(STAGING_BRANCH).parentId()); + + // Second trigger processes the newer staging snapshot. The index is already up-to-date. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + assertThat(countDataFileTasks(trigger2Commands)).isZero(); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + simulateConvertCommit(table, s2SnapshotId); + } + } + + @Test + void noMainReEmitWhenUnchanged() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Only 1 EQ_DELETE_FILE, no main re-emission + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(2); + } + } + + @Test + void noMainReEmitAfterOwnCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long indexedStagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + + // Record the converter's own commit (COMMITTED_STAGING_SNAPSHOT property) for the staging + // snapshot the first trigger processed. The next trigger reads the property off main and + // advances lastStagingSnapshotId. No reindex should be triggered. + simulateConvertCommit(table, indexedStagingSnapshotId); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Own commits don't trigger index rebuild. + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + } + } + + @Test + void reIndexesAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 1 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(2); + assertThat(planner(harness).reindexCount()).isZero(); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + // External commit: no COMMITTED_STAGING_SNAPSHOT_PROPERTY + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // External commit triggers re-index: main data files are re-emitted + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(2); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(2); + } + } + + @Test + void emitsClearIndexBroadcastOnReindex() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1 bootstraps the index. Bootstrap does not broadcast CLEAR_INDEX. + sendTrigger(harness); + assertThat(harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM)) + .isNullOrEmpty(); + + // External commit advances main. The next trigger reindexes and must broadcast CLEAR_INDEX + // so workers evict ghost keys (e.g. a PK whose data file was removed by external CoW). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + long mainAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).snapshotId(); + long mainSeqAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).sequenceNumber(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + + List clears = + harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM).stream() + .map(StreamRecord::getValue) + .collect(Collectors.toList()); + assertThat(clears).hasSize(1); + assertThat(clears.get(0).type()).isEqualTo(IndexCommand.Type.CLEAR_INDEX); + assertThat(clears.get(0).mainSnapshotId()).isEqualTo(mainAfterExternal); + assertThat(clears.get(0).mainSequenceNumber()).isEqualTo(mainSeqAfterExternal); + } + } + + @Test + void detectsMainBranchChangeWithoutNewStagingSnapshots() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: build the index and process the existing eq delete. + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(afterFirst).isGreaterThan(0); + + // External commit on main (no COMMITTED_STAGING_SNAPSHOT_PROPERTY). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + // Trigger 2: nothing new on staging, but the planner must still notice the + // external main change and immediately re-emit main data for index rebuild. + sendTrigger(harness); + List allAfterTrigger2 = harness.extractOutputValues(); + List trigger2Commands = + allAfterTrigger2.subList(afterFirst, allAfterTrigger2.size()); + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + + // A subsequent staging eq delete should NOT re-emit main data because the index + // was already rebuilt in trigger 2. + int afterSecond = allAfterTrigger2.size(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 10, "z"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List all = harness.extractOutputValues(); + List trigger3 = all.subList(afterSecond, all.size()); + assertThat(countDataFileTasks(trigger3)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger3)).isEqualTo(1); + } + } + + @Test + void stateRestoredAfterCheckpoint() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // First trigger bootstraps the index and processes the staging snapshot. + sendTrigger(harness); + int commandCount = harness.extractOutputValues().size(); + assertThat(commandCount).isGreaterThan(0); + + // Record S1's conversion on main (committer marker) so the planner advances + // lastStagingSnapshotId on the second trigger. + simulateConvertCommit(table, stagingSnapshotId); + + // Second trigger reads the marker and advances lastStagingSnapshotId. + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(commandCount); + + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // Restore from checkpoint: the restored index is not re-indexed because its state matches the + // snapshot COMMITTED_STAGING_SNAPSHOT_PROPERTY property. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void failsOnV2PosDeleteOnStagingBranch() throws Exception { + Table table = createTableWithDelete(2); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Flink-written staging branches are V3 and only produce DVs for deletes. A V2 positional + // delete on staging is a writer-side misconfiguration and should fail the cycle rather than + // be silently absorbed. + DataFile dataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile posDelete = writePosDeleteFile(table, dataFile.location(), 0); + table.newRowDelta().addRows(dataFile).addDeletes(posDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void skipsCommittedSnapshotAfterCommitLandsButCheckpointDoesNot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // Capture the planner's checkpoint before any cycle has run. + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // The staging commit landed on main but lastStagingSnapshotId was not checkpointed. + simulateConvertCommit(table, stagingSnapshotId); + + // On restore, the planner walks main, finds the COMMITTED_STAGING_SNAPSHOT_PROPERTY, + // advances lastStagingSnapshotId, and skips re-emitting the already-committed snapshot. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(0); + // Bootstrap from main creates the index from the main data files on the first trigger even + // though the staging snapshot itself is already committed and skipped. Main now has the + // original insert plus simulateConvertCommit's marker file = 2 data files. + assertThat(countDataFileTasks(commands)).isEqualTo(2); + } + } + + @Test + void failsWhenCommitMarkerDisappears() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + long startSnapshotId = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + simulateConvertCommit(table, stagingSnapshotId); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + table.manageSnapshots().rollbackTo(startSnapshotId).commit(); + table.refresh(); + + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + assertThat( + harness + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .poll() + .getValue() + .getMessage()) + .contains("No COMMITTED_STAGING_SNAPSHOT marker reachable"); + } + } + + @Test + void failsOnEqFieldIdsChangeAcrossRestart() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + assertThatThrownBy(() -> harness.initializeState(state)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Equality field IDs changed across restart"); + } + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch) throws Exception { + // Default matches the [1, 2] equality-field-id list produced by writeEqualityDelete. + return createHarness(stagingBranch, Lists.newArrayList(1, 2)); + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch, List eqFieldIds) throws Exception { + return new OneInputStreamOperatorTestHarness<>( + new EqualityConvertPlanner( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + stagingBranch, + SnapshotRef.MAIN_BRANCH, + Sets.newHashSet(eqFieldIds))); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness) + throws Exception { + sendTrigger(harness, System.currentTimeMillis()); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness, long time) + throws Exception { + harness.processElement(new StreamRecord<>(Trigger.create(time, 0), time)); + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(record), + idOnly); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private static long countDataFileTasks(List commands) { + return commands.stream().filter(c -> c.task() instanceof FileScanTask).count(); + } + + private static long countEqDeleteTasks(List commands) { + return commands.stream().filter(TestEqualityConvertPlanner::isEqDelete).count(); + } + + private static EqualityConvertPlanner planner( + OneInputStreamOperatorTestHarness harness) { + return (EqualityConvertPlanner) harness.getOperator(); + } + + private static boolean isEqDelete(ReadCommand cmd) { + return cmd.task() instanceof EqualityDeleteFileScanTask; + } + + private static String filePath(ReadCommand cmd) { + return cmd.task().file().location(); + } + + private void simulateConvertCommit(Table table, long stagingSnapshotId) throws IOException { + DataFile dummy = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(-1, "marker" + stagingSnapshotId))); + table + .newRowDelta() + .addRows(dummy) + .set( + EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY, + String.valueOf(stagingSnapshotId)) + .commit(); + table.refresh(); + } +} diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java new file mode 100644 index 000000000000..c3c1785290cf --- /dev/null +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -0,0 +1,762 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.OutputTag; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotChanges; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Planner for the equality delete conversion pipeline. For each trigger, it picks the oldest + * staging snapshot that hasn't been converted yet and emits {@link ReadCommand}s describing the + * files its downstream readers and workers must process. + * + *

Each trigger runs two steps in order: + * + *

    + *
  1. {@link #ensureIndexCurrent}: updates {@link #lastStagingSnapshotId} from main's history, + * bootstraps the worker index from main on first run, and reindexes when external commits + * (e.g. compaction) have advanced main past the currently-indexed snapshot. + *
  2. {@link #processStagingSnapshot}: resolve the chosen staging snapshot's eq deletes against + * the (now-current) index, pass through any DV files, and index the snapshot's new data files + * for the next cycle. + *
+ * + * Watermarks separate phases that gate the worker's keyed state. The contract is documented on + * {@link #advancePhase()}. + * + *

An {@link EqualityConvertPlan} with the current cycle's metadata is emitted via the {@link + * #METADATA_STREAM} side output after the read commands. + * + *

Assumes a single equality-field set supplied via the builder; staging eq-deletes with a + * different {@code equalityFieldIds} fail fast in {@link #retrieveStagingFiles}. Concurrent writes + * on the target branch are handled by {@link #ensureIndexCurrent} reindexing from the new main + * snapshot; commit-time conflicts are caught by {@code RowDelta.validateFromSnapshot}. + */ +@Internal +public class EqualityConvertPlanner extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertPlanner.class); + + public static final OutputTag METADATA_STREAM = + new OutputTag<>("metadata-stream") {}; + + public static final OutputTag CLEAR_BROADCAST_STREAM = + new OutputTag<>("clear-broadcast-stream") {}; + + private static final String PROCESSED_EQ_DELETE_FILE_NUM_METRIC = "processedEqDeleteFileNum"; + private static final String PROCESSED_STAGING_SNAPSHOT_NUM_METRIC = "processedStagingSnapshotNum"; + private static final String SKIPPED_NO_OP_CYCLES_METRIC = "skippedNoOpCycles"; + private static final String REINDEX_COUNT_METRIC = "reindexCount"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String stagingBranch; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + // Equality-field-id set the worker keys on. Supplied via the builder; every staging + // eq-delete's equalityFieldIds() must match exactly. + private final Set eqFieldIds; + + // Main snapshot id the worker's index reflects. + private transient ListState indexSnapshotState; + // Main sequence number the worker's index reflects. + private transient ListState indexedSequenceNumberState; + // Equality field IDs the index was built with, allows to detect reconfiguration. + private transient ListState eqFieldIdsState; + + private transient Table table; + + private transient Long lastMainSnapshotId; + private transient Long lastStagingSnapshotId; + private transient Long indexSnapshotId; + private transient Long indexedSequenceNumber; + + private transient long nextPhaseTs; + + private transient Counter processedEqDeleteFileNumCounter; + private transient Counter processedStagingSnapshotNumCounter; + private transient Counter skippedNoOpCyclesCounter; + private transient Counter reindexCounter; + private transient Counter errorCounter; + + public EqualityConvertPlanner( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch, + Set eqFieldIds) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.stagingBranch = stagingBranch; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + Preconditions.checkArgument( + eqFieldIds != null && !eqFieldIds.isEmpty(), "eqFieldIds must not be null or empty"); + this.eqFieldIds = ImmutableSet.copyOf(eqFieldIds); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.processedEqDeleteFileNumCounter = + taskMetricGroup.counter(PROCESSED_EQ_DELETE_FILE_NUM_METRIC); + this.processedStagingSnapshotNumCounter = + taskMetricGroup.counter(PROCESSED_STAGING_SNAPSHOT_NUM_METRIC); + this.skippedNoOpCyclesCounter = taskMetricGroup.counter(SKIPPED_NO_OP_CYCLES_METRIC); + this.reindexCounter = taskMetricGroup.counter(REINDEX_COUNT_METRIC); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + indexSnapshotState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexSnapshotId", Types.LONG)); + + indexSnapshotId = null; + for (Long stateValue : indexSnapshotState.get()) { + Preconditions.checkState( + indexSnapshotId == null, "indexSnapshotId state should hold at most one value"); + indexSnapshotId = stateValue; + } + + indexedSequenceNumberState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexedSequenceNumber", Types.LONG)); + + indexedSequenceNumber = null; + for (Long stateValue : indexedSequenceNumberState.get()) { + Preconditions.checkState( + indexedSequenceNumber == null, + "indexedSequenceNumber state should hold at most one value"); + indexedSequenceNumber = stateValue; + } + + eqFieldIdsState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("eqFieldIds", Types.INT)); + Set restoredEqFieldIds = Sets.newHashSet(eqFieldIdsState.get()); + Preconditions.checkState( + restoredEqFieldIds.isEmpty() || restoredEqFieldIds.equals(eqFieldIds), + "Equality field IDs changed across restart: restored=%s, configured=%s. " + + "Reconfiguring equality-field columns is not supported; " + + "restart from a clean state (no savepoint).", + restoredEqFieldIds, + eqFieldIds); + } + + @Override + public void snapshotState(StateSnapshotContext context) throws Exception { + super.snapshotState(context); + indexSnapshotState.clear(); + if (indexSnapshotId != null) { + indexSnapshotState.add(indexSnapshotId); + } + + indexedSequenceNumberState.clear(); + if (indexedSequenceNumber != null) { + indexedSequenceNumberState.add(indexedSequenceNumber); + } + + eqFieldIdsState.clear(); + for (int id : eqFieldIds) { + eqFieldIdsState.add(id); + } + } + + @Override + public void processElement(StreamRecord element) throws Exception { + long triggerTs = element.getTimestamp(); + nextPhaseTs = Math.max(triggerTs, nextPhaseTs + 1); + + Long currentMainSnapshotId = lastMainSnapshotId; + try { + table.refresh(); + Snapshot mainSnapshot = table.snapshot(targetBranch); + currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + ensureIndexCurrent(mainSnapshot); + + Snapshot nextToProcess = + nextUnprocessedStagingSnapshot(table.snapshot(stagingBranch), mainSnapshot); + + if (nextToProcess == null) { + LOG.info("Nothing new to convert on staging branch '{}'.", stagingBranch); + emitNoOpResult(triggerTs, currentMainSnapshotId); + return; + } + + processStagingSnapshot(nextToProcess, triggerTs, currentMainSnapshotId); + } catch (Exception e) { + LOG.error("Error processing equality deletes for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + emitDrainResult(triggerTs, currentMainSnapshotId); + } + } + + /** + * Brings the worker's index up to date with the current state of the target branch: + * + *

    + *
  • Updates {@link #lastStagingSnapshotId} from the most recent committer marker on main. + *
  • Bootstraps the index from main on the first trigger with a non-null main snapshot. + *
  • Reindexes from main when external commits (e.g. compaction or direct writes) have + * advanced main past the currently-indexed snapshot. + *
+ * + *

No-op when main hasn't moved since the last trigger. Otherwise the history walk is bounded + * to commits added since {@link #lastMainSnapshotId}. + */ + private void ensureIndexCurrent(Snapshot mainSnapshot) { + Long currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + if (Objects.equals(lastMainSnapshotId, currentMainSnapshotId)) { + return; + } + + LastCommittedWork info = discoverLastCommittedWork(mainSnapshot); + updateLastStagingSnapshotId(info); + + boolean bootstrap = mainSnapshot != null && indexSnapshotId == null; + boolean reindex = indexSnapshotId != null && info.externalCommitCount() > 0; + if (bootstrap || reindex) { + LOG.info( + "{} worker index from main snapshot {} for field IDs {}.", + bootstrap ? "Bootstrapping" : "Reindexing", + currentMainSnapshotId, + eqFieldIds); + if (reindex) { + // Evict keyed entries the reindex will not re-add (e.g. data file removed by CoW). + output.collect( + CLEAR_BROADCAST_STREAM, + new StreamRecord<>( + IndexCommand.clearBeforeReindex( + currentMainSnapshotId, mainSnapshot.sequenceNumber()))); + reindexCounter.inc(); + } + + indexSnapshotId = currentMainSnapshotId; + indexedSequenceNumber = mainSnapshot.sequenceNumber(); + emitMainDataReadCommands(mainSnapshot); + } + + lastMainSnapshotId = currentMainSnapshotId; + } + + private void updateLastStagingSnapshotId(LastCommittedWork info) { + if (info.lastCommittedStaging() != null) { + lastStagingSnapshotId = info.lastCommittedStaging(); + return; + } + + Preconditions.checkState( + lastMainSnapshotId == null || lastStagingSnapshotId == null || info.reachedLastInspected(), + "No COMMITTED_STAGING_SNAPSHOT marker reachable on target branch '%s' for table %s, " + + "but a prior marker was seen (lastStagingSnapshotId=%s). Target may have been " + + "rewritten (rollback, replace_main, or snapshot expiration). " + + "Manual intervention required.", + targetBranch, + tableName, + lastStagingSnapshotId); + } + + /** + * Walks main back from head looking for the most recent snapshot tagged with {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}. Returns the staging snapshot id + * recorded there (or {@code null} if not reached) and the count of intervening external commits. + * + *

The walk stops at whichever comes first: + * + *

    + *
  • The first snapshot carrying the committer marker. + *
  • {@link #lastMainSnapshotId} — anything older was inspected on a previous trigger. + *
+ */ + private LastCommittedWork discoverLastCommittedWork(Snapshot mainSnapshot) { + Long lastCommittedStaging = null; + int externalCount = 0; + boolean reachedLastInspected = false; + Snapshot current = mainSnapshot; + while (current != null) { + if (lastMainSnapshotId != null && current.snapshotId() == lastMainSnapshotId) { + reachedLastInspected = true; + break; + } + + String prop = + current.summary().get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY); + if (prop != null) { + lastCommittedStaging = Long.parseLong(prop); + break; + } + + externalCount++; + current = parentOf(current); + } + + return new LastCommittedWork(lastCommittedStaging, externalCount, reachedLastInspected); + } + + private record LastCommittedWork( + Long lastCommittedStaging, int externalCommitCount, boolean reachedLastInspected) {} + + /** + * Walks staging history from head back to the stop point and returns the oldest unprocessed + * snapshot to convert this cycle, or {@code null} if there's nothing new. + * + *

The stop point is: + * + *

    + *
  • the last-processed staging snapshot, if known; + *
  • otherwise, on cold start with {@code stagingBranch != targetBranch}, the common ancestor + * with target; + *
  • otherwise, on cold start with {@code stagingBranch == targetBranch}, the root of history + * ({@code null}). {@link #shouldSkip} filters already-converted snapshots in O(1) via the + * {@code COMMITTED_STAGING_SNAPSHOT_PROPERTY} marker, and pure-insert snapshots via the + * same-branch eq-delete predicate. + *
+ * + *

When {@code stagingBranch == targetBranch}, the writer commits eq-deletes and data files + * directly to target; the converter performs in-place eq-delete-to-DV compaction. On cold start + * we walk the full history so eq-deletes committed before the converter started are still picked + * up. + */ + private Snapshot nextUnprocessedStagingSnapshot(Snapshot stagingHead, Snapshot mainSnapshot) { + if (stagingHead == null) { + return null; + } + + Long stopAt; + if (lastStagingSnapshotId != null) { + stopAt = lastStagingSnapshotId; + } else if (stagingOnTargetBranch) { + stopAt = null; + } else { + stopAt = findCommonAncestor(stagingHead, mainSnapshot); + } + + Snapshot current = stagingHead; + Snapshot oldestUnprocessed = null; + while (current != null) { + if (stopAt != null && current.snapshotId() == stopAt) { + break; + } + + if (!shouldSkip(current)) { + oldestUnprocessed = current; + } + + current = parentOf(current); + } + + return oldestUnprocessed; + } + + /** + * Resolves the eq deletes in {@code stagingSnapshot} against the current index and emits the + * cycle's metadata. Phase ordering (separated by watermarks): + * + *

    + *
  1. Eq delete read commands. Eq deletes resolve in the worker. + *
  2. Staging data files. Indexed for the NEXT cycle's eq-delete resolution. + *
+ * + * Cold-start bootstrap of the index from main data is handled separately in {@link + * #processElement}, which runs once before the first cycle. + */ + private void processStagingSnapshot( + Snapshot stagingSnapshot, long triggerTs, Long currentMainSnapshotId) { + + StagingInputs inputs = retrieveStagingFiles(stagingSnapshot); + Preconditions.checkState( + !inputs.isEmpty(), + "Staging snapshot %s has no convertible inputs; shouldSkip should have filtered it.", + stagingSnapshot.snapshotId()); + + emitDeletePhase(inputs.eqDeleteFiles()); + emitSnapshotDataPhase(inputs.newDataFiles()); + + LOG.info( + "Emitted read commands for {} new data files from staging branch '{}'.", + inputs.newDataFiles().size(), + stagingBranch); + + processedStagingSnapshotNumCounter.inc(); + + output.collect( + METADATA_STREAM, + new StreamRecord<>( + new EqualityConvertPlan( + inputs.newDataFiles(), + inputs.stagingDVFiles(), + stagingSnapshot.snapshotId(), + currentMainSnapshotId, + triggerTs, + nextPhaseTs))); + + advancePhase(); + } + + /** + * Classifies the files added by {@code stagingSnapshot} into data files, eq delete files, and DV + * files. Throws if the snapshot: + * + *
    + *
  • Removes data files (rewrites on the staging branch aren't supported). + *
  • Contains V2 positional delete files (the converter expects a V3 staging branch written by + * Flink, which produces only deletion vectors for deletes). + *
  • Contains an eq-delete file whose {@code equalityFieldIds()} doesn't match the + * builder-configured set (silent wrong-key serialization otherwise). + *
+ */ + private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { + SnapshotChanges changes = SnapshotChanges.builderFor(table).snapshot(stagingSnapshot).build(); + + // Rewrites on the staging branch would require rewriting the corresponding DVs against new + // data files on target. Not implemented; fail fast instead of silently dropping work. + if (changes.removedDataFiles().iterator().hasNext()) { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' removes data files; " + + "equality delete conversion does not support rewrites on the staging branch. " + + "Run compaction on the target branch instead.", + stagingSnapshot.snapshotId(), stagingBranch)); + } + + List newDataFiles = Lists.newArrayList(); + List stagingDVFiles = Lists.newArrayList(); + List eqDeleteFiles = Lists.newArrayList(); + + for (DataFile dataFile : changes.addedDataFiles()) { + newDataFiles.add(dataFile); + } + + for (DeleteFile deleteFile : changes.addedDeleteFiles()) { + if (deleteFile.content() == FileContent.EQUALITY_DELETES) { + Set deleteFieldIds = Sets.newHashSet(deleteFile.equalityFieldIds()); + Preconditions.checkState( + deleteFieldIds.equals(eqFieldIds), + "Staging snapshot %s on branch '%s' contains an equality delete file %s with " + + "equalityFieldIds=%s, which does not match the configured eqFieldIds=%s. " + + "The writer must use the same equality field IDs as the converter.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + deleteFieldIds, + eqFieldIds); + eqDeleteFiles.add(deleteFile); + } else if (ContentFileUtil.isDV(deleteFile)) { + stagingDVFiles.add(deleteFile); + } else { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' contains a V2 positional delete file (%s); " + + "equality delete conversion expects a V3 staging branch written by Flink, " + + "which produces only deletion vectors for deletes.", + stagingSnapshot.snapshotId(), stagingBranch, deleteFile.location())); + } + } + + return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); + } + + /** Files added by one staging snapshot, classified for cycle emission. */ + private record StagingInputs( + List newDataFiles, + List stagingDVFiles, + List eqDeleteFiles) { + + boolean isEmpty() { + return newDataFiles.isEmpty() && eqDeleteFiles.isEmpty() && stagingDVFiles.isEmpty(); + } + } + + private void emitDeletePhase(List eqDeleteFiles) { + for (DeleteFile deleteFile : eqDeleteFiles) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.eqDeleteFile( + deleteFile, + spec, + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(deleteFile)), + nextPhaseTs)); + processedEqDeleteFileNumCounter.inc(); + } + + advancePhase(); + } + + private void emitSnapshotDataPhase(List snapshotDataFiles) { + // Shared-branch skip: when stagingBranch == targetBranch, these files are already on target + // and were indexed by bootstrap/reindex. Re-emitting would duplicate entries in + // dataRowPositions. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : snapshotDataFiles) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.stagingDataFile( + new FlinkAddedRowsScanTask(dataFile, spec), + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(dataFile)), + nextPhaseTs)); + } + } + + advancePhase(); + } + + private void emitNoOpResult(long triggerTimestamp, Long currentMainSnapshotId) { + skippedNoOpCyclesCounter.inc(); + emitDrainResult(triggerTimestamp, currentMainSnapshotId); + } + + /** Emits an empty plan result and advances the phase so the pipeline drains. */ + private void emitDrainResult(long triggerTimestamp, Long currentMainSnapshotId) { + output.collect( + METADATA_STREAM, + new StreamRecord<>( + EqualityConvertPlan.noOp(currentMainSnapshotId, triggerTimestamp, nextPhaseTs))); + advancePhase(); + } + + /** + * Emits {@link ReadCommand}s for every data file on {@code mainSnapshot} so the worker indexes + * them for the configured equality-field set. Existing DVs attached to a data file are loaded by + * the reader and their positions are skipped. V2 positional deletes are not expected on main; the + * reader throws if it encounters one. Equality deletes attached to the scan task are skipped + * during indexing (they are processed via the planner's eq-delete read commands). + */ + private void emitMainDataReadCommands(Snapshot mainSnapshot) { + long commitSnapshotId = mainSnapshot.snapshotId(); + + try (CloseableIterable tasks = + table.newScan().useSnapshot(commitSnapshotId).planFiles()) { + for (FileScanTask task : tasks) { + output.collect( + new StreamRecord<>( + ReadCommand.dataFile( + task, indexSnapshotId, indexedSequenceNumber, dataSequenceNumber(task.file())), + nextPhaseTs)); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to plan files for main index", e); + } + + LOG.info( + "Emitted main data read commands for field IDs {} from snapshot {}.", + eqFieldIds, + commitSnapshotId); + + advancePhase(); + } + + /** + * Emits a phase-end watermark and bumps the phase timestamp. Every phase-emitting method must + * call this exactly once after its records; the worker uses these watermarks to gate keyed-state + * transitions. Missing or extra calls silently break ordering. + */ + private void advancePhase() { + output.emitWatermark(new Watermark(nextPhaseTs)); + nextPhaseTs++; + } + + /** + * Returns {@code true} if {@code snapshot} can be skipped: + * + *
    + *
  • It was already committed by us (carries {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}), OR + *
  • it adds no data or delete files (e.g. delete-file-only removals), OR + *
  • {@code stagingBranch == targetBranch} and the snapshot adds no equality-delete files. + * Pure-insert (and data-file-only) commits on the shared branch don't need a conversion + * cycle: their data is already on target, and the worker's index stays fresh via {@link + * #ensureIndexCurrent} when the next eq-delete arrives. + *
+ * + *

Filter checks read per-snapshot counts from {@link Snapshot#summary()}; we don't parse + * manifests here. Manifest parsing happens later in {@link #retrieveStagingFiles} only for the + * chosen snapshot. + */ + private boolean shouldSkip(Snapshot snapshot) { + Map summary = snapshot.summary(); + if (summary.containsKey(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) { + return true; + } + + long addedDataFiles = summaryCount(summary, SnapshotSummary.ADDED_FILES_PROP); + long addedDeleteFiles = summaryCount(summary, SnapshotSummary.ADDED_DELETE_FILES_PROP); + if (addedDataFiles == 0 && addedDeleteFiles == 0) { + LOG.info( + "Skipping staging snapshot {}: no added data or delete files.", snapshot.snapshotId()); + return true; + } + + if (stagingOnTargetBranch + && summaryCount(summary, SnapshotSummary.ADD_EQ_DELETE_FILES_PROP) == 0) { + return true; + } + + return false; + } + + private static long summaryCount(Map summary, String key) { + String value = summary.get(key); + return value == null ? 0L : Long.parseLong(value); + } + + /** + * Returns the id of the newest snapshot that is reachable from both the staging and target branch + * heads (i.e. the most recent common ancestor where the two branches last matched), or {@code + * null} if they share no history. Used on cold start to know where staging diverged from target + * so we can skip converting snapshots that already exist on target. + */ + private Long findCommonAncestor(Snapshot stagingHead, Snapshot mainHead) { + if (mainHead == null) { + return null; + } + + Set stagingSeen = Sets.newHashSet(); + Set mainSeen = Sets.newHashSet(); + Snapshot stagingCurrent = stagingHead; + Snapshot mainCurrent = mainHead; + + while (stagingCurrent != null || mainCurrent != null) { + if (stagingCurrent != null) { + long id = stagingCurrent.snapshotId(); + if (mainSeen.contains(id)) { + return id; + } + + stagingSeen.add(id); + stagingCurrent = parentOf(stagingCurrent); + } + + if (mainCurrent != null) { + long id = mainCurrent.snapshotId(); + if (stagingSeen.contains(id)) { + return id; + } + + mainSeen.add(id); + mainCurrent = parentOf(mainCurrent); + } + } + + return null; + } + + /** Returns the parent snapshot, or {@code null} if {@code snapshot} has no parent. */ + private Snapshot parentOf(Snapshot snapshot) { + Long parentId = snapshot.parentId(); + return parentId != null ? table.snapshot(parentId) : null; + } + + private static long dataSequenceNumber(ContentFile file) { + Long sequenceNumber = file.dataSequenceNumber(); + Preconditions.checkNotNull( + sequenceNumber, "Missing data sequence number for committed file %s", file.location()); + return sequenceNumber; + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + @VisibleForTesting + long reindexCount() { + return reindexCounter.getCount(); + } + + @VisibleForTesting + long skippedNoOpCycles() { + return skippedNoOpCyclesCounter.getCount(); + } + + @VisibleForTesting + long processedStagingSnapshotNum() { + return processedStagingSnapshotNumCounter.getCount(); + } + + @VisibleForTesting + long processedEqDeleteFileNum() { + return processedEqDeleteFileNumCounter.getCount(); + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java new file mode 100644 index 000000000000..82860a0de881 --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -0,0 +1,1115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestEqualityConvertPlanner extends OperatorTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void bootstrapsIndexFromBuilderEqualityFieldIds() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Staging branch exists but contains no eq-delete files yet. The planner still populates the + // worker index from main using the builder-configured equality field set so the index is ready + // when the first delete arrives. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isZero(); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Builder configured for [1, 2], but writer produces an eq delete with [1] only. + DeleteFile mismatched = writeIdOnlyEqualityDelete(table, 1); + table.newRowDelta().addDeletes(mismatched).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { + // When stagingBranch == targetBranch, the writer commits new data files directly to main. + // Bootstrap scans the main snapshot (which already includes those files) and indexes them. + // The planner must NOT also emit the staging-data phase for the same files — that would + // duplicate ADD_DATA_ROW commands for the same (PK, file, position) in the worker's index. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Writer commits (new-data id=3) + (eq-delete id=1) in one RowDelta directly to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(SnapshotRef.MAIN_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + + // Bootstrap from main emits one data-file command per file (id=1, id=2, id=3) = 3. + // Without the same-branch guard, emitSnapshotDataPhase would also emit newDataFile → 4. + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + long newDataFileCount = + commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .filter(c -> c.task().file().location().equals(newDataFile.location())) + .count(); + assertThat(newDataFileCount).isEqualTo(1); + } + } + + @Test + void emitsReadCommandsForEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 3 DATA_FILE (from main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).isEmpty(); + } + } + + @Test + void emitsDataFileFromInsertOnlyStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: eq delete targeting main data + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s1SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // S2: insert-only (no eq deletes), but its data must still be indexed for the configured + // equality fields + DataFile insertS2 = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(insertS2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: processes S1 (eq delete) + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + // Record S1's conversion on main, so the planner walks past S1 before processing S2. + simulateConvertCommit(table, s1SnapshotId); + + // Trigger 2: processes S2 (insert-only). Must emit its data file so the configured equality + // fields stay indexed. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + + Set dataFilePaths = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .map(TestEqualityConvertPlanner::filePath) + .collect(Collectors.toSet()); + + assertThat(dataFilePaths).contains(insertS2.location()); + } + } + + @Test + void includesNewDataFilesFromStaging() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 1 main data file + 1 eq delete + 1 staging data file + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).hasSize(1); + } + } + + @Test + void findsIntersectionWhenMainAdvancedAfterStagingFork() throws Exception { + Table table = createTableWithDelete(3); + // Pre-fork main: two snapshots. + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Fork staging from current main head. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Main advances with two more snapshots after the fork. These are on main's + // lineage but not on staging's. + insert(table, 3, "c"); + insert(table, 4, "d"); + + // Staging gets one new snapshot containing an equality delete. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Planner identifies the fork point and processes only the staging-only + // commit (the eq delete), and emits all four current main data files + // for the index refresh. + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(countDataFileTasks(commands)).isEqualTo(4); + } + } + + @Test + void skipsAlreadyProcessedStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + assertThat(firstTriggerCount).isGreaterThan(0); + assertThat(planner(harness).skippedNoOpCycles()).isZero(); + + // Simulate the committer committing to main with the staging snapshot property + // (the planner promotes pending only after confirming the commit landed on main). + simulateConvertCommit(table, stagingSnapshotId); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(firstTriggerCount); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + } + } + + @Test + void noOutputWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Bootstrap from main for the configured field set: 1 main DATA_FILE; no-op metadata still + // emitted because there's nothing on staging to convert. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(1); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isZero(); + } + } + + @Test + void propagatesStagingOnlyPositionalDeletes() throws Exception { + Table table = createTableWithDelete(3); + DataFile mainData = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(mainData).commit(); + table.refresh(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot has only a DV (no eq deletes, no new data files). Without the + // planner forwarding stagingDVFiles in this case, the committer would never + // see the DV and it would be lost. + DeleteFile stagingDV = writeStagingDV(table, mainData.location(), 0L); + table.newRowDelta().addDeletes(stagingDV).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(commands)).isZero(); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + EqualityConvertPlan result = metadata.get(0).getValue(); + assertThat(result.dataFiles()).isEmpty(); + assertThat(result.stagingDVFiles()).hasSize(1); + assertThat(result.stagingDVFiles().get(0).location()).isEqualTo(stagingDV.location()); + } + } + + @Test + void phaseTimestampsAreMonotonicallyIncreasing() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging: eq delete + new data file (triggers 3 phases: main data, eq delete, staging data) + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + long triggerTs = 100L; + sendTrigger(harness, triggerTs); + + List output = Lists.newArrayList(harness.getOutput()); + assertThat(((StreamRecord) output.get(0)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(1)).isEqualTo(new Watermark(triggerTs)); + + assertThat(((StreamRecord) output.get(2)).getValue().task()) + .isInstanceOf(EqualityDeleteFileScanTask.class); + assertThat(output.get(3)).isEqualTo(new Watermark(triggerTs + 1)); + + assertThat(((StreamRecord) output.get(4)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(5)).isEqualTo(new Watermark(triggerTs + 2)); + } + } + + @Test + void routesExceptionToErrorStream() throws Exception { + createTableWithDelete(3); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + dropTable(); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + sendTrigger(harness); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsOnRemovedDataFilesOnStagingBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + // Rewrite on the staging branch (not main). Equality delete conversion does not support + // rewrites on staging; the planner must fail the cycle instead of silently dropping the + // removed files. + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + // Bootstrap runs before processCycle's failure, so the main data commands are already on + // the wire. Only the cycle itself fails. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + } + } + + @Test + void reEmitsMainDataAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + + // Compact data files on main: rewrite 2 files into 1 + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // 1 DATA_FILE (compacted main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + // DATA_FILE should reference the compacted file + List dataCmds = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .collect(Collectors.toList()); + for (ReadCommand cmd : dataCmds) { + assertThat(cmd.task().file().location()).isEqualTo(compactedFile.location()); + } + } + } + + @Test + void refreshesIndexBeforeFirstCycleProcessesEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Stage MULTIPLE eq-delete snapshots before the first trigger arrives. The first trigger + // bootstraps the index from main for the configured equality fields before processing any eq + // deletes, then processes one snapshot per trigger (oldest first). + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s2SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + + // First trigger processes the OLDER staging snapshot (eqDelete1). Bootstrap built the index + // from main once: 2 main DATA_FILE + 1 EQ_DELETE_FILE = 3 commands. + assertThat(afterFirst).isEqualTo(3); + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + simulateConvertCommit(table, table.snapshot(STAGING_BRANCH).parentId()); + + // Second trigger processes the newer staging snapshot. The index is already up-to-date. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + assertThat(countDataFileTasks(trigger2Commands)).isZero(); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + simulateConvertCommit(table, s2SnapshotId); + } + } + + @Test + void noMainReEmitWhenUnchanged() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Only 1 EQ_DELETE_FILE, no main re-emission + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(2); + } + } + + @Test + void noMainReEmitAfterOwnCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long indexedStagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + + // Record the converter's own commit (COMMITTED_STAGING_SNAPSHOT property) for the staging + // snapshot the first trigger processed. The next trigger reads the property off main and + // advances lastStagingSnapshotId. No reindex should be triggered. + simulateConvertCommit(table, indexedStagingSnapshotId); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Own commits don't trigger index rebuild. + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + } + } + + @Test + void reIndexesAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 1 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(2); + assertThat(planner(harness).reindexCount()).isZero(); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + // External commit: no COMMITTED_STAGING_SNAPSHOT_PROPERTY + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // External commit triggers re-index: main data files are re-emitted + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(2); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(2); + } + } + + @Test + void emitsClearIndexBroadcastOnReindex() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1 bootstraps the index. Bootstrap does not broadcast CLEAR_INDEX. + sendTrigger(harness); + assertThat(harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM)) + .isNullOrEmpty(); + + // External commit advances main. The next trigger reindexes and must broadcast CLEAR_INDEX + // so workers evict ghost keys (e.g. a PK whose data file was removed by external CoW). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + long mainAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).snapshotId(); + long mainSeqAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).sequenceNumber(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + + List clears = + harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM).stream() + .map(StreamRecord::getValue) + .collect(Collectors.toList()); + assertThat(clears).hasSize(1); + assertThat(clears.get(0).type()).isEqualTo(IndexCommand.Type.CLEAR_INDEX); + assertThat(clears.get(0).mainSnapshotId()).isEqualTo(mainAfterExternal); + assertThat(clears.get(0).mainSequenceNumber()).isEqualTo(mainSeqAfterExternal); + } + } + + @Test + void detectsMainBranchChangeWithoutNewStagingSnapshots() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: build the index and process the existing eq delete. + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(afterFirst).isGreaterThan(0); + + // External commit on main (no COMMITTED_STAGING_SNAPSHOT_PROPERTY). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + // Trigger 2: nothing new on staging, but the planner must still notice the + // external main change and immediately re-emit main data for index rebuild. + sendTrigger(harness); + List allAfterTrigger2 = harness.extractOutputValues(); + List trigger2Commands = + allAfterTrigger2.subList(afterFirst, allAfterTrigger2.size()); + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + + // A subsequent staging eq delete should NOT re-emit main data because the index + // was already rebuilt in trigger 2. + int afterSecond = allAfterTrigger2.size(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 10, "z"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List all = harness.extractOutputValues(); + List trigger3 = all.subList(afterSecond, all.size()); + assertThat(countDataFileTasks(trigger3)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger3)).isEqualTo(1); + } + } + + @Test + void stateRestoredAfterCheckpoint() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // First trigger bootstraps the index and processes the staging snapshot. + sendTrigger(harness); + int commandCount = harness.extractOutputValues().size(); + assertThat(commandCount).isGreaterThan(0); + + // Record S1's conversion on main (committer marker) so the planner advances + // lastStagingSnapshotId on the second trigger. + simulateConvertCommit(table, stagingSnapshotId); + + // Second trigger reads the marker and advances lastStagingSnapshotId. + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(commandCount); + + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // Restore from checkpoint: the restored index is not re-indexed because its state matches the + // snapshot COMMITTED_STAGING_SNAPSHOT_PROPERTY property. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void failsOnV2PosDeleteOnStagingBranch() throws Exception { + Table table = createTableWithDelete(2); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Flink-written staging branches are V3 and only produce DVs for deletes. A V2 positional + // delete on staging is a writer-side misconfiguration and should fail the cycle rather than + // be silently absorbed. + DataFile dataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile posDelete = writePosDeleteFile(table, dataFile.location(), 0); + table.newRowDelta().addRows(dataFile).addDeletes(posDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void skipsCommittedSnapshotAfterCommitLandsButCheckpointDoesNot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // Capture the planner's checkpoint before any cycle has run. + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // The staging commit landed on main but lastStagingSnapshotId was not checkpointed. + simulateConvertCommit(table, stagingSnapshotId); + + // On restore, the planner walks main, finds the COMMITTED_STAGING_SNAPSHOT_PROPERTY, + // advances lastStagingSnapshotId, and skips re-emitting the already-committed snapshot. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(0); + // Bootstrap from main creates the index from the main data files on the first trigger even + // though the staging snapshot itself is already committed and skipped. Main now has the + // original insert plus simulateConvertCommit's marker file = 2 data files. + assertThat(countDataFileTasks(commands)).isEqualTo(2); + } + } + + @Test + void failsWhenCommitMarkerDisappears() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + long startSnapshotId = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + simulateConvertCommit(table, stagingSnapshotId); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + table.manageSnapshots().rollbackTo(startSnapshotId).commit(); + table.refresh(); + + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + assertThat( + harness + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .poll() + .getValue() + .getMessage()) + .contains("No COMMITTED_STAGING_SNAPSHOT marker reachable"); + } + } + + @Test + void failsOnEqFieldIdsChangeAcrossRestart() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + assertThatThrownBy(() -> harness.initializeState(state)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Equality field IDs changed across restart"); + } + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch) throws Exception { + // Default matches the [1, 2] equality-field-id list produced by writeEqualityDelete. + return createHarness(stagingBranch, Lists.newArrayList(1, 2)); + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch, List eqFieldIds) throws Exception { + return new OneInputStreamOperatorTestHarness<>( + new EqualityConvertPlanner( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + stagingBranch, + SnapshotRef.MAIN_BRANCH, + Sets.newHashSet(eqFieldIds))); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness) + throws Exception { + sendTrigger(harness, System.currentTimeMillis()); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness, long time) + throws Exception { + harness.processElement(new StreamRecord<>(Trigger.create(time, 0), time)); + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(record), + idOnly); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private static long countDataFileTasks(List commands) { + return commands.stream().filter(c -> c.task() instanceof FileScanTask).count(); + } + + private static long countEqDeleteTasks(List commands) { + return commands.stream().filter(TestEqualityConvertPlanner::isEqDelete).count(); + } + + private static EqualityConvertPlanner planner( + OneInputStreamOperatorTestHarness harness) { + return (EqualityConvertPlanner) harness.getOperator(); + } + + private static boolean isEqDelete(ReadCommand cmd) { + return cmd.task() instanceof EqualityDeleteFileScanTask; + } + + private static String filePath(ReadCommand cmd) { + return cmd.task().file().location(); + } + + private void simulateConvertCommit(Table table, long stagingSnapshotId) throws IOException { + DataFile dummy = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(-1, "marker" + stagingSnapshotId))); + table + .newRowDelta() + .addRows(dummy) + .set( + EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY, + String.valueOf(stagingSnapshotId)) + .commit(); + table.refresh(); + } +} From 6174dfa1e1825601384874d5a693b9219cf459e6 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 24 Jun 2026 19:22:26 +0200 Subject: [PATCH 36/73] Revert "Core: Migrate switch statements to switch expressions (#16881)" (#16953) This reverts commit 534c3fd9427b7037b5fa683d51b4dd5fc255c472. --- .../iceberg/BaseDistributedDataScan.java | 23 +- .../java/org/apache/iceberg/BaseFile.java | 168 ++++++++----- .../iceberg/BaseIncrementalChangelogScan.java | 46 ++-- .../iceberg/BasePartitionStatistics.java | 101 +++++--- .../java/org/apache/iceberg/BaseScan.java | 12 +- .../java/org/apache/iceberg/BaseSnapshot.java | 25 +- .../org/apache/iceberg/BaseTransaction.java | 19 +- .../java/org/apache/iceberg/CatalogUtil.java | 37 ++- .../org/apache/iceberg/ContentFileParser.java | 18 +- .../org/apache/iceberg/DeleteFileIndex.java | 12 +- .../apache/iceberg/DeletionVectorStruct.java | 41 ++-- .../java/org/apache/iceberg/FileMetadata.java | 14 +- .../apache/iceberg/GenericManifestEntry.java | 45 ++-- .../apache/iceberg/GenericManifestFile.java | 133 +++++++---- .../iceberg/GenericPartitionFieldSummary.java | 38 ++- .../org/apache/iceberg/ManifestFiles.java | 74 +++--- .../apache/iceberg/ManifestInfoStruct.java | 87 ++++--- .../org/apache/iceberg/ManifestLists.java | 52 ++-- .../org/apache/iceberg/ManifestWriter.java | 12 +- .../iceberg/MergingSnapshotProducer.java | 31 ++- .../apache/iceberg/MetadataTableUtils.java | 56 +++-- .../apache/iceberg/MetadataUpdateParser.java | 224 +++++++++++------- .../org/apache/iceberg/PartitionData.java | 21 +- .../apache/iceberg/PartitionStatsHandler.java | 36 +-- .../org/apache/iceberg/PartitionsTable.java | 18 +- .../apache/iceberg/RewriteTablePathUtil.java | 27 ++- .../java/org/apache/iceberg/ScanSummary.java | 26 +- .../org/apache/iceberg/ScanTaskParser.java | 21 +- .../java/org/apache/iceberg/SchemaParser.java | 15 +- .../java/org/apache/iceberg/SchemaUpdate.java | 18 +- .../org/apache/iceberg/SingleValueParser.java | 164 ++++++------- .../org/apache/iceberg/SnapshotChanges.java | 16 +- .../org/apache/iceberg/SnapshotProducer.java | 12 +- .../org/apache/iceberg/SnapshotSummary.java | 44 ++-- .../org/apache/iceberg/SortOrderParser.java | 14 +- .../org/apache/iceberg/TrackedFileStruct.java | 125 +++++++--- .../org/apache/iceberg/TrackingStruct.java | 73 ++++-- .../iceberg/UpdateRequirementParser.java | 103 ++++---- .../java/org/apache/iceberg/V1Metadata.java | 114 +++++---- .../java/org/apache/iceberg/V2Metadata.java | 151 +++++++----- .../java/org/apache/iceberg/V3Metadata.java | 181 ++++++++------ .../java/org/apache/iceberg/V4Metadata.java | 177 ++++++++------ .../iceberg/actions/RewriteFileGroup.java | 22 +- .../actions/RewritePositionDeletesGroup.java | 25 +- .../java/org/apache/iceberg/avro/Avro.java | 31 ++- .../avro/AvroCustomOrderSchemaVisitor.java | 29 ++- .../apache/iceberg/avro/AvroFormatModel.java | 15 +- .../apache/iceberg/avro/AvroSchemaUtil.java | 15 +- .../iceberg/avro/AvroSchemaVisitor.java | 33 +-- .../avro/AvroSchemaWithTypeVisitor.java | 25 +- .../AvroWithPartnerByStructureVisitor.java | 29 ++- .../iceberg/avro/AvroWithPartnerVisitor.java | 42 ++-- .../apache/iceberg/avro/BaseWriteBuilder.java | 67 ++++-- .../iceberg/avro/BuildAvroProjection.java | 23 +- .../iceberg/avro/GenericAvroReader.java | 88 ++++--- .../apache/iceberg/avro/InternalReader.java | 86 ++++--- .../org/apache/iceberg/avro/SchemaToType.java | 36 ++- .../org/apache/iceberg/avro/TypeToSchema.java | 64 +++-- .../org/apache/iceberg/avro/ValueReaders.java | 16 +- .../apache/iceberg/data/GenericDataUtil.java | 30 +-- .../data/IdentityPartitionConverters.java | 30 +-- .../apache/iceberg/data/avro/DataReader.java | 89 ++++--- .../apache/iceberg/data/avro/DataWriter.java | 75 +++--- .../iceberg/data/avro/PlannedDataReader.java | 97 ++++---- .../iceberg/deletes/DeleteGranularity.java | 12 +- .../iceberg/deletes/PositionDelete.java | 31 ++- .../SortingPositionOnlyDeleteWriter.java | 13 +- .../encryption/StandardKeyMetadata.java | 31 ++- .../iceberg/expressions/ExpressionParser.java | 74 +++--- .../iceberg/hadoop/HadoopMetricsContext.java | 52 ++-- .../io/ClusteredPositionDeleteWriter.java | 12 +- .../iceberg/metrics/TimerResultParser.java | 30 ++- .../apache/iceberg/puffin/PuffinFormat.java | 41 ++-- .../apache/iceberg/puffin/PuffinReader.java | 12 +- .../apache/iceberg/rest/CatalogHandlers.java | 27 ++- .../apache/iceberg/rest/ErrorHandlers.java | 107 +++++---- .../iceberg/rest/RESTTableOperations.java | 21 +- .../apache/iceberg/rest/RESTTableScan.java | 55 +++-- .../iceberg/rest/auth/AuthManagers.java | 29 ++- .../apache/iceberg/rest/auth/OAuth2Util.java | 22 +- .../schema/SchemaWithPartnerVisitor.java | 29 ++- .../iceberg/variants/PrimitiveWrapper.java | 169 +++++++------ .../iceberg/variants/VariantVisitor.java | 19 +- .../view/ViewRepresentationParser.java | 25 +- 84 files changed, 2636 insertions(+), 1766 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java b/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java index 94c3b02e56a3..027a3d2298f0 100644 --- a/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java @@ -240,14 +240,21 @@ private boolean shouldPlanLocally(PlanningMode mode, List manifest return true; } - return switch (mode) { - case LOCAL -> true; - case DISTRIBUTED -> manifests.isEmpty(); - case AUTO -> - remoteParallelism() <= localParallelism - || manifests.size() <= 2 * localParallelism - || totalSize(manifests) <= localPlanningSizeThreshold; - }; + switch (mode) { + case LOCAL: + return true; + + case DISTRIBUTED: + return manifests.isEmpty(); + + case AUTO: + return remoteParallelism() <= localParallelism + || manifests.size() <= 2 * localParallelism + || totalSize(manifests) <= localPlanningSizeThreshold; + + default: + throw new IllegalArgumentException("Unknown planning mode: " + mode); + } } private long totalSize(List manifests) { diff --git a/core/src/main/java/org/apache/iceberg/BaseFile.java b/core/src/main/java/org/apache/iceberg/BaseFile.java index e42989ea1f31..7147ba58787b 100644 --- a/core/src/main/java/org/apache/iceberg/BaseFile.java +++ b/core/src/main/java/org/apache/iceberg/BaseFile.java @@ -316,40 +316,79 @@ public void put(int i, Object value) { @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0 -> - this.content = value != null ? FileContent.fromId((Integer) value) : FileContent.DATA; - case 1 -> this.filePath = value.toString(); // always coerce to String for Serializable - case 2 -> this.format = FileFormat.fromString(value.toString()); - case 3 -> this.partitionSpecId = (value != null) ? (Integer) value : -1; - case 4 -> { + case 0: + this.content = value != null ? FileContent.fromId((Integer) value) : FileContent.DATA; + return; + case 1: + // always coerce to String for Serializable + this.filePath = value.toString(); + return; + case 2: + this.format = FileFormat.fromString(value.toString()); + return; + case 3: + this.partitionSpecId = (value != null) ? (Integer) value : -1; + return; + case 4: // Preserve the constructor-initialized partitionData when the reader returns null // (e.g., v4 Parquet manifests for unpartitioned tables omit the partition field). if (value != null) { this.partitionData = (PartitionData) value; } - } - case 5 -> this.recordCount = (Long) value; - case 6 -> this.fileSizeInBytes = (Long) value; - case 7 -> this.columnSizes = (Map) value; - case 8 -> this.valueCounts = (Map) value; - case 9 -> this.nullValueCounts = (Map) value; - case 10 -> this.nanValueCounts = (Map) value; - case 11 -> - this.lowerBounds = SerializableByteBufferMap.wrap((Map) value); - case 12 -> - this.upperBounds = SerializableByteBufferMap.wrap((Map) value); - case 13 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); - case 14 -> this.splitOffsets = ArrayUtil.toLongArray((List) value); - case 15 -> this.equalityIds = ArrayUtil.toIntArray((List) value); - case 16 -> this.sortOrderId = (Integer) value; - case 17 -> this.firstRowId = (Long) value; - case 18 -> this.referencedDataFile = value != null ? value.toString() : null; - case 19 -> this.contentOffset = (Long) value; - case 20 -> this.contentSizeInBytes = (Long) value; - case 21 -> this.fileOrdinal = (long) value; - default -> { + return; + case 5: + this.recordCount = (Long) value; + return; + case 6: + this.fileSizeInBytes = (Long) value; + return; + case 7: + this.columnSizes = (Map) value; + return; + case 8: + this.valueCounts = (Map) value; + return; + case 9: + this.nullValueCounts = (Map) value; + return; + case 10: + this.nanValueCounts = (Map) value; + return; + case 11: + this.lowerBounds = SerializableByteBufferMap.wrap((Map) value); + return; + case 12: + this.upperBounds = SerializableByteBufferMap.wrap((Map) value); + return; + case 13: + this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + return; + case 14: + this.splitOffsets = ArrayUtil.toLongArray((List) value); + return; + case 15: + this.equalityIds = ArrayUtil.toIntArray((List) value); + return; + case 16: + this.sortOrderId = (Integer) value; + return; + case 17: + this.firstRowId = (Long) value; + return; + case 18: + this.referencedDataFile = value != null ? value.toString() : null; + return; + case 19: + this.contentOffset = (Long) value; + return; + case 20: + this.contentSizeInBytes = (Long) value; + return; + case 21: + this.fileOrdinal = (long) value; + return; + default: // ignore the object, it must be from a newer version of the format - } } } @@ -359,31 +398,54 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int basePos) { - return switch (basePos) { - case 0 -> content.id(); - case 1 -> filePath; - case 2 -> format != null ? format.toString() : null; - case 3 -> partitionSpecId; - case 4 -> partitionData; - case 5 -> recordCount; - case 6 -> fileSizeInBytes; - case 7 -> columnSizes; - case 8 -> valueCounts; - case 9 -> nullValueCounts; - case 10 -> nanValueCounts; - case 11 -> lowerBounds; - case 12 -> upperBounds; - case 13 -> keyMetadata(); - case 14 -> splitOffsets(); - case 15 -> equalityFieldIds(); - case 16 -> sortOrderId; - case 17 -> firstRowId; - case 18 -> referencedDataFile; - case 19 -> contentOffset; - case 20 -> contentSizeInBytes; - case 21 -> fileOrdinal; - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + basePos); - }; + switch (basePos) { + case 0: + return content.id(); + case 1: + return filePath; + case 2: + return format != null ? format.toString() : null; + case 3: + return partitionSpecId; + case 4: + return partitionData; + case 5: + return recordCount; + case 6: + return fileSizeInBytes; + case 7: + return columnSizes; + case 8: + return valueCounts; + case 9: + return nullValueCounts; + case 10: + return nanValueCounts; + case 11: + return lowerBounds; + case 12: + return upperBounds; + case 13: + return keyMetadata(); + case 14: + return splitOffsets(); + case 15: + return equalityFieldIds(); + case 16: + return sortOrderId; + case 17: + return firstRowId; + case 18: + return referencedDataFile; + case 19: + return contentOffset; + case 20: + return contentSizeInBytes; + case 21: + return fileOrdinal; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + basePos); + } } @Override diff --git a/core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java b/core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java index ee16ae6d2a18..2d54a94e8d73 100644 --- a/core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java @@ -153,28 +153,30 @@ public CloseableIterable apply( int changeOrdinal = snapshotOrdinals.get(commitSnapshotId); DataFile dataFile = entry.file().copy(context.shouldKeepStats()); - return switch (entry.status()) { - case ADDED -> - new BaseAddedRowsScanTask( - changeOrdinal, - commitSnapshotId, - dataFile, - NO_DELETES, - context.schemaAsString(), - context.specAsString(), - context.residuals()); - case DELETED -> - new BaseDeletedDataFileScanTask( - changeOrdinal, - commitSnapshotId, - dataFile, - NO_DELETES, - context.schemaAsString(), - context.specAsString(), - context.residuals()); - default -> - throw new IllegalArgumentException("Unexpected entry status: " + entry.status()); - }; + switch (entry.status()) { + case ADDED: + return new BaseAddedRowsScanTask( + changeOrdinal, + commitSnapshotId, + dataFile, + NO_DELETES, + context.schemaAsString(), + context.specAsString(), + context.residuals()); + + case DELETED: + return new BaseDeletedDataFileScanTask( + changeOrdinal, + commitSnapshotId, + dataFile, + NO_DELETES, + context.schemaAsString(), + context.specAsString(), + context.residuals()); + + default: + throw new IllegalArgumentException("Unexpected entry status: " + entry.status()); + } }); } } diff --git a/core/src/main/java/org/apache/iceberg/BasePartitionStatistics.java b/core/src/main/java/org/apache/iceberg/BasePartitionStatistics.java index 56b020bb9b55..1cdb4a49e341 100644 --- a/core/src/main/java/org/apache/iceberg/BasePartitionStatistics.java +++ b/core/src/main/java/org/apache/iceberg/BasePartitionStatistics.java @@ -145,22 +145,36 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - return switch (pos) { - case PARTITION_POSITION -> partition; - case SPEC_ID_POSITION -> specId; - case DATA_RECORD_COUNT_POSITION -> dataRecordCount; - case DATA_FILE_COUNT_POSITION -> dataFileCount; - case TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION -> totalDataFileSizeInBytes; - case POSITION_DELETE_RECORD_COUNT_POSITION -> positionDeleteRecordCount; - case POSITION_DELETE_FILE_COUNT_POSITION -> positionDeleteFileCount; - case EQUALITY_DELETE_RECORD_COUNT_POSITION -> equalityDeleteRecordCount; - case EQUALITY_DELETE_FILE_COUNT_POSITION -> equalityDeleteFileCount; - case TOTAL_RECORD_COUNT_POSITION -> totalRecordCount; - case LAST_UPDATED_AT_POSITION -> lastUpdatedAt; - case LAST_UPDATED_SNAPSHOT_ID_POSITION -> lastUpdatedSnapshotId; - case DV_COUNT_POSITION -> dvCount; - default -> throw new UnsupportedOperationException("Unknown position: " + pos); - }; + switch (pos) { + case PARTITION_POSITION: + return partition; + case SPEC_ID_POSITION: + return specId; + case DATA_RECORD_COUNT_POSITION: + return dataRecordCount; + case DATA_FILE_COUNT_POSITION: + return dataFileCount; + case TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION: + return totalDataFileSizeInBytes; + case POSITION_DELETE_RECORD_COUNT_POSITION: + return positionDeleteRecordCount; + case POSITION_DELETE_FILE_COUNT_POSITION: + return positionDeleteFileCount; + case EQUALITY_DELETE_RECORD_COUNT_POSITION: + return equalityDeleteRecordCount; + case EQUALITY_DELETE_FILE_COUNT_POSITION: + return equalityDeleteFileCount; + case TOTAL_RECORD_COUNT_POSITION: + return totalRecordCount; + case LAST_UPDATED_AT_POSITION: + return lastUpdatedAt; + case LAST_UPDATED_SNAPSHOT_ID_POSITION: + return lastUpdatedSnapshotId; + case DV_COUNT_POSITION: + return dvCount; + default: + throw new UnsupportedOperationException("Unknown position: " + pos); + } } @Override @@ -170,20 +184,47 @@ protected void internalSet(int pos, T value) { } switch (pos) { - case PARTITION_POSITION -> this.partition = (StructLike) value; - case SPEC_ID_POSITION -> this.specId = (int) value; - case DATA_RECORD_COUNT_POSITION -> this.dataRecordCount = (long) value; - case DATA_FILE_COUNT_POSITION -> this.dataFileCount = (int) value; - case TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION -> this.totalDataFileSizeInBytes = (long) value; - case POSITION_DELETE_RECORD_COUNT_POSITION -> this.positionDeleteRecordCount = (long) value; - case POSITION_DELETE_FILE_COUNT_POSITION -> this.positionDeleteFileCount = (int) value; - case EQUALITY_DELETE_RECORD_COUNT_POSITION -> this.equalityDeleteRecordCount = (long) value; - case EQUALITY_DELETE_FILE_COUNT_POSITION -> this.equalityDeleteFileCount = (int) value; - case TOTAL_RECORD_COUNT_POSITION -> this.totalRecordCount = (Long) value; - case LAST_UPDATED_AT_POSITION -> this.lastUpdatedAt = (Long) value; - case LAST_UPDATED_SNAPSHOT_ID_POSITION -> this.lastUpdatedSnapshotId = (Long) value; - case DV_COUNT_POSITION -> this.dvCount = (int) value; - default -> throw new UnsupportedOperationException("Unknown position: " + pos); + case PARTITION_POSITION: + this.partition = (StructLike) value; + break; + case SPEC_ID_POSITION: + this.specId = (int) value; + break; + case DATA_RECORD_COUNT_POSITION: + this.dataRecordCount = (long) value; + break; + case DATA_FILE_COUNT_POSITION: + this.dataFileCount = (int) value; + break; + case TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION: + this.totalDataFileSizeInBytes = (long) value; + break; + case POSITION_DELETE_RECORD_COUNT_POSITION: + this.positionDeleteRecordCount = (long) value; + break; + case POSITION_DELETE_FILE_COUNT_POSITION: + this.positionDeleteFileCount = (int) value; + break; + case EQUALITY_DELETE_RECORD_COUNT_POSITION: + this.equalityDeleteRecordCount = (long) value; + break; + case EQUALITY_DELETE_FILE_COUNT_POSITION: + this.equalityDeleteFileCount = (int) value; + break; + case TOTAL_RECORD_COUNT_POSITION: + this.totalRecordCount = (Long) value; + break; + case LAST_UPDATED_AT_POSITION: + this.lastUpdatedAt = (Long) value; + break; + case LAST_UPDATED_SNAPSHOT_ID_POSITION: + this.lastUpdatedSnapshotId = (Long) value; + break; + case DV_COUNT_POSITION: + this.dvCount = (int) value; + break; + default: + throw new UnsupportedOperationException("Unknown position: " + pos); } } } diff --git a/core/src/main/java/org/apache/iceberg/BaseScan.java b/core/src/main/java/org/apache/iceberg/BaseScan.java index 82fb5a3fdc2a..242a5aaacc09 100644 --- a/core/src/main/java/org/apache/iceberg/BaseScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseScan.java @@ -319,9 +319,13 @@ public ThisT minRowsRequested(long numRows) { * @return a list of column names corresponding to the specified manifest content type. */ static List scanColumns(ManifestContent content) { - return switch (content) { - case DATA -> BaseScan.SCAN_COLUMNS; - case DELETES -> BaseScan.DELETE_SCAN_COLUMNS; - }; + switch (content) { + case DATA: + return BaseScan.SCAN_COLUMNS; + case DELETES: + return BaseScan.DELETE_SCAN_COLUMNS; + default: + throw new UnsupportedOperationException("Cannot read unknown manifest type: " + content); + } } } diff --git a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java index d095fd2f1249..b8ea6db22938 100644 --- a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java +++ b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java @@ -276,11 +276,14 @@ private void cacheDeleteFileChanges(FileIO fileIO) { ManifestFiles.readDeleteManifest(manifest, fileIO, null)) { for (ManifestEntry entry : reader.entries()) { switch (entry.status()) { - case ADDED -> adds.add(entry.file().copy()); - case DELETED -> deletes.add(entry.file().copyWithoutStats()); - default -> { + case ADDED: + adds.add(entry.file().copy()); + break; + case DELETED: + deletes.add(entry.file().copyWithoutStats()); + break; + default: // ignore existing - } } } } catch (IOException e) { @@ -306,11 +309,15 @@ private void cacheDataFileChanges(FileIO fileIO) { new ManifestGroup(fileIO, changedManifests).ignoreExisting().entries()) { for (ManifestEntry entry : entries) { switch (entry.status()) { - case ADDED -> adds.add(entry.file().copy()); - case DELETED -> deletes.add(entry.file().copyWithoutStats()); - default -> - throw new IllegalStateException( - "Unexpected entry status, not added or deleted: " + entry); + case ADDED: + adds.add(entry.file().copy()); + break; + case DELETED: + deletes.add(entry.file().copyWithoutStats()); + break; + default: + throw new IllegalStateException( + "Unexpected entry status, not added or deleted: " + entry); } } } catch (IOException e) { diff --git a/core/src/main/java/org/apache/iceberg/BaseTransaction.java b/core/src/main/java/org/apache/iceberg/BaseTransaction.java index 2227676a7968..9884ac297079 100644 --- a/core/src/main/java/org/apache/iceberg/BaseTransaction.java +++ b/core/src/main/java/org/apache/iceberg/BaseTransaction.java @@ -253,10 +253,21 @@ public void commitTransaction() { hasLastOpCommitted, "Cannot commit transaction: last operation has not committed"); switch (type) { - case CREATE_TABLE -> commitCreateTransaction(); - case REPLACE_TABLE -> commitReplaceTransaction(false); - case CREATE_OR_REPLACE_TABLE -> commitReplaceTransaction(true); - case SIMPLE -> commitSimpleTransaction(); + case CREATE_TABLE: + commitCreateTransaction(); + break; + + case REPLACE_TABLE: + commitReplaceTransaction(false); + break; + + case CREATE_OR_REPLACE_TABLE: + commitReplaceTransaction(true); + break; + + case SIMPLE: + commitSimpleTransaction(); + break; } } diff --git a/core/src/main/java/org/apache/iceberg/CatalogUtil.java b/core/src/main/java/org/apache/iceberg/CatalogUtil.java index e48c4a7dc905..2b400ccebc8b 100644 --- a/core/src/main/java/org/apache/iceberg/CatalogUtil.java +++ b/core/src/main/java/org/apache/iceberg/CatalogUtil.java @@ -314,18 +314,31 @@ public static Catalog buildIcebergCatalog(String name, Map optio if (catalogImpl == null) { String catalogType = PropertyUtil.propertyAsString(options, ICEBERG_CATALOG_TYPE, ICEBERG_CATALOG_TYPE_HIVE); - catalogImpl = - switch (catalogType.toLowerCase(Locale.ENGLISH)) { - case ICEBERG_CATALOG_TYPE_HIVE -> ICEBERG_CATALOG_HIVE; - case ICEBERG_CATALOG_TYPE_HADOOP -> ICEBERG_CATALOG_HADOOP; - case ICEBERG_CATALOG_TYPE_REST -> ICEBERG_CATALOG_REST; - case ICEBERG_CATALOG_TYPE_GLUE -> ICEBERG_CATALOG_GLUE; - case ICEBERG_CATALOG_TYPE_NESSIE -> ICEBERG_CATALOG_NESSIE; - case ICEBERG_CATALOG_TYPE_JDBC -> ICEBERG_CATALOG_JDBC; - case ICEBERG_CATALOG_TYPE_BIGQUERY -> ICEBERG_CATALOG_BIGQUERY; - default -> - throw new UnsupportedOperationException("Unknown catalog type: " + catalogType); - }; + switch (catalogType.toLowerCase(Locale.ENGLISH)) { + case ICEBERG_CATALOG_TYPE_HIVE: + catalogImpl = ICEBERG_CATALOG_HIVE; + break; + case ICEBERG_CATALOG_TYPE_HADOOP: + catalogImpl = ICEBERG_CATALOG_HADOOP; + break; + case ICEBERG_CATALOG_TYPE_REST: + catalogImpl = ICEBERG_CATALOG_REST; + break; + case ICEBERG_CATALOG_TYPE_GLUE: + catalogImpl = ICEBERG_CATALOG_GLUE; + break; + case ICEBERG_CATALOG_TYPE_NESSIE: + catalogImpl = ICEBERG_CATALOG_NESSIE; + break; + case ICEBERG_CATALOG_TYPE_JDBC: + catalogImpl = ICEBERG_CATALOG_JDBC; + break; + case ICEBERG_CATALOG_TYPE_BIGQUERY: + catalogImpl = ICEBERG_CATALOG_BIGQUERY; + break; + default: + throw new UnsupportedOperationException("Unknown catalog type: " + catalogType); + } } else { String catalogType = options.get(ICEBERG_CATALOG_TYPE); Preconditions.checkArgument( diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java b/core/src/main/java/org/apache/iceberg/ContentFileParser.java index d626407fb1b6..f024a24b18ce 100644 --- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java +++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java @@ -355,19 +355,21 @@ private static PartitionData partitionFromJson( } private static FileContent fileContentFromJson(String content) { - return switch (content) { - case CONTENT_DATA -> FileContent.DATA; - case CONTENT_POSITION_DELETES -> FileContent.POSITION_DELETES; - case CONTENT_EQUALITY_DELETES -> FileContent.EQUALITY_DELETES; - default -> { + switch (content) { + case CONTENT_DATA: + return FileContent.DATA; + case CONTENT_POSITION_DELETES: + return FileContent.POSITION_DELETES; + case CONTENT_EQUALITY_DELETES: + return FileContent.EQUALITY_DELETES; + default: // In 1.10 and before, file content is serialized as the FileContent enum value try { - yield FileContent.valueOf(content); + return FileContent.valueOf(content); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( String.format("Invalid file content value: '%s'", content), e); } - } - }; + } } } diff --git a/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java b/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java index 9d566860089a..872fcd212b8a 100644 --- a/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java +++ b/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java @@ -501,16 +501,18 @@ DeleteFileIndex build() { for (DeleteFile file : files) { switch (file.content()) { - case POSITION_DELETES -> { + case POSITION_DELETES: if (ContentFileUtil.isDV(file)) { add(dvByPath, file); } else { add(posDeletesByPath, posDeletesByPartition, file); } - } - case EQUALITY_DELETES -> add(globalDeletes, eqDeletesByPartition, file, fieldLookup); - default -> - throw new UnsupportedOperationException("Unsupported content: " + file.content()); + break; + case EQUALITY_DELETES: + add(globalDeletes, eqDeletesByPartition, file, fieldLookup); + break; + default: + throw new UnsupportedOperationException("Unsupported content: " + file.content()); } ScanMetricsUtil.indexedDeleteFile(scanMetrics, file); } diff --git a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java index 4708942a88b9..3f5be0756fad 100644 --- a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java +++ b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java @@ -90,27 +90,38 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - return switch (pos) { - case 0 -> location; - case 1 -> offset; - case 2 -> sizeInBytes; - case 3 -> cardinality; - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + switch (pos) { + case 0: + return location; + case 1: + return offset; + case 2: + return sizeInBytes; + case 3: + return cardinality; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0 -> - // always coerce to String for Serializable - this.location = value.toString(); - case 1 -> this.offset = (Long) value; - case 2 -> this.sizeInBytes = (Long) value; - case 3 -> this.cardinality = (Long) value; - default -> { + case 0: + // always coerce to String for Serializable + this.location = value.toString(); + break; + case 1: + this.offset = (Long) value; + break; + case 2: + this.sizeInBytes = (Long) value; + break; + case 3: + this.cardinality = (Long) value; + break; + default: // ignore the object, it must be from a newer version of the format - } } } diff --git a/core/src/main/java/org/apache/iceberg/FileMetadata.java b/core/src/main/java/org/apache/iceberg/FileMetadata.java index 4f9ac72df2c7..a5266101c252 100644 --- a/core/src/main/java/org/apache/iceberg/FileMetadata.java +++ b/core/src/main/java/org/apache/iceberg/FileMetadata.java @@ -272,15 +272,17 @@ public DeleteFile build() { } switch (content) { - case POSITION_DELETES -> - Preconditions.checkArgument( - sortOrderId == null, "Position delete file should not have sort order"); - case EQUALITY_DELETES -> { + case POSITION_DELETES: + Preconditions.checkArgument( + sortOrderId == null, "Position delete file should not have sort order"); + break; + case EQUALITY_DELETES: if (sortOrderId == null) { sortOrderId = SortOrder.unsorted().orderId(); } - } - default -> throw new IllegalStateException("Unknown content type " + content); + break; + default: + throw new IllegalStateException("Unknown content type " + content); } return new GenericDeleteFile( diff --git a/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java b/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java index 99f5e8ba7f5e..f154c982d1c7 100644 --- a/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java +++ b/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java @@ -157,14 +157,23 @@ public void setFileSequenceNumber(long newFileSequenceNumber) { @SuppressWarnings("unchecked") public void put(int i, Object v) { switch (i) { - case 0 -> this.status = Status.fromId((Integer) v); - case 1 -> this.snapshotId = (Long) v; - case 2 -> this.dataSequenceNumber = (Long) v; - case 3 -> this.fileSequenceNumber = (Long) v; - case 4 -> this.file = (F) v; - default -> { + case 0: + this.status = Status.fromId((Integer) v); + return; + case 1: + this.snapshotId = (Long) v; + return; + case 2: + this.dataSequenceNumber = (Long) v; + return; + case 3: + this.fileSequenceNumber = (Long) v; + return; + case 4: + this.file = (F) v; + return; + default: // ignore the object, it must be from a newer version of the format - } } } @@ -175,14 +184,20 @@ public void set(int pos, T value) { @Override public Object get(int i) { - return switch (i) { - case 0 -> status.id(); - case 1 -> snapshotId; - case 2 -> dataSequenceNumber; - case 3 -> fileSequenceNumber; - case 4 -> file; - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + i); - }; + switch (i) { + case 0: + return status.id(); + case 1: + return snapshotId; + case 2: + return dataSequenceNumber; + case 3: + return fileSequenceNumber; + case 4: + return file; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + i); + } } @Override diff --git a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java index 2131a60b8be6..9624484ffe0c 100644 --- a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java +++ b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java @@ -288,55 +288,102 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int basePos) { - return switch (basePos) { - case 0 -> manifestPath; - case 1 -> lazyLength(); - case 2 -> specId; - case 3 -> content.id(); - case 4 -> sequenceNumber; - case 5 -> minSequenceNumber; - case 6 -> snapshotId; - case 7 -> addedFilesCount; - case 8 -> existingFilesCount; - case 9 -> deletedFilesCount; - case 10 -> addedRowsCount; - case 11 -> existingRowsCount; - case 12 -> deletedRowsCount; - case 13 -> partitions(); - case 14 -> keyMetadata(); - case 15 -> firstRowId(); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + basePos); - }; + switch (basePos) { + case 0: + return manifestPath; + case 1: + return lazyLength(); + case 2: + return specId; + case 3: + return content.id(); + case 4: + return sequenceNumber; + case 5: + return minSequenceNumber; + case 6: + return snapshotId; + case 7: + return addedFilesCount; + case 8: + return existingFilesCount; + case 9: + return deletedFilesCount; + case 10: + return addedRowsCount; + case 11: + return existingRowsCount; + case 12: + return deletedRowsCount; + case 13: + return partitions(); + case 14: + return keyMetadata(); + case 15: + return firstRowId(); + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + basePos); + } } @Override protected void internalSet(int basePos, T value) { switch (basePos) { - case 0 -> this.manifestPath = value.toString(); // always coerce to String for Serializable - case 1 -> this.length = (Long) value; - case 2 -> this.specId = (Integer) value; - case 3 -> - this.content = - value != null ? ManifestContent.fromId((Integer) value) : ManifestContent.DATA; - case 4 -> this.sequenceNumber = value != null ? (Long) value : 0; - case 5 -> this.minSequenceNumber = value != null ? (Long) value : 0; - case 6 -> this.snapshotId = (Long) value; - case 7 -> this.addedFilesCount = (Integer) value; - case 8 -> this.existingFilesCount = (Integer) value; - case 9 -> this.deletedFilesCount = (Integer) value; - case 10 -> this.addedRowsCount = (Long) value; - case 11 -> this.existingRowsCount = (Long) value; - case 12 -> this.deletedRowsCount = (Long) value; - case 13 -> - this.partitions = - value == null - ? null - : ((List) value).toArray(new PartitionFieldSummary[0]); - case 14 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); - case 15 -> this.firstRowId = (Long) value; - default -> { + case 0: + // always coerce to String for Serializable + this.manifestPath = value.toString(); + return; + case 1: + this.length = (Long) value; + return; + case 2: + this.specId = (Integer) value; + return; + case 3: + this.content = + value != null ? ManifestContent.fromId((Integer) value) : ManifestContent.DATA; + return; + case 4: + this.sequenceNumber = value != null ? (Long) value : 0; + return; + case 5: + this.minSequenceNumber = value != null ? (Long) value : 0; + return; + case 6: + this.snapshotId = (Long) value; + return; + case 7: + this.addedFilesCount = (Integer) value; + return; + case 8: + this.existingFilesCount = (Integer) value; + return; + case 9: + this.deletedFilesCount = (Integer) value; + return; + case 10: + this.addedRowsCount = (Long) value; + return; + case 11: + this.existingRowsCount = (Long) value; + return; + case 12: + this.deletedRowsCount = (Long) value; + return; + case 13: + this.partitions = + value == null + ? null + : ((List) value).toArray(new PartitionFieldSummary[0]); + return; + case 14: + this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + return; + case 15: + this.firstRowId = (Long) value; + return; + default: // ignore the object, it must be from a newer version of the format - } } } diff --git a/core/src/main/java/org/apache/iceberg/GenericPartitionFieldSummary.java b/core/src/main/java/org/apache/iceberg/GenericPartitionFieldSummary.java index 62e7f080eb3f..e75f37d2ec12 100644 --- a/core/src/main/java/org/apache/iceberg/GenericPartitionFieldSummary.java +++ b/core/src/main/java/org/apache/iceberg/GenericPartitionFieldSummary.java @@ -149,13 +149,18 @@ public Object get(int i) { if (fromProjectionPos != null) { pos = fromProjectionPos[i]; } - return switch (pos) { - case 0 -> containsNull; - case 1 -> containsNaN; - case 2 -> lowerBound(); - case 3 -> upperBound(); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + switch (pos) { + case 0: + return containsNull; + case 1: + return containsNaN; + case 2: + return lowerBound(); + case 3: + return upperBound(); + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -167,13 +172,20 @@ public void set(int i, T value) { pos = fromProjectionPos[i]; } switch (pos) { - case 0 -> this.containsNull = (Boolean) value; - case 1 -> this.containsNaN = (Boolean) value; - case 2 -> this.lowerBound = ByteBuffers.toByteArray((ByteBuffer) value); - case 3 -> this.upperBound = ByteBuffers.toByteArray((ByteBuffer) value); - default -> { + case 0: + this.containsNull = (Boolean) value; + return; + case 1: + this.containsNaN = (Boolean) value; + return; + case 2: + this.lowerBound = ByteBuffers.toByteArray((ByteBuffer) value); + return; + case 3: + this.upperBound = ByteBuffers.toByteArray((ByteBuffer) value); + return; + default: // ignore the object, it must be from a newer version of the format - } } } diff --git a/core/src/main/java/org/apache/iceberg/ManifestFiles.java b/core/src/main/java/org/apache/iceberg/ManifestFiles.java index e5ab7b94add6..dae46e5ec49e 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestFiles.java +++ b/core/src/main/java/org/apache/iceberg/ManifestFiles.java @@ -303,21 +303,20 @@ static ManifestWriter newWriter( Long snapshotId, Long firstRowId, Map writerProperties) { - return switch (formatVersion) { - case 1 -> - new ManifestWriter.V1Writer(spec, encryptedOutputFile, snapshotId, writerProperties); - case 2 -> - new ManifestWriter.V2Writer(spec, encryptedOutputFile, snapshotId, writerProperties); - case 3 -> - new ManifestWriter.V3Writer( - spec, encryptedOutputFile, snapshotId, firstRowId, writerProperties); - case 4 -> - new ManifestWriter.V4Writer( - spec, encryptedOutputFile, snapshotId, firstRowId, writerProperties); - default -> - throw new UnsupportedOperationException( - "Cannot write manifest for table version: " + formatVersion); - }; + switch (formatVersion) { + case 1: + return new ManifestWriter.V1Writer(spec, encryptedOutputFile, snapshotId, writerProperties); + case 2: + return new ManifestWriter.V2Writer(spec, encryptedOutputFile, snapshotId, writerProperties); + case 3: + return new ManifestWriter.V3Writer( + spec, encryptedOutputFile, snapshotId, firstRowId, writerProperties); + case 4: + return new ManifestWriter.V4Writer( + spec, encryptedOutputFile, snapshotId, firstRowId, writerProperties); + } + throw new UnsupportedOperationException( + "Cannot write manifest for table version: " + formatVersion); } /** @@ -409,15 +408,18 @@ public static ManifestWriter writeDeleteManifest( EncryptedOutputFile outputFile, Long snapshotId, Map writerProperties) { - return switch (formatVersion) { - case 1 -> throw new IllegalArgumentException("Cannot write delete files in a v1 table"); - case 2 -> new ManifestWriter.V2DeleteWriter(spec, outputFile, snapshotId, writerProperties); - case 3 -> new ManifestWriter.V3DeleteWriter(spec, outputFile, snapshotId, writerProperties); - case 4 -> new ManifestWriter.V4DeleteWriter(spec, outputFile, snapshotId, writerProperties); - default -> - throw new UnsupportedOperationException( - "Cannot write manifest for table version: " + formatVersion); - }; + switch (formatVersion) { + case 1: + throw new IllegalArgumentException("Cannot write delete files in a v1 table"); + case 2: + return new ManifestWriter.V2DeleteWriter(spec, outputFile, snapshotId, writerProperties); + case 3: + return new ManifestWriter.V3DeleteWriter(spec, outputFile, snapshotId, writerProperties); + case 4: + return new ManifestWriter.V4DeleteWriter(spec, outputFile, snapshotId, writerProperties); + } + throw new UnsupportedOperationException( + "Cannot write manifest for table version: " + formatVersion); } /** @@ -456,10 +458,14 @@ static ManifestReader open(ManifestFile manifest, FileIO io) { static ManifestReader open( ManifestFile manifest, FileIO io, Map specsById) { - return switch (manifest.content()) { - case DATA -> ManifestFiles.read(manifest, io, specsById); - case DELETES -> ManifestFiles.readDeleteManifest(manifest, io, specsById); - }; + switch (manifest.content()) { + case DATA: + return ManifestFiles.read(manifest, io, specsById); + case DELETES: + return ManifestFiles.readDeleteManifest(manifest, io, specsById); + } + throw new UnsupportedOperationException( + "Cannot read unknown manifest type: " + manifest.content()); } static ManifestFile copyAppendManifest( @@ -537,15 +543,17 @@ private static ManifestFile copyManifestInternal( entry.status(), allowedEntryStatus); switch (entry.status()) { - case ADDED -> { + case ADDED: summaryBuilder.addedFile(reader.spec(), entry.file()); writer.add(entry); - } - case EXISTING -> writer.existing(entry); - case DELETED -> { + break; + case EXISTING: + writer.existing(entry); + break; + case DELETED: summaryBuilder.deletedFile(reader.spec(), entry.file()); writer.delete(entry); - } + break; } } diff --git a/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java b/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java index b598b97069b4..6a7ccea6b679 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java +++ b/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java @@ -166,39 +166,72 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - return switch (pos) { - case 0 -> addedFilesCount; - case 1 -> existingFilesCount; - case 2 -> deletedFilesCount; - case 3 -> replacedFilesCount; - case 4 -> addedRowsCount; - case 5 -> existingRowsCount; - case 6 -> deletedRowsCount; - case 7 -> replacedRowsCount; - case 8 -> minSequenceNumber; - case 9 -> dv(); - case 10 -> dvCardinality; - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + switch (pos) { + case 0: + return addedFilesCount; + case 1: + return existingFilesCount; + case 2: + return deletedFilesCount; + case 3: + return replacedFilesCount; + case 4: + return addedRowsCount; + case 5: + return existingRowsCount; + case 6: + return deletedRowsCount; + case 7: + return replacedRowsCount; + case 8: + return minSequenceNumber; + case 9: + return dv(); + case 10: + return dvCardinality; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0 -> this.addedFilesCount = (Integer) value; - case 1 -> this.existingFilesCount = (Integer) value; - case 2 -> this.deletedFilesCount = (Integer) value; - case 3 -> this.replacedFilesCount = (Integer) value; - case 4 -> this.addedRowsCount = (Long) value; - case 5 -> this.existingRowsCount = (Long) value; - case 6 -> this.deletedRowsCount = (Long) value; - case 7 -> this.replacedRowsCount = (Long) value; - case 8 -> this.minSequenceNumber = (Long) value; - case 9 -> this.dv = ByteBuffers.toByteArray((ByteBuffer) value); - case 10 -> this.dvCardinality = (Long) value; - default -> { + case 0: + this.addedFilesCount = (Integer) value; + break; + case 1: + this.existingFilesCount = (Integer) value; + break; + case 2: + this.deletedFilesCount = (Integer) value; + break; + case 3: + this.replacedFilesCount = (Integer) value; + break; + case 4: + this.addedRowsCount = (Long) value; + break; + case 5: + this.existingRowsCount = (Long) value; + break; + case 6: + this.deletedRowsCount = (Long) value; + break; + case 7: + this.replacedRowsCount = (Long) value; + break; + case 8: + this.minSequenceNumber = (Long) value; + break; + case 9: + this.dv = ByteBuffers.toByteArray((ByteBuffer) value); + break; + case 10: + this.dvCardinality = (Long) value; + break; + default: // ignore the object, it must be from a newer version of the format - } } } diff --git a/core/src/main/java/org/apache/iceberg/ManifestLists.java b/core/src/main/java/org/apache/iceberg/ManifestLists.java index b1deea83ecf6..dbe080584b13 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestLists.java +++ b/core/src/main/java/org/apache/iceberg/ManifestLists.java @@ -66,37 +66,35 @@ static ManifestListWriter write( Long parentSnapshotId, long sequenceNumber, Long firstRowId) { - return switch (formatVersion) { - case 1 -> { + switch (formatVersion) { + case 1: Preconditions.checkArgument( sequenceNumber == TableMetadata.INITIAL_SEQUENCE_NUMBER, "Invalid sequence number for v1 manifest list: %s", sequenceNumber); - yield new ManifestListWriter.V1Writer( + return new ManifestListWriter.V1Writer( manifestListFile, encryptionManager, snapshotId, parentSnapshotId); - } - case 2 -> - new ManifestListWriter.V2Writer( - manifestListFile, encryptionManager, snapshotId, parentSnapshotId, sequenceNumber); - case 3 -> - new ManifestListWriter.V3Writer( - manifestListFile, - encryptionManager, - snapshotId, - parentSnapshotId, - sequenceNumber, - firstRowId); - case 4 -> - new ManifestListWriter.V4Writer( - manifestListFile, - encryptionManager, - snapshotId, - parentSnapshotId, - sequenceNumber, - firstRowId); - default -> - throw new UnsupportedOperationException( - "Cannot write manifest list for table version: " + formatVersion); - }; + case 2: + return new ManifestListWriter.V2Writer( + manifestListFile, encryptionManager, snapshotId, parentSnapshotId, sequenceNumber); + case 3: + return new ManifestListWriter.V3Writer( + manifestListFile, + encryptionManager, + snapshotId, + parentSnapshotId, + sequenceNumber, + firstRowId); + case 4: + return new ManifestListWriter.V4Writer( + manifestListFile, + encryptionManager, + snapshotId, + parentSnapshotId, + sequenceNumber, + firstRowId); + } + throw new UnsupportedOperationException( + "Cannot write manifest list for table version: " + formatVersion); } } diff --git a/core/src/main/java/org/apache/iceberg/ManifestWriter.java b/core/src/main/java/org/apache/iceberg/ManifestWriter.java index d1524fbfd4a1..321bcd89d8b1 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestWriter.java +++ b/core/src/main/java/org/apache/iceberg/ManifestWriter.java @@ -110,18 +110,18 @@ protected ManifestContent content() { void addEntry(ManifestEntry entry) { switch (entry.status()) { - case ADDED -> { + case ADDED: addedFiles += 1; addedRows += entry.file().recordCount(); - } - case EXISTING -> { + break; + case EXISTING: existingFiles += 1; existingRows += entry.file().recordCount(); - } - case DELETED -> { + break; + case DELETED: deletedFiles += 1; deletedRows += entry.file().recordCount(); - } + break; } stats.update(entry.file().partition()); diff --git a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java index a4d45fd640f6..1a70b4f90b8f 100644 --- a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java @@ -294,19 +294,24 @@ protected void validateNewDeleteFile(DeleteFile file) { private static void validateDeleteFileForVersion(DeleteFile file, int formatVersion) { switch (formatVersion) { - case 1 -> throw new IllegalArgumentException("Deletes are supported in V2 and above"); - case 2 -> - Preconditions.checkArgument( - file.content() == FileContent.EQUALITY_DELETES || !ContentFileUtil.isDV(file), - "Must not use DVs for position deletes in V2: %s", - ContentFileUtil.dvDesc(file)); - case 3, 4 -> - Preconditions.checkArgument( - file.content() == FileContent.EQUALITY_DELETES || ContentFileUtil.isDV(file), - "Must use DVs for position deletes in V%s: %s", - formatVersion, - file.location()); - default -> throw new IllegalArgumentException("Unsupported format version: " + formatVersion); + case 1: + throw new IllegalArgumentException("Deletes are supported in V2 and above"); + case 2: + Preconditions.checkArgument( + file.content() == FileContent.EQUALITY_DELETES || !ContentFileUtil.isDV(file), + "Must not use DVs for position deletes in V2: %s", + ContentFileUtil.dvDesc(file)); + break; + case 3: + case 4: + Preconditions.checkArgument( + file.content() == FileContent.EQUALITY_DELETES || ContentFileUtil.isDV(file), + "Must use DVs for position deletes in V%s: %s", + formatVersion, + file.location()); + break; + default: + throw new IllegalArgumentException("Unsupported format version: " + formatVersion); } } diff --git a/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java b/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java index 735d62362a4c..adb0f18ba1ad 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java +++ b/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java @@ -20,6 +20,7 @@ import java.util.Locale; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchTableException; public class MetadataTableUtils { private MetadataTableUtils() {} @@ -54,24 +55,43 @@ public static Table createMetadataTableInstance( private static Table createMetadataTableInstance( Table baseTable, String metadataTableName, MetadataTableType type) { - return switch (type) { - case ENTRIES -> new ManifestEntriesTable(baseTable, metadataTableName); - case FILES -> new FilesTable(baseTable, metadataTableName); - case DATA_FILES -> new DataFilesTable(baseTable, metadataTableName); - case DELETE_FILES -> new DeleteFilesTable(baseTable, metadataTableName); - case HISTORY -> new HistoryTable(baseTable, metadataTableName); - case SNAPSHOTS -> new SnapshotsTable(baseTable, metadataTableName); - case METADATA_LOG_ENTRIES -> new MetadataLogEntriesTable(baseTable, metadataTableName); - case REFS -> new RefsTable(baseTable, metadataTableName); - case MANIFESTS -> new ManifestsTable(baseTable, metadataTableName); - case PARTITIONS -> new PartitionsTable(baseTable, metadataTableName); - case ALL_DATA_FILES -> new AllDataFilesTable(baseTable, metadataTableName); - case ALL_DELETE_FILES -> new AllDeleteFilesTable(baseTable, metadataTableName); - case ALL_FILES -> new AllFilesTable(baseTable, metadataTableName); - case ALL_MANIFESTS -> new AllManifestsTable(baseTable, metadataTableName); - case ALL_ENTRIES -> new AllEntriesTable(baseTable, metadataTableName); - case POSITION_DELETES -> new PositionDeletesTable(baseTable, metadataTableName); - }; + switch (type) { + case ENTRIES: + return new ManifestEntriesTable(baseTable, metadataTableName); + case FILES: + return new FilesTable(baseTable, metadataTableName); + case DATA_FILES: + return new DataFilesTable(baseTable, metadataTableName); + case DELETE_FILES: + return new DeleteFilesTable(baseTable, metadataTableName); + case HISTORY: + return new HistoryTable(baseTable, metadataTableName); + case SNAPSHOTS: + return new SnapshotsTable(baseTable, metadataTableName); + case METADATA_LOG_ENTRIES: + return new MetadataLogEntriesTable(baseTable, metadataTableName); + case REFS: + return new RefsTable(baseTable, metadataTableName); + case MANIFESTS: + return new ManifestsTable(baseTable, metadataTableName); + case PARTITIONS: + return new PartitionsTable(baseTable, metadataTableName); + case ALL_DATA_FILES: + return new AllDataFilesTable(baseTable, metadataTableName); + case ALL_DELETE_FILES: + return new AllDeleteFilesTable(baseTable, metadataTableName); + case ALL_FILES: + return new AllFilesTable(baseTable, metadataTableName); + case ALL_MANIFESTS: + return new AllManifestsTable(baseTable, metadataTableName); + case ALL_ENTRIES: + return new AllEntriesTable(baseTable, metadataTableName); + case POSITION_DELETES: + return new PositionDeletesTable(baseTable, metadataTableName); + default: + throw new NoSuchTableException( + "Unknown metadata table type: %s for %s", type, metadataTableName); + } } public static Table createMetadataTableInstance( diff --git a/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java b/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java index a0490b3c5d8e..280fe7565d75 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java +++ b/core/src/main/java/org/apache/iceberg/MetadataUpdateParser.java @@ -192,62 +192,89 @@ public static void toJson(MetadataUpdate metadataUpdate, JsonGenerator generator generator.writeStringField(ACTION, updateAction); switch (updateAction) { - case ASSIGN_UUID -> writeAssignUUID((MetadataUpdate.AssignUUID) metadataUpdate, generator); - case UPGRADE_FORMAT_VERSION -> - writeUpgradeFormatVersion( - (MetadataUpdate.UpgradeFormatVersion) metadataUpdate, generator); - case ADD_SCHEMA -> writeAddSchema((MetadataUpdate.AddSchema) metadataUpdate, generator); - case SET_CURRENT_SCHEMA -> - writeSetCurrentSchema((MetadataUpdate.SetCurrentSchema) metadataUpdate, generator); - case ADD_PARTITION_SPEC -> - writeAddPartitionSpec((MetadataUpdate.AddPartitionSpec) metadataUpdate, generator); - case SET_DEFAULT_PARTITION_SPEC -> - writeSetDefaultPartitionSpec( - (MetadataUpdate.SetDefaultPartitionSpec) metadataUpdate, generator); - case ADD_SORT_ORDER -> - writeAddSortOrder((MetadataUpdate.AddSortOrder) metadataUpdate, generator); - case SET_DEFAULT_SORT_ORDER -> - writeSetDefaultSortOrder((MetadataUpdate.SetDefaultSortOrder) metadataUpdate, generator); - case SET_STATISTICS -> - writeSetStatistics((MetadataUpdate.SetStatistics) metadataUpdate, generator); - case REMOVE_STATISTICS -> - writeRemoveStatistics((MetadataUpdate.RemoveStatistics) metadataUpdate, generator); - case SET_PARTITION_STATISTICS -> - writeSetPartitionStatistics( - (MetadataUpdate.SetPartitionStatistics) metadataUpdate, generator); - case REMOVE_PARTITION_STATISTICS -> - writeRemovePartitionStatistics( - (MetadataUpdate.RemovePartitionStatistics) metadataUpdate, generator); - case ADD_SNAPSHOT -> writeAddSnapshot((MetadataUpdate.AddSnapshot) metadataUpdate, generator); - case REMOVE_SNAPSHOTS -> - writeRemoveSnapshots((MetadataUpdate.RemoveSnapshots) metadataUpdate, generator); - case REMOVE_SNAPSHOT_REF -> - writeRemoveSnapshotRef((MetadataUpdate.RemoveSnapshotRef) metadataUpdate, generator); - case SET_SNAPSHOT_REF -> - writeSetSnapshotRef((MetadataUpdate.SetSnapshotRef) metadataUpdate, generator); - case SET_PROPERTIES -> - writeSetProperties((MetadataUpdate.SetProperties) metadataUpdate, generator); - case REMOVE_PROPERTIES -> - writeRemoveProperties((MetadataUpdate.RemoveProperties) metadataUpdate, generator); - case SET_LOCATION -> writeSetLocation((MetadataUpdate.SetLocation) metadataUpdate, generator); - case ADD_VIEW_VERSION -> - writeAddViewVersion((MetadataUpdate.AddViewVersion) metadataUpdate, generator); - case SET_CURRENT_VIEW_VERSION -> - writeSetCurrentViewVersionId( - (MetadataUpdate.SetCurrentViewVersion) metadataUpdate, generator); - case REMOVE_PARTITION_SPECS -> - writeRemovePartitionSpecs( - (MetadataUpdate.RemovePartitionSpecs) metadataUpdate, generator); - case REMOVE_SCHEMAS -> - writeRemoveSchemas((MetadataUpdate.RemoveSchemas) metadataUpdate, generator); - case ADD_ENCRYPTION_KEY -> - writeAddEncryptionKey((MetadataUpdate.AddEncryptionKey) metadataUpdate, generator); - case REMOVE_ENCRYPTION_KEY -> - writeRemoveEncryptionKey((MetadataUpdate.RemoveEncryptionKey) metadataUpdate, generator); - default -> - throw new IllegalArgumentException( - String.format( - "Cannot convert metadata update to json. Unrecognized action: %s", updateAction)); + case ASSIGN_UUID: + writeAssignUUID((MetadataUpdate.AssignUUID) metadataUpdate, generator); + break; + case UPGRADE_FORMAT_VERSION: + writeUpgradeFormatVersion((MetadataUpdate.UpgradeFormatVersion) metadataUpdate, generator); + break; + case ADD_SCHEMA: + writeAddSchema((MetadataUpdate.AddSchema) metadataUpdate, generator); + break; + case SET_CURRENT_SCHEMA: + writeSetCurrentSchema((MetadataUpdate.SetCurrentSchema) metadataUpdate, generator); + break; + case ADD_PARTITION_SPEC: + writeAddPartitionSpec((MetadataUpdate.AddPartitionSpec) metadataUpdate, generator); + break; + case SET_DEFAULT_PARTITION_SPEC: + writeSetDefaultPartitionSpec( + (MetadataUpdate.SetDefaultPartitionSpec) metadataUpdate, generator); + break; + case ADD_SORT_ORDER: + writeAddSortOrder((MetadataUpdate.AddSortOrder) metadataUpdate, generator); + break; + case SET_DEFAULT_SORT_ORDER: + writeSetDefaultSortOrder((MetadataUpdate.SetDefaultSortOrder) metadataUpdate, generator); + break; + case SET_STATISTICS: + writeSetStatistics((MetadataUpdate.SetStatistics) metadataUpdate, generator); + break; + case REMOVE_STATISTICS: + writeRemoveStatistics((MetadataUpdate.RemoveStatistics) metadataUpdate, generator); + break; + case SET_PARTITION_STATISTICS: + writeSetPartitionStatistics( + (MetadataUpdate.SetPartitionStatistics) metadataUpdate, generator); + break; + case REMOVE_PARTITION_STATISTICS: + writeRemovePartitionStatistics( + (MetadataUpdate.RemovePartitionStatistics) metadataUpdate, generator); + break; + case ADD_SNAPSHOT: + writeAddSnapshot((MetadataUpdate.AddSnapshot) metadataUpdate, generator); + break; + case REMOVE_SNAPSHOTS: + writeRemoveSnapshots((MetadataUpdate.RemoveSnapshots) metadataUpdate, generator); + break; + case REMOVE_SNAPSHOT_REF: + writeRemoveSnapshotRef((MetadataUpdate.RemoveSnapshotRef) metadataUpdate, generator); + break; + case SET_SNAPSHOT_REF: + writeSetSnapshotRef((MetadataUpdate.SetSnapshotRef) metadataUpdate, generator); + break; + case SET_PROPERTIES: + writeSetProperties((MetadataUpdate.SetProperties) metadataUpdate, generator); + break; + case REMOVE_PROPERTIES: + writeRemoveProperties((MetadataUpdate.RemoveProperties) metadataUpdate, generator); + break; + case SET_LOCATION: + writeSetLocation((MetadataUpdate.SetLocation) metadataUpdate, generator); + break; + case ADD_VIEW_VERSION: + writeAddViewVersion((MetadataUpdate.AddViewVersion) metadataUpdate, generator); + break; + case SET_CURRENT_VIEW_VERSION: + writeSetCurrentViewVersionId( + (MetadataUpdate.SetCurrentViewVersion) metadataUpdate, generator); + break; + case REMOVE_PARTITION_SPECS: + writeRemovePartitionSpecs((MetadataUpdate.RemovePartitionSpecs) metadataUpdate, generator); + break; + case REMOVE_SCHEMAS: + writeRemoveSchemas((MetadataUpdate.RemoveSchemas) metadataUpdate, generator); + break; + case ADD_ENCRYPTION_KEY: + writeAddEncryptionKey((MetadataUpdate.AddEncryptionKey) metadataUpdate, generator); + break; + case REMOVE_ENCRYPTION_KEY: + writeRemoveEncryptionKey((MetadataUpdate.RemoveEncryptionKey) metadataUpdate, generator); + break; + default: + throw new IllegalArgumentException( + String.format( + "Cannot convert metadata update to json. Unrecognized action: %s", updateAction)); } generator.writeEndObject(); @@ -272,36 +299,61 @@ public static MetadataUpdate fromJson(JsonNode jsonNode) { jsonNode.hasNonNull(ACTION), "Cannot parse metadata update. Missing field: action"); String action = JsonUtil.getString(ACTION, jsonNode).toLowerCase(Locale.ROOT); - return switch (action) { - case ASSIGN_UUID -> readAssignUUID(jsonNode); - case UPGRADE_FORMAT_VERSION -> readUpgradeFormatVersion(jsonNode); - case ADD_SCHEMA -> readAddSchema(jsonNode); - case SET_CURRENT_SCHEMA -> readSetCurrentSchema(jsonNode); - case ADD_PARTITION_SPEC -> readAddPartitionSpec(jsonNode); - case SET_DEFAULT_PARTITION_SPEC -> readSetDefaultPartitionSpec(jsonNode); - case ADD_SORT_ORDER -> readAddSortOrder(jsonNode); - case SET_DEFAULT_SORT_ORDER -> readSetDefaultSortOrder(jsonNode); - case SET_STATISTICS -> readSetStatistics(jsonNode); - case REMOVE_STATISTICS -> readRemoveStatistics(jsonNode); - case SET_PARTITION_STATISTICS -> readSetPartitionStatistics(jsonNode); - case REMOVE_PARTITION_STATISTICS -> readRemovePartitionStatistics(jsonNode); - case ADD_SNAPSHOT -> readAddSnapshot(jsonNode); - case REMOVE_SNAPSHOTS -> readRemoveSnapshots(jsonNode); - case REMOVE_SNAPSHOT_REF -> readRemoveSnapshotRef(jsonNode); - case SET_SNAPSHOT_REF -> readSetSnapshotRef(jsonNode); - case SET_PROPERTIES -> readSetProperties(jsonNode); - case REMOVE_PROPERTIES -> readRemoveProperties(jsonNode); - case SET_LOCATION -> readSetLocation(jsonNode); - case ADD_VIEW_VERSION -> readAddViewVersion(jsonNode); - case SET_CURRENT_VIEW_VERSION -> readCurrentViewVersionId(jsonNode); - case REMOVE_PARTITION_SPECS -> readRemovePartitionSpecs(jsonNode); - case REMOVE_SCHEMAS -> readRemoveSchemas(jsonNode); - case ADD_ENCRYPTION_KEY -> readAddEncryptionKey(jsonNode); - case REMOVE_ENCRYPTION_KEY -> readRemoveEncryptionKey(jsonNode); - default -> - throw new UnsupportedOperationException( - String.format("Cannot convert metadata update action to json: %s", action)); - }; + switch (action) { + case ASSIGN_UUID: + return readAssignUUID(jsonNode); + case UPGRADE_FORMAT_VERSION: + return readUpgradeFormatVersion(jsonNode); + case ADD_SCHEMA: + return readAddSchema(jsonNode); + case SET_CURRENT_SCHEMA: + return readSetCurrentSchema(jsonNode); + case ADD_PARTITION_SPEC: + return readAddPartitionSpec(jsonNode); + case SET_DEFAULT_PARTITION_SPEC: + return readSetDefaultPartitionSpec(jsonNode); + case ADD_SORT_ORDER: + return readAddSortOrder(jsonNode); + case SET_DEFAULT_SORT_ORDER: + return readSetDefaultSortOrder(jsonNode); + case SET_STATISTICS: + return readSetStatistics(jsonNode); + case REMOVE_STATISTICS: + return readRemoveStatistics(jsonNode); + case SET_PARTITION_STATISTICS: + return readSetPartitionStatistics(jsonNode); + case REMOVE_PARTITION_STATISTICS: + return readRemovePartitionStatistics(jsonNode); + case ADD_SNAPSHOT: + return readAddSnapshot(jsonNode); + case REMOVE_SNAPSHOTS: + return readRemoveSnapshots(jsonNode); + case REMOVE_SNAPSHOT_REF: + return readRemoveSnapshotRef(jsonNode); + case SET_SNAPSHOT_REF: + return readSetSnapshotRef(jsonNode); + case SET_PROPERTIES: + return readSetProperties(jsonNode); + case REMOVE_PROPERTIES: + return readRemoveProperties(jsonNode); + case SET_LOCATION: + return readSetLocation(jsonNode); + case ADD_VIEW_VERSION: + return readAddViewVersion(jsonNode); + case SET_CURRENT_VIEW_VERSION: + return readCurrentViewVersionId(jsonNode); + case REMOVE_PARTITION_SPECS: + return readRemovePartitionSpecs(jsonNode); + case REMOVE_SCHEMAS: + return readRemoveSchemas(jsonNode); + case ADD_ENCRYPTION_KEY: + return readAddEncryptionKey(jsonNode); + case REMOVE_ENCRYPTION_KEY: + return readRemoveEncryptionKey(jsonNode); + default: + throw new UnsupportedOperationException( + String.format("Cannot convert metadata update action to json: %s", action)); + } } private static void writeAssignUUID(MetadataUpdate.AssignUUID update, JsonGenerator gen) diff --git a/core/src/main/java/org/apache/iceberg/PartitionData.java b/core/src/main/java/org/apache/iceberg/PartitionData.java index 1f95bf820bdb..41ad72bf0bac 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionData.java +++ b/core/src/main/java/org/apache/iceberg/PartitionData.java @@ -203,16 +203,21 @@ public static Object[] copyData(Types.StructType type, Object[] data) { } else { Types.NestedField field = fields.get(i); switch (field.type().typeId()) { - case STRUCT, LIST, MAP -> - throw new IllegalArgumentException("Unsupported type in partition data: " + type); - case BINARY, FIXED -> { + case STRUCT: + case LIST: + case MAP: + throw new IllegalArgumentException("Unsupported type in partition data: " + type); + case BINARY: + case FIXED: byte[] buffer = (byte[]) data[i]; copy[i] = Arrays.copyOf(buffer, buffer.length); - } - case STRING -> copy[i] = data[i].toString(); - default -> - // no need to copy the object - copy[i] = data[i]; + break; + case STRING: + copy[i] = data[i].toString(); + break; + default: + // no need to copy the object + copy[i] = data[i]; } } } diff --git a/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java b/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java index cff78ecd7b71..611bd3cd0783 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java +++ b/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java @@ -355,7 +355,7 @@ private static void liveEntry(PartitionStatistics stats, ContentFile file, Sn Preconditions.checkArgument(stats.specId() == file.specId(), "Spec IDs must match"); switch (file.content()) { - case DATA -> { + case DATA: stats.set( PartitionStatistics.DATA_RECORD_COUNT_POSITION, stats.dataRecordCount() + file.recordCount()); @@ -363,8 +363,8 @@ private static void liveEntry(PartitionStatistics stats, ContentFile file, Sn stats.set( PartitionStatistics.TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION, stats.totalDataFileSizeInBytes() + file.fileSizeInBytes()); - } - case POSITION_DELETES -> { + break; + case POSITION_DELETES: stats.set( PartitionStatistics.POSITION_DELETE_RECORD_COUNT_POSITION, stats.positionDeleteRecordCount() + file.recordCount()); @@ -375,18 +375,18 @@ private static void liveEntry(PartitionStatistics stats, ContentFile file, Sn PartitionStatistics.POSITION_DELETE_FILE_COUNT_POSITION, stats.positionDeleteFileCount() + 1); } - } - case EQUALITY_DELETES -> { + + break; + case EQUALITY_DELETES: stats.set( PartitionStatistics.EQUALITY_DELETE_RECORD_COUNT_POSITION, stats.equalityDeleteRecordCount() + file.recordCount()); stats.set( PartitionStatistics.EQUALITY_DELETE_FILE_COUNT_POSITION, stats.equalityDeleteFileCount() + 1); - } - default -> - throw new UnsupportedOperationException( - "Unsupported file content type: " + file.content()); + break; + default: + throw new UnsupportedOperationException("Unsupported file content type: " + file.content()); } if (snapshot != null) { @@ -420,7 +420,7 @@ private static void deletedEntryForIncrementalCompute( Preconditions.checkArgument(stats.specId() == file.specId(), "Spec IDs must match"); switch (file.content()) { - case DATA -> { + case DATA: stats.set( PartitionStatistics.DATA_RECORD_COUNT_POSITION, stats.dataRecordCount() - file.recordCount()); @@ -428,8 +428,8 @@ private static void deletedEntryForIncrementalCompute( stats.set( PartitionStatistics.TOTAL_DATA_FILE_SIZE_IN_BYTES_POSITION, stats.totalDataFileSizeInBytes() - file.fileSizeInBytes()); - } - case POSITION_DELETES -> { + break; + case POSITION_DELETES: stats.set( PartitionStatistics.POSITION_DELETE_RECORD_COUNT_POSITION, stats.positionDeleteRecordCount() - file.recordCount()); @@ -440,18 +440,18 @@ private static void deletedEntryForIncrementalCompute( PartitionStatistics.POSITION_DELETE_FILE_COUNT_POSITION, stats.positionDeleteFileCount() - 1); } - } - case EQUALITY_DELETES -> { + + break; + case EQUALITY_DELETES: stats.set( PartitionStatistics.EQUALITY_DELETE_RECORD_COUNT_POSITION, stats.equalityDeleteRecordCount() - file.recordCount()); stats.set( PartitionStatistics.EQUALITY_DELETE_FILE_COUNT_POSITION, stats.equalityDeleteFileCount() - 1); - } - default -> - throw new UnsupportedOperationException( - "Unsupported file content type: " + file.content()); + break; + default: + throw new UnsupportedOperationException("Unsupported file content type: " + file.content()); } if (snapshot != null) { diff --git a/core/src/main/java/org/apache/iceberg/PartitionsTable.java b/core/src/main/java/org/apache/iceberg/PartitionsTable.java index b3506ffbde9c..b5dd5c284ce2 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionsTable.java +++ b/core/src/main/java/org/apache/iceberg/PartitionsTable.java @@ -340,22 +340,22 @@ void update(ContentFile file, Snapshot snapshot) { } switch (file.content()) { - case DATA -> { + case DATA: this.dataRecordCount += file.recordCount(); this.dataFileCount += 1; this.dataFileSizeInBytes += file.fileSizeInBytes(); - } - case POSITION_DELETES -> { + break; + case POSITION_DELETES: this.posDeleteRecordCount += file.recordCount(); this.posDeleteFileCount += 1; - } - case EQUALITY_DELETES -> { + break; + case EQUALITY_DELETES: this.eqDeleteRecordCount += file.recordCount(); this.eqDeleteFileCount += 1; - } - default -> - throw new UnsupportedOperationException( - "Unsupported file content type: " + file.content()); + break; + default: + throw new UnsupportedOperationException( + "Unsupported file content type: " + file.content()); } } diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java index a091d349d096..69f82931833e 100644 --- a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java +++ b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java @@ -451,7 +451,7 @@ private static RewriteResult writeDeleteFileEntry( RewriteResult result = new RewriteResult<>(); switch (file.content()) { - case POSITION_DELETES -> { + case POSITION_DELETES: DeleteFile posDeleteFile = newPositionDeleteEntry(file, spec, sourcePrefix, targetPrefix); appendEntryWithFile(entry, writer, posDeleteFile); // keep the following entries in metadata but exclude them from copyPlan @@ -467,8 +467,7 @@ private static RewriteResult writeDeleteFileEntry( } result.toRewrite().add(file.copy()); return result; - } - case EQUALITY_DELETES -> { + case EQUALITY_DELETES: DeleteFile eqDeleteFile = newEqualityDeleteEntry(file, spec, sourcePrefix, targetPrefix); appendEntryWithFile(entry, writer, eqDeleteFile); // keep the following entries in metadata but exclude them from copyPlan @@ -479,10 +478,9 @@ private static RewriteResult writeDeleteFileEntry( result.copyPlan().add(Pair.of(file.location(), eqDeleteFile.location())); } return result; - } - default -> - throw new UnsupportedOperationException( - "Unsupported delete file type: " + file.content()); + + default: + throw new UnsupportedOperationException("Unsupported delete file type: " + file.content()); } } @@ -490,11 +488,16 @@ private static > void appendEntryWithFile( ManifestEntry entry, ManifestWriter writer, F file) { switch (entry.status()) { - case ADDED -> writer.add(file); - case EXISTING -> - writer.existing( - file, entry.snapshotId(), entry.dataSequenceNumber(), entry.fileSequenceNumber()); - case DELETED -> writer.delete(file, entry.dataSequenceNumber(), entry.fileSequenceNumber()); + case ADDED: + writer.add(file); + break; + case EXISTING: + writer.existing( + file, entry.snapshotId(), entry.dataSequenceNumber(), entry.fileSequenceNumber()); + break; + case DELETED: + writer.delete(file, entry.dataSequenceNumber(), entry.fileSequenceNumber()); + break; } } diff --git a/core/src/main/java/org/apache/iceberg/ScanSummary.java b/core/src/main/java/org/apache/iceberg/ScanSummary.java index 99b5bd87c6f9..5f8e66c0b450 100644 --- a/core/src/main/java/org/apache/iceberg/ScanSummary.java +++ b/core/src/main/java/org/apache/iceberg/ScanSummary.java @@ -388,37 +388,37 @@ static Pair timestampRange(List> timeFilters) for (UnboundPredicate pred : timeFilters) { long value = pred.literal().value(); switch (pred.op()) { - case LT -> { + case LT: if (value - 1 < maxTimestamp) { maxTimestamp = value - 1; } - } - case LT_EQ -> { + break; + case LT_EQ: if (value < maxTimestamp) { maxTimestamp = value; } - } - case GT -> { + break; + case GT: if (value + 1 > minTimestamp) { minTimestamp = value + 1; } - } - case GT_EQ -> { + break; + case GT_EQ: if (value > minTimestamp) { minTimestamp = value; } - } - case EQ -> { + break; + case EQ: if (value < maxTimestamp) { maxTimestamp = value; } if (value > minTimestamp) { minTimestamp = value; } - } - default -> - throw new UnsupportedOperationException( - "Cannot filter timestamps using predicate: " + pred); + break; + default: + throw new UnsupportedOperationException( + "Cannot filter timestamps using predicate: " + pred); } } diff --git a/core/src/main/java/org/apache/iceberg/ScanTaskParser.java b/core/src/main/java/org/apache/iceberg/ScanTaskParser.java index cf9faefa247d..67e44cea7d07 100644 --- a/core/src/main/java/org/apache/iceberg/ScanTaskParser.java +++ b/core/src/main/java/org/apache/iceberg/ScanTaskParser.java @@ -113,12 +113,19 @@ private static FileScanTask fromJson(JsonNode jsonNode, boolean caseSensitive) { taskType = TaskType.fromTypeName(taskTypeStr); } - return switch (taskType) { - case FILE_SCAN_TASK -> FileScanTaskParser.fromJson(jsonNode, caseSensitive); - case DATA_TASK -> DataTaskParser.fromJson(jsonNode); - case FILES_TABLE_TASK -> FilesTableTaskParser.fromJson(jsonNode); - case ALL_MANIFESTS_TABLE_TASK -> AllManifestsTableTaskParser.fromJson(jsonNode); - case MANIFEST_ENTRIES_TABLE_TASK -> ManifestEntriesTableTaskParser.fromJson(jsonNode); - }; + switch (taskType) { + case FILE_SCAN_TASK: + return FileScanTaskParser.fromJson(jsonNode, caseSensitive); + case DATA_TASK: + return DataTaskParser.fromJson(jsonNode); + case FILES_TABLE_TASK: + return FilesTableTaskParser.fromJson(jsonNode); + case ALL_MANIFESTS_TABLE_TASK: + return AllManifestsTableTaskParser.fromJson(jsonNode); + case MANIFEST_ENTRIES_TABLE_TASK: + return ManifestEntriesTableTaskParser.fromJson(jsonNode); + default: + throw new UnsupportedOperationException("Unsupported task type: " + taskType.typeName()); + } } } diff --git a/core/src/main/java/org/apache/iceberg/SchemaParser.java b/core/src/main/java/org/apache/iceberg/SchemaParser.java index 6b31f0cc9c0d..7481af0284f6 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaParser.java +++ b/core/src/main/java/org/apache/iceberg/SchemaParser.java @@ -146,10 +146,17 @@ static void toJson(Type type, JsonGenerator generator) throws IOException { } else { Type.NestedType nested = type.asNestedType(); switch (type.typeId()) { - case STRUCT -> toJson(nested.asStructType(), generator); - case LIST -> toJson(nested.asListType(), generator); - case MAP -> toJson(nested.asMapType(), generator); - default -> throw new IllegalArgumentException("Cannot write unknown type: " + type); + case STRUCT: + toJson(nested.asStructType(), generator); + break; + case LIST: + toJson(nested.asListType(), generator); + break; + case MAP: + toJson(nested.asMapType(), generator); + break; + default: + throw new IllegalArgumentException("Cannot write unknown type: " + type); } } } diff --git a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java index 1cb2d020ba4c..1fa6ebbe8fef 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java +++ b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java @@ -790,21 +790,27 @@ private static List moveFields( reordered.remove(toMove); switch (move.type()) { - case FIRST -> reordered.addFirst(toMove); - case BEFORE -> { + case FIRST: + reordered.addFirst(toMove); + break; + + case BEFORE: Types.NestedField before = Iterables.find(reordered, field -> field.fieldId() == move.referenceFieldId()); int beforeIndex = reordered.indexOf(before); // insert the new node at the index of the existing node reordered.add(beforeIndex, toMove); - } - case AFTER -> { + break; + + case AFTER: Types.NestedField after = Iterables.find(reordered, field -> field.fieldId() == move.referenceFieldId()); int afterIndex = reordered.indexOf(after); reordered.add(afterIndex + 1, toMove); - } - default -> throw new UnsupportedOperationException("Unknown move type: " + move.type()); + break; + + default: + throw new UnsupportedOperationException("Unknown move type: " + move.type()); } } diff --git a/core/src/main/java/org/apache/iceberg/SingleValueParser.java b/core/src/main/java/org/apache/iceberg/SingleValueParser.java index 0da8f4177532..c7f07ea1a2d4 100644 --- a/core/src/main/java/org/apache/iceberg/SingleValueParser.java +++ b/core/src/main/java/org/apache/iceberg/SingleValueParser.java @@ -46,51 +46,45 @@ private SingleValueParser() {} private static final String KEYS = "keys"; private static final String VALUES = "values"; - @SuppressWarnings("MethodLength") public static Object fromJson(Type type, JsonNode defaultValue) { if (defaultValue == null || defaultValue.isNull()) { return null; } - return switch (type.typeId()) { - case BOOLEAN -> { + switch (type.typeId()) { + case BOOLEAN: Preconditions.checkArgument( defaultValue.isBoolean(), "Cannot parse default as a %s value: %s", type, defaultValue); - yield defaultValue.booleanValue(); - } - case INTEGER -> { + return defaultValue.booleanValue(); + case INTEGER: Preconditions.checkArgument( defaultValue.isIntegralNumber() && defaultValue.canConvertToInt(), "Cannot parse default as a %s value: %s", type, defaultValue); - yield defaultValue.intValue(); - } - case LONG -> { + return defaultValue.intValue(); + case LONG: Preconditions.checkArgument( defaultValue.isIntegralNumber() && defaultValue.canConvertToLong(), "Cannot parse default as a %s value: %s", type, defaultValue); - yield defaultValue.longValue(); - } - case FLOAT -> { + return defaultValue.longValue(); + case FLOAT: Preconditions.checkArgument( defaultValue.isFloatingPointNumber(), "Cannot parse default as a %s value: %s", type, defaultValue); - yield defaultValue.floatValue(); - } - case DOUBLE -> { + return defaultValue.floatValue(); + case DOUBLE: Preconditions.checkArgument( defaultValue.isFloatingPointNumber(), "Cannot parse default as a %s value: %s", type, defaultValue); - yield defaultValue.doubleValue(); - } - case DECIMAL -> { + return defaultValue.doubleValue(); + case DECIMAL: Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); BigDecimal retDecimal; @@ -105,14 +99,12 @@ public static Object fromJson(Type type, JsonNode defaultValue) { "Cannot parse default as a %s value: %s, the scale doesn't match", type, defaultValue); - yield retDecimal; - } - case STRING -> { + return retDecimal; + case STRING: Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); - yield defaultValue.textValue(); - } - case UUID -> { + return defaultValue.textValue(); + case UUID: Preconditions.checkArgument( defaultValue.isTextual() && defaultValue.textValue().length() == 36, "Cannot parse default as a %s value: %s", @@ -125,19 +117,16 @@ public static Object fromJson(Type type, JsonNode defaultValue) { throw new IllegalArgumentException( String.format("Cannot parse default as a %s value: %s", type, defaultValue), e); } - yield uuid; - } - case DATE -> { + return uuid; + case DATE: Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); - yield DateTimeUtil.isoDateToDays(defaultValue.textValue()); - } - case TIME -> { + return DateTimeUtil.isoDateToDays(defaultValue.textValue()); + case TIME: Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); - yield DateTimeUtil.isoTimeToMicros(defaultValue.textValue()); - } - case TIMESTAMP -> { + return DateTimeUtil.isoTimeToMicros(defaultValue.textValue()); + case TIMESTAMP: Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); if (((Types.TimestampType) type).shouldAdjustToUTC()) { @@ -147,12 +136,11 @@ public static Object fromJson(Type type, JsonNode defaultValue) { "Cannot parse default as a %s value: %s, offset must be +00:00", type, defaultValue); - yield DateTimeUtil.isoTimestamptzToMicros(timestampTz); + return DateTimeUtil.isoTimestamptzToMicros(timestampTz); } else { - yield DateTimeUtil.isoTimestampToMicros(defaultValue.textValue()); + return DateTimeUtil.isoTimestampToMicros(defaultValue.textValue()); } - } - case TIMESTAMP_NANO -> { + case TIMESTAMP_NANO: Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); if (((Types.TimestampNanoType) type).shouldAdjustToUTC()) { @@ -162,12 +150,11 @@ public static Object fromJson(Type type, JsonNode defaultValue) { "Cannot parse default as a %s value: %s, offset must be +00:00", type, defaultValue); - yield DateTimeUtil.isoTimestamptzToNanos(timestampTzNano); + return DateTimeUtil.isoTimestamptzToNanos(timestampTzNano); } else { - yield DateTimeUtil.isoTimestampToNanos(defaultValue.textValue()); + return DateTimeUtil.isoTimestampToNanos(defaultValue.textValue()); } - } - case FIXED -> { + case FIXED: Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); int defaultLength = defaultValue.textValue().length(); @@ -180,21 +167,22 @@ public static Object fromJson(Type type, JsonNode defaultValue) { defaultLength); byte[] fixedBytes = BaseEncoding.base16().decode(defaultValue.textValue().toUpperCase(Locale.ROOT)); - yield ByteBuffer.wrap(fixedBytes); - } - case BINARY -> { + return ByteBuffer.wrap(fixedBytes); + case BINARY: Preconditions.checkArgument( defaultValue.isTextual(), "Cannot parse default as a %s value: %s", type, defaultValue); byte[] binaryBytes = BaseEncoding.base16().decode(defaultValue.textValue().toUpperCase(Locale.ROOT)); - yield ByteBuffer.wrap(binaryBytes); - } - case LIST -> listFromJson(type, defaultValue); - case MAP -> mapFromJson(type, defaultValue); - case STRUCT -> structFromJson(type, defaultValue); - default -> - throw new UnsupportedOperationException(String.format("Type: %s is not supported", type)); - }; + return ByteBuffer.wrap(binaryBytes); + case LIST: + return listFromJson(type, defaultValue); + case MAP: + return mapFromJson(type, defaultValue); + case STRUCT: + return structFromJson(type, defaultValue); + default: + throw new UnsupportedOperationException(String.format("Type: %s is not supported", type)); + } } private static StructLike structFromJson(Type type, JsonNode defaultValue) { @@ -271,42 +259,42 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } switch (type.typeId()) { - case BOOLEAN -> { + case BOOLEAN: Preconditions.checkArgument( defaultValue instanceof Boolean, "Invalid default %s value: %s", type, defaultValue); generator.writeBoolean((Boolean) defaultValue); - } - case INTEGER -> { + break; + case INTEGER: Preconditions.checkArgument( defaultValue instanceof Integer, "Invalid default %s value: %s", type, defaultValue); generator.writeNumber((Integer) defaultValue); - } - case LONG -> { + break; + case LONG: Preconditions.checkArgument( defaultValue instanceof Long, "Invalid default %s value: %s", type, defaultValue); generator.writeNumber((Long) defaultValue); - } - case FLOAT -> { + break; + case FLOAT: Preconditions.checkArgument( defaultValue instanceof Float, "Invalid default %s value: %s", type, defaultValue); generator.writeNumber((Float) defaultValue); - } - case DOUBLE -> { + break; + case DOUBLE: Preconditions.checkArgument( defaultValue instanceof Double, "Invalid default %s value: %s", type, defaultValue); generator.writeNumber((Double) defaultValue); - } - case DATE -> { + break; + case DATE: Preconditions.checkArgument( defaultValue instanceof Integer, "Invalid default %s value: %s", type, defaultValue); generator.writeString(DateTimeUtil.daysToIsoDate((Integer) defaultValue)); - } - case TIME -> { + break; + case TIME: Preconditions.checkArgument( defaultValue instanceof Long, "Invalid default %s value: %s", type, defaultValue); generator.writeString(DateTimeUtil.microsToIsoTime((Long) defaultValue)); - } - case TIMESTAMP -> { + break; + case TIMESTAMP: Preconditions.checkArgument( defaultValue instanceof Long, "Invalid default %s value: %s", type, defaultValue); if (((Types.TimestampType) type).shouldAdjustToUTC()) { @@ -314,8 +302,8 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } else { generator.writeString(DateTimeUtil.microsToIsoTimestamp((Long) defaultValue)); } - } - case TIMESTAMP_NANO -> { + break; + case TIMESTAMP_NANO: Preconditions.checkArgument( defaultValue instanceof Long, "Invalid default %s value: %s", type, defaultValue); if (((Types.TimestampNanoType) type).shouldAdjustToUTC()) { @@ -323,21 +311,21 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } else { generator.writeString(DateTimeUtil.nanosToIsoTimestamp((Long) defaultValue)); } - } - case STRING -> { + break; + case STRING: Preconditions.checkArgument( defaultValue instanceof CharSequence, "Invalid default %s value: %s", type, defaultValue); generator.writeString(((CharSequence) defaultValue).toString()); - } - case UUID -> { + break; + case UUID: Preconditions.checkArgument( defaultValue instanceof UUID, "Invalid default %s value: %s", type, defaultValue); generator.writeString(defaultValue.toString()); - } - case FIXED -> { + break; + case FIXED: Preconditions.checkArgument( defaultValue instanceof ByteBuffer, "Invalid default %s value: %s", type, defaultValue); ByteBuffer byteBufferValue = (ByteBuffer) defaultValue; @@ -349,14 +337,14 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato byteBufferValue.remaining()); generator.writeString( BaseEncoding.base16().encode(ByteBuffers.toByteArray(byteBufferValue))); - } - case BINARY -> { + break; + case BINARY: Preconditions.checkArgument( defaultValue instanceof ByteBuffer, "Invalid default %s value: %s", type, defaultValue); generator.writeString( BaseEncoding.base16().encode(ByteBuffers.toByteArray((ByteBuffer) defaultValue))); - } - case DECIMAL -> { + break; + case DECIMAL: Preconditions.checkArgument( defaultValue instanceof BigDecimal && ((BigDecimal) defaultValue).scale() == ((Types.DecimalType) type).scale(), @@ -369,8 +357,8 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } else { generator.writeString(decimalValue.toString()); } - } - case LIST -> { + break; + case LIST: Preconditions.checkArgument( defaultValue instanceof List, "Invalid default %s value: %s", type, defaultValue); List defaultList = (List) defaultValue; @@ -380,8 +368,8 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato toJson(elementType, element, generator); } generator.writeEndArray(); - } - case MAP -> { + break; + case MAP: Preconditions.checkArgument( defaultValue instanceof Map, "Invalid default %s value: %s", type, defaultValue); Map defaultMap = (Map) defaultValue; @@ -402,8 +390,8 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } generator.writeEndArray(); generator.writeEndObject(); - } - case STRUCT -> { + break; + case STRUCT: Preconditions.checkArgument( defaultValue instanceof StructLike, "Invalid default %s value: %s", type, defaultValue); Types.StructType structType = type.asStructType(); @@ -421,9 +409,9 @@ public static void toJson(Type type, Object defaultValue, JsonGenerator generato } } generator.writeEndObject(); - } - default -> - throw new UnsupportedOperationException(String.format("Type: %s is not supported", type)); + break; + default: + throw new UnsupportedOperationException(String.format("Type: %s is not supported", type)); } } } diff --git a/core/src/main/java/org/apache/iceberg/SnapshotChanges.java b/core/src/main/java/org/apache/iceberg/SnapshotChanges.java index 857fee5b512a..38a81bb966c8 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotChanges.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotChanges.java @@ -139,8 +139,12 @@ private void cacheDataFileChanges() { iterate(manifestReadTasks)) { for (Pair pair : changedDataFiles) { switch (pair.first()) { - case ADDED -> adds.add(pair.second()); - case DELETED -> deletes.add(pair.second()); + case ADDED: + adds.add(pair.second()); + break; + case DELETED: + deletes.add(pair.second()); + break; } } } catch (IOException e) { @@ -186,8 +190,12 @@ private void cacheDeleteFileChanges() { iterate(manifestReadTasks)) { for (Pair pair : changedDeleteFiles) { switch (pair.first()) { - case ADDED -> adds.add(pair.second()); - case DELETED -> deletes.add(pair.second()); + case ADDED: + adds.add(pair.second()); + break; + case DELETED: + deletes.add(pair.second()); + break; } } } catch (IOException e) { diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index 676024613568..d97c63b61608 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -833,24 +833,24 @@ private static ManifestFile addMetadata(TableOperations ops, ManifestFile manife } switch (entry.status()) { - case ADDED -> { + case ADDED: addedFiles += 1; addedRows += entry.file().recordCount(); if (snapshotId == null) { snapshotId = entry.snapshotId(); } - } - case EXISTING -> { + break; + case EXISTING: existingFiles += 1; existingRows += entry.file().recordCount(); - } - case DELETED -> { + break; + case DELETED: deletedFiles += 1; deletedRows += entry.file().recordCount(); if (snapshotId == null) { snapshotId = entry.snapshotId(); } - } + break; } stats.update(entry.file().partition()); diff --git a/core/src/main/java/org/apache/iceberg/SnapshotSummary.java b/core/src/main/java/org/apache/iceberg/SnapshotSummary.java index fdecb787e4f7..392fad623f59 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotSummary.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotSummary.java @@ -291,11 +291,11 @@ void addTo(ImmutableMap.Builder builder) { void addedFile(ContentFile file) { this.addedSize += ScanTaskUtil.contentSizeInBytes(file); switch (file.content()) { - case DATA -> { + case DATA: this.addedFiles += 1; this.addedRecords += file.recordCount(); - } - case POSITION_DELETES -> { + break; + case POSITION_DELETES: DeleteFile deleteFile = (DeleteFile) file; if (ContentFileUtil.isDV(deleteFile)) { this.addedDVs += 1; @@ -304,26 +304,26 @@ void addedFile(ContentFile file) { } this.addedDeleteFiles += 1; this.addedPosDeletes += file.recordCount(); - } - case EQUALITY_DELETES -> { + break; + case EQUALITY_DELETES: this.addedDeleteFiles += 1; this.addedEqDeleteFiles += 1; this.addedEqDeletes += file.recordCount(); - } - default -> - throw new UnsupportedOperationException( - "Unsupported file content type: " + file.content()); + break; + default: + throw new UnsupportedOperationException( + "Unsupported file content type: " + file.content()); } } void removedFile(ContentFile file) { this.removedSize += ScanTaskUtil.contentSizeInBytes(file); switch (file.content()) { - case DATA -> { + case DATA: this.removedFiles += 1; this.deletedRecords += file.recordCount(); - } - case POSITION_DELETES -> { + break; + case POSITION_DELETES: DeleteFile deleteFile = (DeleteFile) file; if (ContentFileUtil.isDV(deleteFile)) { this.removedDVs += 1; @@ -332,31 +332,31 @@ void removedFile(ContentFile file) { } this.removedDeleteFiles += 1; this.removedPosDeletes += file.recordCount(); - } - case EQUALITY_DELETES -> { + break; + case EQUALITY_DELETES: this.removedDeleteFiles += 1; this.removedEqDeleteFiles += 1; this.removedEqDeletes += file.recordCount(); - } - default -> - throw new UnsupportedOperationException( - "Unsupported file content type: " + file.content()); + break; + default: + throw new UnsupportedOperationException( + "Unsupported file content type: " + file.content()); } } void addedManifest(ManifestFile manifest) { switch (manifest.content()) { - case DATA -> { + case DATA: this.addedFiles += manifest.addedFilesCount(); this.addedRecords += manifest.addedRowsCount(); this.removedFiles += manifest.deletedFilesCount(); this.deletedRecords += manifest.deletedRowsCount(); - } - case DELETES -> { + break; + case DELETES: this.addedDeleteFiles += manifest.addedFilesCount(); this.removedDeleteFiles += manifest.deletedFilesCount(); this.trustSizeAndDeleteCounts = false; - } + break; } } diff --git a/core/src/main/java/org/apache/iceberg/SortOrderParser.java b/core/src/main/java/org/apache/iceberg/SortOrderParser.java index 695621a8e3c7..53d7e5090c76 100644 --- a/core/src/main/java/org/apache/iceberg/SortOrderParser.java +++ b/core/src/main/java/org/apache/iceberg/SortOrderParser.java @@ -168,11 +168,13 @@ private static void buildFromJsonFields(UnboundSortOrder.Builder builder, JsonNo } private static NullOrder toNullOrder(String nullOrderingAsString) { - return switch (nullOrderingAsString.toLowerCase(Locale.ROOT)) { - case "nulls-first" -> NULLS_FIRST; - case "nulls-last" -> NULLS_LAST; - default -> - throw new IllegalArgumentException("Unexpected null order: " + nullOrderingAsString); - }; + switch (nullOrderingAsString.toLowerCase(Locale.ROOT)) { + case "nulls-first": + return NULLS_FIRST; + case "nulls-last": + return NULLS_LAST; + default: + throw new IllegalArgumentException("Unexpected null order: " + nullOrderingAsString); + } } } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 76b16ad7439a..9a44e8045cbb 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -274,51 +274,98 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - return switch (pos) { - case 0 -> tracking; - case 1 -> contentType != null ? contentType.id() : null; - case 2 -> writerFormatVersion; - case 3 -> location; - case 4 -> fileFormat != null ? fileFormat.toString() : null; - case 5 -> recordCount; - case 6 -> fileSizeInBytes; - case 7 -> specId; - case 8 -> partitionData; - case 9 -> contentStats; - case 10 -> sortOrderId; - case 11 -> deletionVector; - case 12 -> manifestInfo; - case 13 -> keyMetadata(); - case 14 -> splitOffsets(); - case 15 -> equalityIds(); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + switch (pos) { + case 0: + return tracking; + case 1: + return contentType != null ? contentType.id() : null; + case 2: + return writerFormatVersion; + case 3: + return location; + case 4: + return fileFormat != null ? fileFormat.toString() : null; + case 5: + return recordCount; + case 6: + return fileSizeInBytes; + case 7: + return specId; + case 8: + return partitionData; + case 9: + return contentStats; + case 10: + return sortOrderId; + case 11: + return deletionVector; + case 12: + return manifestInfo; + case 13: + return keyMetadata(); + case 14: + return splitOffsets(); + case 15: + return equalityIds(); + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0 -> this.tracking = (Tracking) value; - case 1 -> this.contentType = FileContent.fromId((Integer) value); - case 2 -> this.writerFormatVersion = (int) value; - case 3 -> - // always coerce to String for Serializable - this.location = value.toString(); - case 4 -> this.fileFormat = FileFormat.fromString(value.toString()); - case 5 -> this.recordCount = (long) value; - case 6 -> this.fileSizeInBytes = (long) value; - case 7 -> this.specId = (Integer) value; - case 8 -> this.partitionData = (PartitionData) value; - case 9 -> this.contentStats = (ContentStats) value; - case 10 -> this.sortOrderId = (Integer) value; - case 11 -> this.deletionVector = (DeletionVector) value; - case 12 -> this.manifestInfo = (ManifestInfo) value; - case 13 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); - case 14 -> this.splitOffsets = ArrayUtil.toLongArray((List) value); - case 15 -> this.equalityIds = ArrayUtil.toIntArray((List) value); - default -> { + case 0: + this.tracking = (Tracking) value; + break; + case 1: + this.contentType = FileContent.fromId((Integer) value); + break; + case 2: + this.writerFormatVersion = (int) value; + break; + case 3: + // always coerce to String for Serializable + this.location = value.toString(); + break; + case 4: + this.fileFormat = FileFormat.fromString(value.toString()); + break; + case 5: + this.recordCount = (long) value; + break; + case 6: + this.fileSizeInBytes = (long) value; + break; + case 7: + this.specId = (Integer) value; + break; + case 8: + this.partitionData = (PartitionData) value; + break; + case 9: + this.contentStats = (ContentStats) value; + break; + case 10: + this.sortOrderId = (Integer) value; + break; + case 11: + this.deletionVector = (DeletionVector) value; + break; + case 12: + this.manifestInfo = (ManifestInfo) value; + break; + case 13: + this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + break; + case 14: + this.splitOffsets = ArrayUtil.toLongArray((List) value); + break; + case 15: + this.equalityIds = ArrayUtil.toIntArray((List) value); + break; + default: // ignore the object, it must be from a newer version of the format - } } } diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java index b4fae388c96b..8ae4b7e4ce88 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java @@ -195,35 +195,62 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - return switch (pos) { - case 0 -> status != null ? status.id() : null; - case 1 -> snapshotId(); - case 2 -> dataSequenceNumber(); - case 3 -> fileSequenceNumber(); - case 4 -> dvSnapshotId; - case 5 -> firstRowId; - case 6 -> deletedPositions(); - case 7 -> replacedPositions(); - case 8 -> manifestPos; - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + switch (pos) { + case 0: + return status != null ? status.id() : null; + case 1: + return snapshotId(); + case 2: + return dataSequenceNumber(); + case 3: + return fileSequenceNumber(); + case 4: + return dvSnapshotId; + case 5: + return firstRowId; + case 6: + return deletedPositions(); + case 7: + return replacedPositions(); + case 8: + return manifestPos; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0 -> this.status = EntryStatus.fromId((Integer) value); - case 1 -> this.snapshotId = (Long) value; - case 2 -> this.dataSequenceNumber = (Long) value; - case 3 -> this.fileSequenceNumber = (Long) value; - case 4 -> this.dvSnapshotId = (Long) value; - case 5 -> this.firstRowId = (Long) value; - case 6 -> this.deletedPositions = ByteBuffers.toByteArray((ByteBuffer) value); - case 7 -> this.replacedPositions = ByteBuffers.toByteArray((ByteBuffer) value); - case 8 -> this.manifestPos = (long) value; - default -> { + case 0: + this.status = EntryStatus.fromId((Integer) value); + break; + case 1: + this.snapshotId = (Long) value; + break; + case 2: + this.dataSequenceNumber = (Long) value; + break; + case 3: + this.fileSequenceNumber = (Long) value; + break; + case 4: + this.dvSnapshotId = (Long) value; + break; + case 5: + this.firstRowId = (Long) value; + break; + case 6: + this.deletedPositions = ByteBuffers.toByteArray((ByteBuffer) value); + break; + case 7: + this.replacedPositions = ByteBuffers.toByteArray((ByteBuffer) value); + break; + case 8: + this.manifestPos = (long) value; + break; + default: // ignore the object, it must be from a newer version of the format - } } } diff --git a/core/src/main/java/org/apache/iceberg/UpdateRequirementParser.java b/core/src/main/java/org/apache/iceberg/UpdateRequirementParser.java index d4a6e0d49039..5c4dc2221290 100644 --- a/core/src/main/java/org/apache/iceberg/UpdateRequirementParser.java +++ b/core/src/main/java/org/apache/iceberg/UpdateRequirementParser.java @@ -97,36 +97,44 @@ public static void toJson(UpdateRequirement updateRequirement, JsonGenerator gen generator.writeStringField(TYPE, requirementType); switch (requirementType) { - case ASSERT_TABLE_DOES_NOT_EXIST -> { + case ASSERT_TABLE_DOES_NOT_EXIST: // No fields beyond the requirement itself - } - case ASSERT_TABLE_UUID -> - writeAssertTableUUID((UpdateRequirement.AssertTableUUID) updateRequirement, generator); - case ASSERT_VIEW_UUID -> - writeAssertViewUUID((UpdateRequirement.AssertViewUUID) updateRequirement, generator); - case ASSERT_REF_SNAPSHOT_ID -> - writeAssertRefSnapshotId( - (UpdateRequirement.AssertRefSnapshotID) updateRequirement, generator); - case ASSERT_LAST_ASSIGNED_FIELD_ID -> - writeAssertLastAssignedFieldId( - (UpdateRequirement.AssertLastAssignedFieldId) updateRequirement, generator); - case ASSERT_LAST_ASSIGNED_PARTITION_ID -> - writeAssertLastAssignedPartitionId( - (UpdateRequirement.AssertLastAssignedPartitionId) updateRequirement, generator); - case ASSERT_CURRENT_SCHEMA_ID -> - writeAssertCurrentSchemaId( - (UpdateRequirement.AssertCurrentSchemaID) updateRequirement, generator); - case ASSERT_DEFAULT_SPEC_ID -> - writeAssertDefaultSpecId( - (UpdateRequirement.AssertDefaultSpecID) updateRequirement, generator); - case ASSERT_DEFAULT_SORT_ORDER_ID -> - writeAssertDefaultSortOrderId( - (UpdateRequirement.AssertDefaultSortOrderID) updateRequirement, generator); - default -> - throw new IllegalArgumentException( - String.format( - "Cannot convert update requirement to json. Unrecognized type: %s", - requirementType)); + break; + case ASSERT_TABLE_UUID: + writeAssertTableUUID((UpdateRequirement.AssertTableUUID) updateRequirement, generator); + break; + case ASSERT_VIEW_UUID: + writeAssertViewUUID((UpdateRequirement.AssertViewUUID) updateRequirement, generator); + break; + case ASSERT_REF_SNAPSHOT_ID: + writeAssertRefSnapshotId( + (UpdateRequirement.AssertRefSnapshotID) updateRequirement, generator); + break; + case ASSERT_LAST_ASSIGNED_FIELD_ID: + writeAssertLastAssignedFieldId( + (UpdateRequirement.AssertLastAssignedFieldId) updateRequirement, generator); + break; + case ASSERT_LAST_ASSIGNED_PARTITION_ID: + writeAssertLastAssignedPartitionId( + (UpdateRequirement.AssertLastAssignedPartitionId) updateRequirement, generator); + break; + case ASSERT_CURRENT_SCHEMA_ID: + writeAssertCurrentSchemaId( + (UpdateRequirement.AssertCurrentSchemaID) updateRequirement, generator); + break; + case ASSERT_DEFAULT_SPEC_ID: + writeAssertDefaultSpecId( + (UpdateRequirement.AssertDefaultSpecID) updateRequirement, generator); + break; + case ASSERT_DEFAULT_SORT_ORDER_ID: + writeAssertDefaultSortOrderId( + (UpdateRequirement.AssertDefaultSortOrderID) updateRequirement, generator); + break; + default: + throw new IllegalArgumentException( + String.format( + "Cannot convert update requirement to json. Unrecognized type: %s", + requirementType)); } generator.writeEndObject(); @@ -151,20 +159,29 @@ public static UpdateRequirement fromJson(JsonNode jsonNode) { jsonNode.hasNonNull(TYPE), "Cannot parse update requirement. Missing field: type"); String type = JsonUtil.getString(TYPE, jsonNode).toLowerCase(Locale.ROOT); - return switch (type) { - case ASSERT_TABLE_DOES_NOT_EXIST -> readAssertTableDoesNotExist(jsonNode); - case ASSERT_TABLE_UUID -> readAssertTableUUID(jsonNode); - case ASSERT_VIEW_UUID -> readAssertViewUUID(jsonNode); - case ASSERT_REF_SNAPSHOT_ID -> readAssertRefSnapshotId(jsonNode); - case ASSERT_LAST_ASSIGNED_FIELD_ID -> readAssertLastAssignedFieldId(jsonNode); - case ASSERT_LAST_ASSIGNED_PARTITION_ID -> readAssertLastAssignedPartitionId(jsonNode); - case ASSERT_CURRENT_SCHEMA_ID -> readAssertCurrentSchemaId(jsonNode); - case ASSERT_DEFAULT_SPEC_ID -> readAssertDefaultSpecId(jsonNode); - case ASSERT_DEFAULT_SORT_ORDER_ID -> readAssertDefaultSortOrderId(jsonNode); - default -> - throw new UnsupportedOperationException( - String.format("Unrecognized update requirement. Cannot convert to json: %s", type)); - }; + switch (type) { + case ASSERT_TABLE_DOES_NOT_EXIST: + return readAssertTableDoesNotExist(jsonNode); + case ASSERT_TABLE_UUID: + return readAssertTableUUID(jsonNode); + case ASSERT_VIEW_UUID: + return readAssertViewUUID(jsonNode); + case ASSERT_REF_SNAPSHOT_ID: + return readAssertRefSnapshotId(jsonNode); + case ASSERT_LAST_ASSIGNED_FIELD_ID: + return readAssertLastAssignedFieldId(jsonNode); + case ASSERT_LAST_ASSIGNED_PARTITION_ID: + return readAssertLastAssignedPartitionId(jsonNode); + case ASSERT_CURRENT_SCHEMA_ID: + return readAssertCurrentSchemaId(jsonNode); + case ASSERT_DEFAULT_SPEC_ID: + return readAssertDefaultSpecId(jsonNode); + case ASSERT_DEFAULT_SORT_ORDER_ID: + return readAssertDefaultSortOrderId(jsonNode); + default: + throw new UnsupportedOperationException( + String.format("Unrecognized update requirement. Cannot convert to json: %s", type)); + } } private static void writeAssertTableUUID( diff --git a/core/src/main/java/org/apache/iceberg/V1Metadata.java b/core/src/main/java/org/apache/iceberg/V1Metadata.java index de7bcf56d668..34bcd691a9cf 100644 --- a/core/src/main/java/org/apache/iceberg/V1Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V1Metadata.java @@ -74,21 +74,34 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> path(); - case 1 -> length(); - case 2 -> partitionSpecId(); - case 3 -> snapshotId(); - case 4 -> addedFilesCount(); - case 5 -> existingFilesCount(); - case 6 -> deletedFilesCount(); - case 7 -> partitions(); - case 8 -> addedRowsCount(); - case 9 -> existingRowsCount(); - case 10 -> deletedRowsCount(); - case 11 -> keyMetadata(); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + switch (pos) { + case 0: + return path(); + case 1: + return length(); + case 2: + return partitionSpecId(); + case 3: + return snapshotId(); + case 4: + return addedFilesCount(); + case 5: + return existingFilesCount(); + case 6: + return deletedFilesCount(); + case 7: + return partitions(); + case 8: + return addedRowsCount(); + case 9: + return existingRowsCount(); + case 10: + return deletedRowsCount(); + case 11: + return keyMetadata(); + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -253,18 +266,20 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.status().id(); - case 1 -> wrapped.snapshotId(); - case 2 -> { + switch (pos) { + case 0: + return wrapped.status().id(); + case 1: + return wrapped.snapshotId(); + case 2: DataFile file = wrapped.file(); if (file != null) { - yield fileWrapper.wrap(file); + return fileWrapper.wrap(file); } - yield null; - } - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + return null; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -349,24 +364,39 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.location(); - case 1 -> wrapped.format() != null ? wrapped.format().toString() : null; - case 2 -> wrapped.partition(); - case 3 -> wrapped.recordCount(); - case 4 -> wrapped.fileSizeInBytes(); - case 5 -> DEFAULT_BLOCK_SIZE; - case 6 -> wrapped.columnSizes(); - case 7 -> wrapped.valueCounts(); - case 8 -> wrapped.nullValueCounts(); - case 9 -> wrapped.nanValueCounts(); - case 10 -> wrapped.lowerBounds(); - case 11 -> wrapped.upperBounds(); - case 12 -> wrapped.keyMetadata(); - case 13 -> wrapped.splitOffsets(); - case 14 -> wrapped.sortOrderId(); - default -> throw new IllegalArgumentException("Unknown field ordinal: " + pos); - }; + switch (pos) { + case 0: + return wrapped.location(); + case 1: + return wrapped.format() != null ? wrapped.format().toString() : null; + case 2: + return wrapped.partition(); + case 3: + return wrapped.recordCount(); + case 4: + return wrapped.fileSizeInBytes(); + case 5: + return DEFAULT_BLOCK_SIZE; + case 6: + return wrapped.columnSizes(); + case 7: + return wrapped.valueCounts(); + case 8: + return wrapped.nullValueCounts(); + case 9: + return wrapped.nanValueCounts(); + case 10: + return wrapped.lowerBounds(); + case 11: + return wrapped.upperBounds(); + case 12: + return wrapped.keyMetadata(); + case 13: + return wrapped.splitOffsets(); + case 14: + return wrapped.sortOrderId(); + } + throw new IllegalArgumentException("Unknown field ordinal: " + pos); } @Override diff --git a/core/src/main/java/org/apache/iceberg/V2Metadata.java b/core/src/main/java/org/apache/iceberg/V2Metadata.java index b22297c64f1c..803905f6b42e 100644 --- a/core/src/main/java/org/apache/iceberg/V2Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V2Metadata.java @@ -85,15 +85,17 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.path(); - case 1 -> wrapped.length(); - case 2 -> wrapped.partitionSpecId(); - case 3 -> { + switch (pos) { + case 0: + return wrapped.path(); + case 1: + return wrapped.length(); + case 2: + return wrapped.partitionSpecId(); + case 3: checkContentType(wrapped.content()); - yield wrapped.content().id(); - } - case 4 -> { + return wrapped.content().id(); + case 4: if (wrapped.sequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // if the sequence number is being assigned here, then the manifest must be created by // the current @@ -102,12 +104,11 @@ private Object get(int pos) { commitSnapshotId == wrapped.snapshotId(), "Found unassigned sequence number for a manifest from snapshot: %s", wrapped.snapshotId()); - yield sequenceNumber; + return sequenceNumber; } else { - yield wrapped.sequenceNumber(); + return wrapped.sequenceNumber(); } - } - case 5 -> { + case 5: if (wrapped.minSequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // same sanity check as above Preconditions.checkState( @@ -118,22 +119,31 @@ private Object get(int pos) { // number for any file // written to the wrapped manifest. replace the unassigned sequence number with the one // for this commit - yield sequenceNumber; + return sequenceNumber; } else { - yield wrapped.minSequenceNumber(); + return wrapped.minSequenceNumber(); } - } - case 6 -> wrapped.snapshotId(); - case 7 -> wrapped.addedFilesCount(); - case 8 -> wrapped.existingFilesCount(); - case 9 -> wrapped.deletedFilesCount(); - case 10 -> wrapped.addedRowsCount(); - case 11 -> wrapped.existingRowsCount(); - case 12 -> wrapped.deletedRowsCount(); - case 13 -> wrapped.partitions(); - case 14 -> wrapped.keyMetadata(); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + case 6: + return wrapped.snapshotId(); + case 7: + return wrapped.addedFilesCount(); + case 8: + return wrapped.existingFilesCount(); + case 9: + return wrapped.deletedFilesCount(); + case 10: + return wrapped.addedRowsCount(); + case 11: + return wrapped.existingRowsCount(); + case 12: + return wrapped.deletedRowsCount(); + case 13: + return wrapped.partitions(); + case 14: + return wrapped.keyMetadata(); + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -302,10 +312,12 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.status().id(); - case 1 -> wrapped.snapshotId(); - case 2 -> { + switch (pos) { + case 0: + return wrapped.status().id(); + case 1: + return wrapped.snapshotId(); + case 2: if (wrapped.dataSequenceNumber() == null) { // if the entry's data sequence number is null, // then it will inherit the sequence number of the current commit. @@ -321,14 +333,16 @@ private Object get(int pos) { wrapped.status() == Status.ADDED, "Only entries with status ADDED can have null sequence number"); - yield null; + return null; } - yield wrapped.dataSequenceNumber(); - } - case 3 -> wrapped.fileSequenceNumber(); - case 4 -> fileWrapper.wrap(wrapped.file()); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + return wrapped.dataSequenceNumber(); + case 3: + return wrapped.fileSequenceNumber(); + case 4: + return fileWrapper.wrap(wrapped.file()); + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -413,35 +427,48 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> { + switch (pos) { + case 0: checkContentType(wrapped.content()); - yield wrapped.content().id(); - } - case 1 -> wrapped.location(); - case 2 -> wrapped.format() != null ? wrapped.format().toString() : null; - case 3 -> wrapped.partition(); - case 4 -> wrapped.recordCount(); - case 5 -> wrapped.fileSizeInBytes(); - case 6 -> wrapped.columnSizes(); - case 7 -> wrapped.valueCounts(); - case 8 -> wrapped.nullValueCounts(); - case 9 -> wrapped.nanValueCounts(); - case 10 -> wrapped.lowerBounds(); - case 11 -> wrapped.upperBounds(); - case 12 -> wrapped.keyMetadata(); - case 13 -> wrapped.splitOffsets(); - case 14 -> wrapped.equalityFieldIds(); - case 15 -> wrapped.sortOrderId(); - case 16 -> { + return wrapped.content().id(); + case 1: + return wrapped.location(); + case 2: + return wrapped.format() != null ? wrapped.format().toString() : null; + case 3: + return wrapped.partition(); + case 4: + return wrapped.recordCount(); + case 5: + return wrapped.fileSizeInBytes(); + case 6: + return wrapped.columnSizes(); + case 7: + return wrapped.valueCounts(); + case 8: + return wrapped.nullValueCounts(); + case 9: + return wrapped.nanValueCounts(); + case 10: + return wrapped.lowerBounds(); + case 11: + return wrapped.upperBounds(); + case 12: + return wrapped.keyMetadata(); + case 13: + return wrapped.splitOffsets(); + case 14: + return wrapped.equalityFieldIds(); + case 15: + return wrapped.sortOrderId(); + case 16: if (wrapped instanceof DeleteFile) { - yield ((DeleteFile) wrapped).referencedDataFile(); + return ((DeleteFile) wrapped).referencedDataFile(); } else { - yield null; + return null; } - } - default -> throw new IllegalArgumentException("Unknown field ordinal: " + pos); - }; + } + throw new IllegalArgumentException("Unknown field ordinal: " + pos); } @Override diff --git a/core/src/main/java/org/apache/iceberg/V3Metadata.java b/core/src/main/java/org/apache/iceberg/V3Metadata.java index ea75b2952b02..4e67d9977e64 100644 --- a/core/src/main/java/org/apache/iceberg/V3Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V3Metadata.java @@ -86,15 +86,17 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.path(); - case 1 -> wrapped.length(); - case 2 -> wrapped.partitionSpecId(); - case 3 -> { + switch (pos) { + case 0: + return wrapped.path(); + case 1: + return wrapped.length(); + case 2: + return wrapped.partitionSpecId(); + case 3: checkContentType(wrapped.content()); - yield wrapped.content().id(); - } - case 4 -> { + return wrapped.content().id(); + case 4: if (wrapped.sequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // if the sequence number is being assigned here, then the manifest must be created by // the current @@ -103,12 +105,11 @@ private Object get(int pos) { commitSnapshotId == wrapped.snapshotId(), "Found unassigned sequence number for a manifest from snapshot: %s", wrapped.snapshotId()); - yield sequenceNumber; + return sequenceNumber; } else { - yield wrapped.sequenceNumber(); + return wrapped.sequenceNumber(); } - } - case 5 -> { + case 5: if (wrapped.minSequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // same sanity check as above Preconditions.checkState( @@ -119,39 +120,47 @@ private Object get(int pos) { // number for any file // written to the wrapped manifest. replace the unassigned sequence number with the one // for this commit - yield sequenceNumber; + return sequenceNumber; } else { - yield wrapped.minSequenceNumber(); + return wrapped.minSequenceNumber(); } - } - case 6 -> wrapped.snapshotId(); - case 7 -> wrapped.addedFilesCount(); - case 8 -> wrapped.existingFilesCount(); - case 9 -> wrapped.deletedFilesCount(); - case 10 -> wrapped.addedRowsCount(); - case 11 -> wrapped.existingRowsCount(); - case 12 -> wrapped.deletedRowsCount(); - case 13 -> wrapped.partitions(); - case 14 -> wrapped.keyMetadata(); - case 15 -> { + case 6: + return wrapped.snapshotId(); + case 7: + return wrapped.addedFilesCount(); + case 8: + return wrapped.existingFilesCount(); + case 9: + return wrapped.deletedFilesCount(); + case 10: + return wrapped.addedRowsCount(); + case 11: + return wrapped.existingRowsCount(); + case 12: + return wrapped.deletedRowsCount(); + case 13: + return wrapped.partitions(); + case 14: + return wrapped.keyMetadata(); + case 15: if (wrappedFirstRowId != null) { // if first-row-id is assigned, ensure that it is valid Preconditions.checkState( wrapped.content() == ManifestContent.DATA && wrapped.firstRowId() == null, "Found invalid first-row-id assignment: %s", wrapped); - yield wrappedFirstRowId; + return wrappedFirstRowId; } else if (wrapped.content() != ManifestContent.DATA) { - yield null; + return null; } else { Preconditions.checkState( wrapped.firstRowId() != null, "Found unassigned first-row-id for file: " + wrapped.path()); - yield wrapped.firstRowId(); + return wrapped.firstRowId(); } - } - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -328,10 +337,12 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.status().id(); - case 1 -> wrapped.snapshotId(); - case 2 -> { + switch (pos) { + case 0: + return wrapped.status().id(); + case 1: + return wrapped.snapshotId(); + case 2: if (wrapped.dataSequenceNumber() == null) { // if the entry's data sequence number is null, // then it will inherit the sequence number of the current commit. @@ -347,14 +358,16 @@ private Object get(int pos) { wrapped.status() == Status.ADDED, "Only entries with status ADDED can have null sequence number"); - yield null; + return null; } - yield wrapped.dataSequenceNumber(); - } - case 3 -> wrapped.fileSequenceNumber(); - case 4 -> fileWrapper.wrap(wrapped.file()); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + return wrapped.dataSequenceNumber(); + case 3: + return wrapped.fileSequenceNumber(); + case 4: + return fileWrapper.wrap(wrapped.file()); + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -440,56 +453,66 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> { + switch (pos) { + case 0: checkContentType(wrapped.content()); - yield wrapped.content().id(); - } - case 1 -> wrapped.location(); - case 2 -> wrapped.format() != null ? wrapped.format().toString() : null; - case 3 -> wrapped.partition(); - case 4 -> wrapped.recordCount(); - case 5 -> wrapped.fileSizeInBytes(); - case 6 -> wrapped.columnSizes(); - case 7 -> wrapped.valueCounts(); - case 8 -> wrapped.nullValueCounts(); - case 9 -> wrapped.nanValueCounts(); - case 10 -> wrapped.lowerBounds(); - case 11 -> wrapped.upperBounds(); - case 12 -> wrapped.keyMetadata(); - case 13 -> wrapped.splitOffsets(); - case 14 -> wrapped.equalityFieldIds(); - case 15 -> wrapped.sortOrderId(); - case 16 -> { + return wrapped.content().id(); + case 1: + return wrapped.location(); + case 2: + return wrapped.format() != null ? wrapped.format().toString() : null; + case 3: + return wrapped.partition(); + case 4: + return wrapped.recordCount(); + case 5: + return wrapped.fileSizeInBytes(); + case 6: + return wrapped.columnSizes(); + case 7: + return wrapped.valueCounts(); + case 8: + return wrapped.nullValueCounts(); + case 9: + return wrapped.nanValueCounts(); + case 10: + return wrapped.lowerBounds(); + case 11: + return wrapped.upperBounds(); + case 12: + return wrapped.keyMetadata(); + case 13: + return wrapped.splitOffsets(); + case 14: + return wrapped.equalityFieldIds(); + case 15: + return wrapped.sortOrderId(); + case 16: if (wrapped.content() == FileContent.DATA) { - yield wrapped.firstRowId(); + return wrapped.firstRowId(); } else { - yield null; + return null; } - } - case 17 -> { + case 17: if (wrapped.content() == FileContent.POSITION_DELETES) { - yield ((DeleteFile) wrapped).referencedDataFile(); + return ((DeleteFile) wrapped).referencedDataFile(); } else { - yield null; + return null; } - } - case 18 -> { + case 18: if (wrapped.content() == FileContent.POSITION_DELETES) { - yield ((DeleteFile) wrapped).contentOffset(); + return ((DeleteFile) wrapped).contentOffset(); } else { - yield null; + return null; } - } - case 19 -> { + case 19: if (wrapped.content() == FileContent.POSITION_DELETES) { - yield ((DeleteFile) wrapped).contentSizeInBytes(); + return ((DeleteFile) wrapped).contentSizeInBytes(); } else { - yield null; + return null; } - } - default -> throw new IllegalArgumentException("Unknown field ordinal: " + pos); - }; + } + throw new IllegalArgumentException("Unknown field ordinal: " + pos); } @Override diff --git a/core/src/main/java/org/apache/iceberg/V4Metadata.java b/core/src/main/java/org/apache/iceberg/V4Metadata.java index c7a121ef735d..06fc75213df0 100644 --- a/core/src/main/java/org/apache/iceberg/V4Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V4Metadata.java @@ -87,12 +87,16 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.path(); - case 1 -> wrapped.length(); - case 2 -> wrapped.partitionSpecId(); - case 3 -> wrapped.content().id(); - case 4 -> { + switch (pos) { + case 0: + return wrapped.path(); + case 1: + return wrapped.length(); + case 2: + return wrapped.partitionSpecId(); + case 3: + return wrapped.content().id(); + case 4: if (wrapped.sequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // if the sequence number is being assigned here, then the manifest must be created by // the current @@ -101,12 +105,11 @@ private Object get(int pos) { commitSnapshotId == wrapped.snapshotId(), "Found unassigned sequence number for a manifest from snapshot: %s", wrapped.snapshotId()); - yield sequenceNumber; + return sequenceNumber; } else { - yield wrapped.sequenceNumber(); + return wrapped.sequenceNumber(); } - } - case 5 -> { + case 5: if (wrapped.minSequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) { // same sanity check as above Preconditions.checkState( @@ -117,39 +120,47 @@ private Object get(int pos) { // number for any file // written to the wrapped manifest. replace the unassigned sequence number with the one // for this commit - yield sequenceNumber; + return sequenceNumber; } else { - yield wrapped.minSequenceNumber(); + return wrapped.minSequenceNumber(); } - } - case 6 -> wrapped.snapshotId(); - case 7 -> wrapped.addedFilesCount(); - case 8 -> wrapped.existingFilesCount(); - case 9 -> wrapped.deletedFilesCount(); - case 10 -> wrapped.addedRowsCount(); - case 11 -> wrapped.existingRowsCount(); - case 12 -> wrapped.deletedRowsCount(); - case 13 -> wrapped.partitions(); - case 14 -> wrapped.keyMetadata(); - case 15 -> { + case 6: + return wrapped.snapshotId(); + case 7: + return wrapped.addedFilesCount(); + case 8: + return wrapped.existingFilesCount(); + case 9: + return wrapped.deletedFilesCount(); + case 10: + return wrapped.addedRowsCount(); + case 11: + return wrapped.existingRowsCount(); + case 12: + return wrapped.deletedRowsCount(); + case 13: + return wrapped.partitions(); + case 14: + return wrapped.keyMetadata(); + case 15: if (wrappedFirstRowId != null) { // if first-row-id is assigned, ensure that it is valid Preconditions.checkState( wrapped.content() == ManifestContent.DATA && wrapped.firstRowId() == null, "Found invalid first-row-id assignment: %s", wrapped); - yield wrappedFirstRowId; + return wrappedFirstRowId; } else if (wrapped.content() != ManifestContent.DATA) { - yield null; + return null; } else { Preconditions.checkState( wrapped.firstRowId() != null, "Found unassigned first-row-id for file: " + wrapped.path()); - yield wrapped.firstRowId(); + return wrapped.firstRowId(); } - } - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -336,10 +347,12 @@ public T get(int pos, Class javaClass) { } private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.status().id(); - case 1 -> wrapped.snapshotId(); - case 2 -> { + switch (pos) { + case 0: + return wrapped.status().id(); + case 1: + return wrapped.snapshotId(); + case 2: if (wrapped.dataSequenceNumber() == null) { // if the entry's data sequence number is null, // then it will inherit the sequence number of the current commit. @@ -355,14 +368,16 @@ private Object get(int pos) { wrapped.status() == Status.ADDED, "Only entries with status ADDED can have null sequence number"); - yield null; + return null; } - yield wrapped.dataSequenceNumber(); - } - case 3 -> wrapped.fileSequenceNumber(); - case 4 -> fileWrapper.wrap(wrapped.file()); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + return wrapped.dataSequenceNumber(); + case 3: + return wrapped.fileSequenceNumber(); + case 4: + return fileWrapper.wrap(wrapped.file()); + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + } } @Override @@ -455,53 +470,65 @@ private Object get(int pos) { // when the partition field is omitted, positions at or after where it would appear // shift down by 1, so adjust back to the canonical field ordering int adjusted = hasPartition ? pos : (pos >= PARTITION_POSITION ? pos + 1 : pos); - return switch (adjusted) { - case 0 -> wrapped.content().id(); - case 1 -> wrapped.location(); - case 2 -> wrapped.format() != null ? wrapped.format().toString() : null; - case 3 -> wrapped.partition(); - case 4 -> wrapped.recordCount(); - case 5 -> wrapped.fileSizeInBytes(); - case 6 -> wrapped.columnSizes(); - case 7 -> wrapped.valueCounts(); - case 8 -> wrapped.nullValueCounts(); - case 9 -> wrapped.nanValueCounts(); - case 10 -> wrapped.lowerBounds(); - case 11 -> wrapped.upperBounds(); - case 12 -> wrapped.keyMetadata(); - case 13 -> wrapped.splitOffsets(); - case 14 -> wrapped.equalityFieldIds(); - case 15 -> wrapped.sortOrderId(); - case 16 -> { + switch (adjusted) { + case 0: + return wrapped.content().id(); + case 1: + return wrapped.location(); + case 2: + return wrapped.format() != null ? wrapped.format().toString() : null; + case 3: + return wrapped.partition(); + case 4: + return wrapped.recordCount(); + case 5: + return wrapped.fileSizeInBytes(); + case 6: + return wrapped.columnSizes(); + case 7: + return wrapped.valueCounts(); + case 8: + return wrapped.nullValueCounts(); + case 9: + return wrapped.nanValueCounts(); + case 10: + return wrapped.lowerBounds(); + case 11: + return wrapped.upperBounds(); + case 12: + return wrapped.keyMetadata(); + case 13: + return wrapped.splitOffsets(); + case 14: + return wrapped.equalityFieldIds(); + case 15: + return wrapped.sortOrderId(); + case 16: if (wrapped.content() == FileContent.DATA) { - yield wrapped.firstRowId(); + return wrapped.firstRowId(); } else { - yield null; + return null; } - } - case 17 -> { + case 17: if (wrapped.content() == FileContent.POSITION_DELETES) { - yield ((DeleteFile) wrapped).referencedDataFile(); + return ((DeleteFile) wrapped).referencedDataFile(); } else { - yield null; + return null; } - } - case 18 -> { + case 18: if (wrapped.content() == FileContent.POSITION_DELETES) { - yield ((DeleteFile) wrapped).contentOffset(); + return ((DeleteFile) wrapped).contentOffset(); } else { - yield null; + return null; } - } - case 19 -> { + case 19: if (wrapped.content() == FileContent.POSITION_DELETES) { - yield ((DeleteFile) wrapped).contentSizeInBytes(); + return ((DeleteFile) wrapped).contentSizeInBytes(); } else { - yield null; + return null; } - } - default -> throw new IllegalArgumentException("Unknown field ordinal: " + pos); - }; + } + throw new IllegalArgumentException("Unknown field ordinal: " + pos); } @Override diff --git a/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java b/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java index 5f3d70de5cb1..72d7f4f65bb0 100644 --- a/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java +++ b/core/src/main/java/org/apache/iceberg/actions/RewriteFileGroup.java @@ -104,14 +104,18 @@ public int outputSpecId() { } public static Comparator comparator(RewriteJobOrder rewriteJobOrder) { - return switch (rewriteJobOrder) { - case BYTES_ASC -> Comparator.comparing(RewriteFileGroup::inputFilesSizeInBytes); - case BYTES_DESC -> - Comparator.comparing(RewriteFileGroup::inputFilesSizeInBytes, Comparator.reverseOrder()); - case FILES_ASC -> Comparator.comparing(RewriteFileGroup::inputFileNum); - case FILES_DESC -> - Comparator.comparing(RewriteFileGroup::inputFileNum, Comparator.reverseOrder()); - default -> (unused, unused2) -> 0; - }; + switch (rewriteJobOrder) { + case BYTES_ASC: + return Comparator.comparing(RewriteFileGroup::inputFilesSizeInBytes); + case BYTES_DESC: + return Comparator.comparing( + RewriteFileGroup::inputFilesSizeInBytes, Comparator.reverseOrder()); + case FILES_ASC: + return Comparator.comparing(RewriteFileGroup::inputFileNum); + case FILES_DESC: + return Comparator.comparing(RewriteFileGroup::inputFileNum, Comparator.reverseOrder()); + default: + return (unused, unused2) -> 0; + } } } diff --git a/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java b/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java index 342cb1d4350a..d5ffe7f1e01b 100644 --- a/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java +++ b/core/src/main/java/org/apache/iceberg/actions/RewritePositionDeletesGroup.java @@ -108,16 +108,19 @@ public long addedBytes() { } public static Comparator comparator(RewriteJobOrder order) { - return switch (order) { - case BYTES_ASC -> Comparator.comparing(RewritePositionDeletesGroup::inputFilesSizeInBytes); - case BYTES_DESC -> - Comparator.comparing( - RewritePositionDeletesGroup::inputFilesSizeInBytes, Comparator.reverseOrder()); - case FILES_ASC -> Comparator.comparing(RewritePositionDeletesGroup::inputFileNum); - case FILES_DESC -> - Comparator.comparing( - RewritePositionDeletesGroup::inputFileNum, Comparator.reverseOrder()); - default -> (unused, unused2) -> 0; - }; + switch (order) { + case BYTES_ASC: + return Comparator.comparing(RewritePositionDeletesGroup::inputFilesSizeInBytes); + case BYTES_DESC: + return Comparator.comparing( + RewritePositionDeletesGroup::inputFilesSizeInBytes, Comparator.reverseOrder()); + case FILES_ASC: + return Comparator.comparing(RewritePositionDeletesGroup::inputFileNum); + case FILES_DESC: + return Comparator.comparing( + RewritePositionDeletesGroup::inputFileNum, Comparator.reverseOrder()); + default: + return (unused, unused2) -> 0; + } } } diff --git a/core/src/main/java/org/apache/iceberg/avro/Avro.java b/core/src/main/java/org/apache/iceberg/avro/Avro.java index 1b1c234ef6d3..4a5136f58e71 100644 --- a/core/src/main/java/org/apache/iceberg/avro/Avro.java +++ b/core/src/main/java/org/apache/iceberg/avro/Avro.java @@ -248,17 +248,26 @@ static Context deleteContext(Map config) { private static CodecFactory toCodec(String codecAsString, String compressionLevel) { CodecFactory codecFactory; try { - codecFactory = - switch (Codec.valueOf(codecAsString.toUpperCase(Locale.ENGLISH))) { - case UNCOMPRESSED -> CodecFactory.nullCodec(); - case SNAPPY -> CodecFactory.snappyCodec(); - case ZSTD -> - CodecFactory.zstandardCodec( - compressionLevelAsInt(compressionLevel, ZSTD_COMPRESSION_LEVEL_DEFAULT)); - case GZIP -> - CodecFactory.deflateCodec( - compressionLevelAsInt(compressionLevel, GZIP_COMPRESSION_LEVEL_DEFAULT)); - }; + switch (Codec.valueOf(codecAsString.toUpperCase(Locale.ENGLISH))) { + case UNCOMPRESSED: + codecFactory = CodecFactory.nullCodec(); + break; + case SNAPPY: + codecFactory = CodecFactory.snappyCodec(); + break; + case ZSTD: + codecFactory = + CodecFactory.zstandardCodec( + compressionLevelAsInt(compressionLevel, ZSTD_COMPRESSION_LEVEL_DEFAULT)); + break; + case GZIP: + codecFactory = + CodecFactory.deflateCodec( + compressionLevelAsInt(compressionLevel, GZIP_COMPRESSION_LEVEL_DEFAULT)); + break; + default: + throw new IllegalArgumentException("Unsupported compression codec: " + codecAsString); + } } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Unsupported compression codec: " + codecAsString); } diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroCustomOrderSchemaVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroCustomOrderSchemaVisitor.java index 52c56cf66c96..f579381ac0c8 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroCustomOrderSchemaVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroCustomOrderSchemaVisitor.java @@ -31,8 +31,8 @@ abstract class AvroCustomOrderSchemaVisitor { private static final String VALUE = "value"; public static T visit(Schema schema, AvroCustomOrderSchemaVisitor visitor) { - return switch (schema.getType()) { - case RECORD -> { + switch (schema.getType()) { + case RECORD: // check to make sure this hasn't been visited before String name = schema.getFullName(); Preconditions.checkState( @@ -42,7 +42,7 @@ public static T visit(Schema schema, AvroCustomOrderSchemaVisitor v Preconditions.checkArgument( AvroSchemaUtil.isVariantSchema(schema), "Invalid variant record: %s", schema); - yield visitor.variant( + return visitor.variant( schema, new VisitFuture<>(schema.getField(METADATA).schema(), visitor), new VisitFuture<>(schema.getField(VALUE).schema(), visitor)); @@ -58,21 +58,26 @@ public static T visit(Schema schema, AvroCustomOrderSchemaVisitor v } visitor.recordLevels.pop(); - yield visitor.record(schema, names, Iterables.transform(results, Supplier::get)); + return visitor.record(schema, names, Iterables.transform(results, Supplier::get)); } - } - case UNION -> { + + case UNION: List types = schema.getTypes(); List> options = Lists.newArrayListWithExpectedSize(types.size()); for (Schema type : types) { options.add(new VisitFuture<>(type, visitor)); } - yield visitor.union(schema, Iterables.transform(options, Supplier::get)); - } - case ARRAY -> visitor.array(schema, new VisitFuture<>(schema.getElementType(), visitor)); - case MAP -> visitor.map(schema, new VisitFuture<>(schema.getValueType(), visitor)); - default -> visitor.primitive(schema); - }; + return visitor.union(schema, Iterables.transform(options, Supplier::get)); + + case ARRAY: + return visitor.array(schema, new VisitFuture<>(schema.getElementType(), visitor)); + + case MAP: + return visitor.map(schema, new VisitFuture<>(schema.getValueType(), visitor)); + + default: + return visitor.primitive(schema); + } } private final Deque recordLevels = Lists.newLinkedList(); diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroFormatModel.java b/core/src/main/java/org/apache/iceberg/avro/AvroFormatModel.java index 441e923f0ac0..e0fcf8952604 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroFormatModel.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroFormatModel.java @@ -159,17 +159,17 @@ public ModelWriteBuilder withAADPrefix(ByteBuffer aadPrefix) { @Override public FileAppender build() throws IOException { switch (content) { - case DATA -> { + case DATA: internal.createContextFunc(Avro.WriteBuilder.Context::dataContext); internal.createWriterFunc( avroSchema -> writerFunction.write(schema, avroSchema, engineSchema)); - } - case EQUALITY_DELETES -> { + break; + case EQUALITY_DELETES: internal.createContextFunc(Avro.WriteBuilder.Context::deleteContext); internal.createWriterFunc( avroSchema -> writerFunction.write(schema, avroSchema, engineSchema)); - } - case POSITION_DELETES -> { + break; + case POSITION_DELETES: Preconditions.checkState( schema == null, "Invalid schema: %s. Position deletes with schema are not supported by the API.", @@ -182,8 +182,9 @@ public FileAppender build() throws IOException { internal.createContextFunc(Avro.WriteBuilder.Context::deleteContext); internal.createWriterFunc(unused -> new Avro.PositionDatumWriter()); internal.schema(DeleteSchemaUtil.pathPosSchema()); - } - default -> throw new IllegalArgumentException("Unknown file content: " + content); + break; + default: + throw new IllegalArgumentException("Unknown file content: " + content); } return internal.build(); diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java index a5ca34314815..c67a3089a6bf 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java @@ -189,15 +189,16 @@ public static boolean isOptionSchema(Schema schema) { } static Schema toOption(Schema schema) { - return switch (schema.getType()) { - case UNION -> { + switch (schema.getType()) { + case UNION: Preconditions.checkArgument( isOptionSchema(schema), "Union schemas are not supported: %s", schema); - yield schema; - } - case NULL -> schema; - default -> Schema.createUnion(NULL, schema); - }; + return schema; + case NULL: + return schema; + default: + return Schema.createUnion(NULL, schema); + } } static Schema fromOption(Schema schema) { diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaVisitor.java index ac31ca9ccb5d..df3ca72fa09f 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaVisitor.java @@ -29,8 +29,8 @@ public abstract class AvroSchemaVisitor { private static final String VALUE = "value"; public static T visit(Schema schema, AvroSchemaVisitor visitor) { - return switch (schema.getType()) { - case RECORD -> { + switch (schema.getType()) { + case RECORD: // check to make sure this hasn't been visited before String name = schema.getFullName(); Preconditions.checkState( @@ -40,7 +40,7 @@ public static T visit(Schema schema, AvroSchemaVisitor visitor) { Preconditions.checkArgument( AvroSchemaUtil.isVariantSchema(schema), "Invalid variant record: %s", schema); - yield visitor.variant( + return visitor.variant( schema, visit(schema.getField(METADATA).schema(), visitor), visit(schema.getField(VALUE).schema(), visitor)); @@ -57,27 +57,30 @@ public static T visit(Schema schema, AvroSchemaVisitor visitor) { } visitor.recordLevels.pop(); - yield visitor.record(schema, names, results); + return visitor.record(schema, names, results); } - } - case UNION -> { + + case UNION: List types = schema.getTypes(); List options = Lists.newArrayListWithExpectedSize(types.size()); for (Schema type : types) { options.add(visit(type, visitor)); } - yield visitor.union(schema, options); - } - case ARRAY -> { + return visitor.union(schema, options); + + case ARRAY: if (schema.getLogicalType() instanceof LogicalMap) { - yield visitor.array(schema, visit(schema.getElementType(), visitor)); + return visitor.array(schema, visit(schema.getElementType(), visitor)); } else { - yield visitor.array(schema, visitWithName("element", schema.getElementType(), visitor)); + return visitor.array(schema, visitWithName("element", schema.getElementType(), visitor)); } - } - case MAP -> visitor.map(schema, visitWithName("value", schema.getValueType(), visitor)); - default -> visitor.primitive(schema); - }; + + case MAP: + return visitor.map(schema, visitWithName("value", schema.getValueType(), visitor)); + + default: + return visitor.primitive(schema); + } } private final Deque recordLevels = Lists.newLinkedList(); diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaWithTypeVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaWithTypeVisitor.java index c6542c2555fe..45892d3de151 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaWithTypeVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaWithTypeVisitor.java @@ -33,19 +33,26 @@ public static T visit( } public static T visit(Type iType, Schema schema, AvroSchemaWithTypeVisitor visitor) { - return switch (schema.getType()) { - case RECORD -> visitRecord(iType != null ? iType.asStructType() : null, schema, visitor); - case UNION -> visitUnion(iType, schema, visitor); - case ARRAY -> visitArray(iType, schema, visitor); - case MAP -> { + switch (schema.getType()) { + case RECORD: + return visitRecord(iType != null ? iType.asStructType() : null, schema, visitor); + + case UNION: + return visitUnion(iType, schema, visitor); + + case ARRAY: + return visitArray(iType, schema, visitor); + + case MAP: Types.MapType map = iType != null ? iType.asMapType() : null; - yield visitor.map( + return visitor.map( map, schema, visit(map != null ? map.valueType() : null, schema.getValueType(), visitor)); - } - default -> visitor.primitive(iType != null ? iType.asPrimitiveType() : null, schema); - }; + + default: + return visitor.primitive(iType != null ? iType.asPrimitiveType() : null, schema); + } } private static T visitRecord( diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerByStructureVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerByStructureVisitor.java index 5df3b65e62dd..991ae12664c4 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerByStructureVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerByStructureVisitor.java @@ -37,26 +37,31 @@ public abstract class AvroWithPartnerByStructureVisitor { public static T visit( P partner, Schema schema, AvroWithPartnerByStructureVisitor visitor) { - return switch (schema.getType()) { - case RECORD -> { + switch (schema.getType()) { + case RECORD: if (schema.getLogicalType() instanceof VariantLogicalType || visitor.isVariantType(partner)) { - yield visitVariant(partner, schema, visitor); + return visitVariant(partner, schema, visitor); } else { - yield visitRecord(partner, schema, visitor); + return visitRecord(partner, schema, visitor); } - } - case UNION -> visitUnion(partner, schema, visitor); - case ARRAY -> visitArray(partner, schema, visitor); - case MAP -> { + + case UNION: + return visitUnion(partner, schema, visitor); + + case ARRAY: + return visitArray(partner, schema, visitor); + + case MAP: P keyType = visitor.mapKeyType(partner); Preconditions.checkArgument( visitor.isStringType(keyType), "Invalid map: %s is not a string", keyType); - yield visitor.map( + return visitor.map( partner, schema, visit(visitor.mapValueType(partner), schema.getValueType(), visitor)); - } - default -> visitor.primitive(partner, schema); - }; + + default: + return visitor.primitive(partner, schema); + } } // ---------------------------------- Static helpers --------------------------------------------- diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerVisitor.java b/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerVisitor.java index 6bc09d635ac8..83ddc9be5e29 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerVisitor.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroWithPartnerVisitor.java @@ -102,27 +102,33 @@ public static R visit( Schema schema, AvroWithPartnerVisitor visitor, PartnerAccessors

accessors) { - return switch (schema.getType()) { - case RECORD -> { + switch (schema.getType()) { + case RECORD: if (schema.getLogicalType() instanceof VariantLogicalType) { - yield visitVariant(partner, schema, visitor, accessors); + return visitVariant(partner, schema, visitor, accessors); } else { - yield visitRecord(partner, schema, visitor, accessors); + return visitRecord(partner, schema, visitor, accessors); } - } - case UNION -> visitUnion(partner, schema, visitor, accessors); - case ARRAY -> visitArray(partner, schema, visitor, accessors); - case MAP -> - visitor.map( - partner, - schema, - visit( - partner != null ? accessors.mapValuePartner(partner) : null, - schema.getValueType(), - visitor, - accessors)); - default -> visitor.primitive(partner, schema); - }; + + case UNION: + return visitUnion(partner, schema, visitor, accessors); + + case ARRAY: + return visitArray(partner, schema, visitor, accessors); + + case MAP: + return visitor.map( + partner, + schema, + visit( + partner != null ? accessors.mapValuePartner(partner) : null, + schema.getValueType(), + visitor, + accessors)); + + default: + return visitor.primitive(partner, schema); + } } private static R visitVariant( diff --git a/core/src/main/java/org/apache/iceberg/avro/BaseWriteBuilder.java b/core/src/main/java/org/apache/iceberg/avro/BaseWriteBuilder.java index 6ddcb6bc9b89..51e02eaade82 100644 --- a/core/src/main/java/org/apache/iceberg/avro/BaseWriteBuilder.java +++ b/core/src/main/java/org/apache/iceberg/avro/BaseWriteBuilder.java @@ -74,31 +74,52 @@ public ValueWriter variant( public ValueWriter primitive(Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - return switch (logicalType.getName()) { - case "date" -> ValueWriters.ints(); - case "time-micros" -> ValueWriters.longs(); - case "timestamp-micros" -> ValueWriters.longs(); - case "timestamp-nanos" -> ValueWriters.longs(); - case "decimal" -> { + switch (logicalType.getName()) { + case "date": + return ValueWriters.ints(); + + case "time-micros": + return ValueWriters.longs(); + + case "timestamp-micros": + return ValueWriters.longs(); + + case "timestamp-nanos": + return ValueWriters.longs(); + + case "decimal": LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; - yield ValueWriters.decimal(decimal.getPrecision(), decimal.getScale()); - } - case "uuid" -> ValueWriters.uuids(); - default -> throw new IllegalArgumentException("Unsupported logical type: " + logicalType); - }; + return ValueWriters.decimal(decimal.getPrecision(), decimal.getScale()); + + case "uuid": + return ValueWriters.uuids(); + + default: + throw new IllegalArgumentException("Unsupported logical type: " + logicalType); + } } - return switch (primitive.getType()) { - case NULL -> ValueWriters.nulls(); - case BOOLEAN -> ValueWriters.booleans(); - case INT -> ValueWriters.ints(); - case LONG -> ValueWriters.longs(); - case FLOAT -> ValueWriters.floats(); - case DOUBLE -> ValueWriters.doubles(); - case STRING -> ValueWriters.strings(); - case FIXED -> fixedWriter(primitive.getFixedSize()); - case BYTES -> ValueWriters.byteBuffers(); - default -> throw new IllegalArgumentException("Unsupported type: " + primitive); - }; + switch (primitive.getType()) { + case NULL: + return ValueWriters.nulls(); + case BOOLEAN: + return ValueWriters.booleans(); + case INT: + return ValueWriters.ints(); + case LONG: + return ValueWriters.longs(); + case FLOAT: + return ValueWriters.floats(); + case DOUBLE: + return ValueWriters.doubles(); + case STRING: + return ValueWriters.strings(); + case FIXED: + return fixedWriter(primitive.getFixedSize()); + case BYTES: + return ValueWriters.byteBuffers(); + default: + throw new IllegalArgumentException("Unsupported type: " + primitive); + } } } diff --git a/core/src/main/java/org/apache/iceberg/avro/BuildAvroProjection.java b/core/src/main/java/org/apache/iceberg/avro/BuildAvroProjection.java index 5a16db9d9619..f4dd2f41302d 100644 --- a/core/src/main/java/org/apache/iceberg/avro/BuildAvroProjection.java +++ b/core/src/main/java/org/apache/iceberg/avro/BuildAvroProjection.java @@ -273,20 +273,21 @@ public Schema variant(Schema variant, Supplier metadata, Supplier { + switch (primitive.getType()) { + case INT: if (current.typeId() == Type.TypeID.LONG) { - yield Schema.create(Schema.Type.LONG); + return Schema.create(Schema.Type.LONG); } - yield primitive; - } - case FLOAT -> { + return primitive; + + case FLOAT: if (current.typeId() == Type.TypeID.DOUBLE) { - yield Schema.create(Schema.Type.DOUBLE); + return Schema.create(Schema.Type.DOUBLE); } - yield primitive; - } - default -> primitive; - }; + return primitive; + + default: + return primitive; + } } } diff --git a/core/src/main/java/org/apache/iceberg/avro/GenericAvroReader.java b/core/src/main/java/org/apache/iceberg/avro/GenericAvroReader.java index 47b3ab91518c..fc2d44f47060 100644 --- a/core/src/main/java/org/apache/iceberg/avro/GenericAvroReader.java +++ b/core/src/main/java/org/apache/iceberg/avro/GenericAvroReader.java @@ -173,47 +173,67 @@ public ValueReader variant( public ValueReader primitive(Type partner, Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - return switch (logicalType.getName()) { + switch (logicalType.getName()) { + case "date": // Spark uses the same representation - case "date" -> ValueReaders.ints(); - case "time-micros" -> ValueReaders.longs(); - case "timestamp-millis" -> // adjust to microseconds - (decoder, ignored) -> ValueReaders.longs().read(decoder, null) * 1000L; - // both timestamp-micros and timestamp-nanos are handled in memory as long values, - // using the type to track units - case "timestamp-micros", "timestamp-nanos" -> ValueReaders.longs(); - case "decimal" -> - ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - case "uuid" -> ValueReaders.uuids(); - default -> throw new IllegalArgumentException("Unknown logical type: " + logicalType); - }; + return ValueReaders.ints(); + + case "time-micros": + return ValueReaders.longs(); + + case "timestamp-millis": + // adjust to microseconds + ValueReader longs = ValueReaders.longs(); + return (ValueReader) (decoder, ignored) -> longs.read(decoder, null) * 1000L; + + case "timestamp-micros": + case "timestamp-nanos": + // both are handled in memory as long values, using the type to track units + return ValueReaders.longs(); + + case "decimal": + return ValueReaders.decimal( + ValueReaders.decimalBytesReader(primitive), + ((LogicalTypes.Decimal) logicalType).getScale()); + + case "uuid": + return ValueReaders.uuids(); + + default: + throw new IllegalArgumentException("Unknown logical type: " + logicalType); + } } - return switch (primitive.getType()) { - case NULL -> ValueReaders.nulls(); - case BOOLEAN -> ValueReaders.booleans(); - case INT -> { + switch (primitive.getType()) { + case NULL: + return ValueReaders.nulls(); + case BOOLEAN: + return ValueReaders.booleans(); + case INT: if (partner != null && partner.typeId() == Type.TypeID.LONG) { - yield ValueReaders.intsAsLongs(); + return ValueReaders.intsAsLongs(); } - yield ValueReaders.ints(); - } - case LONG -> ValueReaders.longs(); - case FLOAT -> { + return ValueReaders.ints(); + case LONG: + return ValueReaders.longs(); + case FLOAT: if (partner != null && partner.typeId() == Type.TypeID.DOUBLE) { - yield ValueReaders.floatsAsDoubles(); + return ValueReaders.floatsAsDoubles(); } - yield ValueReaders.floats(); - } - case DOUBLE -> ValueReaders.doubles(); - case STRING -> ValueReaders.utf8s(); - case FIXED -> ValueReaders.fixed(primitive); - case BYTES -> ValueReaders.byteBuffers(); - case ENUM -> ValueReaders.enums(primitive.getEnumSymbols()); - default -> throw new IllegalArgumentException("Unsupported type: " + primitive); - }; + return ValueReaders.floats(); + case DOUBLE: + return ValueReaders.doubles(); + case STRING: + return ValueReaders.utf8s(); + case FIXED: + return ValueReaders.fixed(primitive); + case BYTES: + return ValueReaders.byteBuffers(); + case ENUM: + return ValueReaders.enums(primitive.getEnumSymbols()); + default: + throw new IllegalArgumentException("Unsupported type: " + primitive); + } } } } diff --git a/core/src/main/java/org/apache/iceberg/avro/InternalReader.java b/core/src/main/java/org/apache/iceberg/avro/InternalReader.java index fe7fec7f6e5f..af3c4f1a822b 100644 --- a/core/src/main/java/org/apache/iceberg/avro/InternalReader.java +++ b/core/src/main/java/org/apache/iceberg/avro/InternalReader.java @@ -170,45 +170,65 @@ public ValueReader variant( public ValueReader primitive(Pair partner, Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - return switch (logicalType.getName()) { - case "date" -> ValueReaders.ints(); - case "time-micros" -> ValueReaders.longs(); - case "timestamp-millis" -> // adjust to microseconds - (decoder, ignored) -> ValueReaders.longs().read(decoder, null) * 1000L; - // both timestamp-micros and timestamp-nanos are handled in memory as long values, - // using the type to track units - case "timestamp-micros", "timestamp-nanos" -> ValueReaders.longs(); - case "decimal" -> - ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - case "uuid" -> ValueReaders.uuids(); - default -> throw new IllegalArgumentException("Unknown logical type: " + logicalType); - }; + switch (logicalType.getName()) { + case "date": + return ValueReaders.ints(); + + case "time-micros": + return ValueReaders.longs(); + + case "timestamp-millis": + // adjust to microseconds + ValueReader longs = ValueReaders.longs(); + return (ValueReader) (decoder, ignored) -> longs.read(decoder, null) * 1000L; + + case "timestamp-micros": + case "timestamp-nanos": + // both are handled in memory as long values, using the type to track units + return ValueReaders.longs(); + + case "decimal": + return ValueReaders.decimal( + ValueReaders.decimalBytesReader(primitive), + ((LogicalTypes.Decimal) logicalType).getScale()); + + case "uuid": + return ValueReaders.uuids(); + + default: + throw new IllegalArgumentException("Unknown logical type: " + logicalType); + } } - return switch (primitive.getType()) { - case NULL -> ValueReaders.nulls(); - case BOOLEAN -> ValueReaders.booleans(); - case INT -> { + switch (primitive.getType()) { + case NULL: + return ValueReaders.nulls(); + case BOOLEAN: + return ValueReaders.booleans(); + case INT: if (partner != null && partner.second().typeId() == Type.TypeID.LONG) { - yield ValueReaders.intsAsLongs(); + return ValueReaders.intsAsLongs(); } - yield ValueReaders.ints(); - } - case LONG -> ValueReaders.longs(); - case FLOAT -> { + return ValueReaders.ints(); + case LONG: + return ValueReaders.longs(); + case FLOAT: if (partner != null && partner.second().typeId() == Type.TypeID.DOUBLE) { - yield ValueReaders.floatsAsDoubles(); + return ValueReaders.floatsAsDoubles(); } - yield ValueReaders.floats(); - } - case DOUBLE -> ValueReaders.doubles(); - case STRING -> ValueReaders.strings(); - case FIXED, BYTES -> ValueReaders.byteBuffers(); - case ENUM -> ValueReaders.enums(primitive.getEnumSymbols()); - default -> throw new IllegalArgumentException("Unsupported type: " + primitive); - }; + return ValueReaders.floats(); + case DOUBLE: + return ValueReaders.doubles(); + case STRING: + return ValueReaders.strings(); + case FIXED: + case BYTES: + return ValueReaders.byteBuffers(); + case ENUM: + return ValueReaders.enums(primitive.getEnumSymbols()); + default: + throw new IllegalArgumentException("Unsupported type: " + primitive); + } } } diff --git a/core/src/main/java/org/apache/iceberg/avro/SchemaToType.java b/core/src/main/java/org/apache/iceberg/avro/SchemaToType.java index 02c9c5f1e1b2..2ffd658f447e 100644 --- a/core/src/main/java/org/apache/iceberg/avro/SchemaToType.java +++ b/core/src/main/java/org/apache/iceberg/avro/SchemaToType.java @@ -224,18 +224,28 @@ public Type primitive(Schema primitive) { } } - return switch (primitive.getType()) { - case BOOLEAN -> Types.BooleanType.get(); - case INT -> Types.IntegerType.get(); - case LONG -> Types.LongType.get(); - case FLOAT -> Types.FloatType.get(); - case DOUBLE -> Types.DoubleType.get(); - case STRING, ENUM -> Types.StringType.get(); - case FIXED -> Types.FixedType.ofLength(primitive.getFixedSize()); - case BYTES -> Types.BinaryType.get(); - case NULL -> Types.UnknownType.get(); - default -> - throw new UnsupportedOperationException("Unsupported primitive type: " + primitive); - }; + switch (primitive.getType()) { + case BOOLEAN: + return Types.BooleanType.get(); + case INT: + return Types.IntegerType.get(); + case LONG: + return Types.LongType.get(); + case FLOAT: + return Types.FloatType.get(); + case DOUBLE: + return Types.DoubleType.get(); + case STRING: + case ENUM: + return Types.StringType.get(); + case FIXED: + return Types.FixedType.ofLength(primitive.getFixedSize()); + case BYTES: + return Types.BinaryType.get(); + case NULL: + return Types.UnknownType.get(); + } + + throw new UnsupportedOperationException("Unsupported primitive type: " + primitive); } } diff --git a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java index b62700dba5f1..4442695b7a44 100644 --- a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java +++ b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java @@ -213,36 +213,58 @@ public Schema variant(Types.VariantType variant) { public Schema primitive(Type.PrimitiveType primitive) { Schema primitiveSchema; switch (primitive.typeId()) { - case UNKNOWN -> primitiveSchema = NULL_SCHEMA; - case BOOLEAN -> primitiveSchema = BOOLEAN_SCHEMA; - case INTEGER -> primitiveSchema = INTEGER_SCHEMA; - case LONG -> primitiveSchema = LONG_SCHEMA; - case FLOAT -> primitiveSchema = FLOAT_SCHEMA; - case DOUBLE -> primitiveSchema = DOUBLE_SCHEMA; - case DATE -> primitiveSchema = DATE_SCHEMA; - case TIME -> primitiveSchema = TIME_SCHEMA; - case TIMESTAMP -> { + case UNKNOWN: + primitiveSchema = NULL_SCHEMA; + break; + case BOOLEAN: + primitiveSchema = BOOLEAN_SCHEMA; + break; + case INTEGER: + primitiveSchema = INTEGER_SCHEMA; + break; + case LONG: + primitiveSchema = LONG_SCHEMA; + break; + case FLOAT: + primitiveSchema = FLOAT_SCHEMA; + break; + case DOUBLE: + primitiveSchema = DOUBLE_SCHEMA; + break; + case DATE: + primitiveSchema = DATE_SCHEMA; + break; + case TIME: + primitiveSchema = TIME_SCHEMA; + break; + case TIMESTAMP: if (((Types.TimestampType) primitive).shouldAdjustToUTC()) { primitiveSchema = TIMESTAMPTZ_SCHEMA; } else { primitiveSchema = TIMESTAMP_SCHEMA; } - } - case TIMESTAMP_NANO -> { + break; + case TIMESTAMP_NANO: if (((Types.TimestampNanoType) primitive).shouldAdjustToUTC()) { primitiveSchema = TIMESTAMPTZ_NANO_SCHEMA; } else { primitiveSchema = TIMESTAMP_NANO_SCHEMA; } - } - case STRING -> primitiveSchema = STRING_SCHEMA; - case UUID -> primitiveSchema = UUID_SCHEMA; - case FIXED -> { + break; + case STRING: + primitiveSchema = STRING_SCHEMA; + break; + case UUID: + primitiveSchema = UUID_SCHEMA; + break; + case FIXED: Types.FixedType fixed = (Types.FixedType) primitive; primitiveSchema = Schema.createFixed("fixed_" + fixed.length(), null, null, fixed.length()); - } - case BINARY -> primitiveSchema = BINARY_SCHEMA; - case DECIMAL -> { + break; + case BINARY: + primitiveSchema = BINARY_SCHEMA; + break; + case DECIMAL: Types.DecimalType decimal = (Types.DecimalType) primitive; primitiveSchema = LogicalTypes.decimal(decimal.precision(), decimal.scale()) @@ -252,9 +274,9 @@ public Schema primitive(Type.PrimitiveType primitive) { null, null, TypeUtil.decimalRequiredBytes(decimal.precision()))); - } - default -> - throw new UnsupportedOperationException("Unsupported type ID: " + primitive.typeId()); + break; + default: + throw new UnsupportedOperationException("Unsupported type ID: " + primitive.typeId()); } cacheSchema(primitive, primitiveSchema); diff --git a/core/src/main/java/org/apache/iceberg/avro/ValueReaders.java b/core/src/main/java/org/apache/iceberg/avro/ValueReaders.java index ce5227f90220..ec46c56c72c3 100644 --- a/core/src/main/java/org/apache/iceberg/avro/ValueReaders.java +++ b/core/src/main/java/org/apache/iceberg/avro/ValueReaders.java @@ -133,13 +133,15 @@ public static ValueReader decimal(ValueReader unscaledReader } public static ValueReader decimalBytesReader(Schema schema) { - return switch (schema.getType()) { - case FIXED -> ValueReaders.fixed(schema.getFixedSize()); - case BYTES -> ValueReaders.bytes(); - default -> - throw new IllegalArgumentException( - "Invalid primitive type for decimal: " + schema.getType()); - }; + switch (schema.getType()) { + case FIXED: + return ValueReaders.fixed(schema.getFixedSize()); + case BYTES: + return ValueReaders.bytes(); + default: + throw new IllegalArgumentException( + "Invalid primitive type for decimal: " + schema.getType()); + } } public static ValueReader variants() { diff --git a/core/src/main/java/org/apache/iceberg/data/GenericDataUtil.java b/core/src/main/java/org/apache/iceberg/data/GenericDataUtil.java index 01ca179fd9ec..9b720c1f865a 100644 --- a/core/src/main/java/org/apache/iceberg/data/GenericDataUtil.java +++ b/core/src/main/java/org/apache/iceberg/data/GenericDataUtil.java @@ -40,25 +40,27 @@ public static Object internalToGeneric(Type type, Object value) { return null; } - return switch (type.typeId()) { - case DATE -> DateTimeUtil.dateFromDays((Integer) value); - case TIME -> DateTimeUtil.timeFromMicros((Long) value); - case TIMESTAMP -> { + switch (type.typeId()) { + case DATE: + return DateTimeUtil.dateFromDays((Integer) value); + case TIME: + return DateTimeUtil.timeFromMicros((Long) value); + case TIMESTAMP: if (((Types.TimestampType) type).shouldAdjustToUTC()) { - yield DateTimeUtil.timestamptzFromMicros((Long) value); + return DateTimeUtil.timestamptzFromMicros((Long) value); } else { - yield DateTimeUtil.timestampFromMicros((Long) value); + return DateTimeUtil.timestampFromMicros((Long) value); } - } - case TIMESTAMP_NANO -> { + case TIMESTAMP_NANO: if (((Types.TimestampNanoType) type).shouldAdjustToUTC()) { - yield DateTimeUtil.timestamptzFromNanos((Long) value); + return DateTimeUtil.timestamptzFromNanos((Long) value); } else { - yield DateTimeUtil.timestampFromNanos((Long) value); + return DateTimeUtil.timestampFromNanos((Long) value); } - } - case FIXED -> ByteBuffers.toByteArray((ByteBuffer) value); - default -> value; - }; + case FIXED: + return ByteBuffers.toByteArray((ByteBuffer) value); + } + + return value; } } diff --git a/core/src/main/java/org/apache/iceberg/data/IdentityPartitionConverters.java b/core/src/main/java/org/apache/iceberg/data/IdentityPartitionConverters.java index 382493ac70e7..4cb41263152d 100644 --- a/core/src/main/java/org/apache/iceberg/data/IdentityPartitionConverters.java +++ b/core/src/main/java/org/apache/iceberg/data/IdentityPartitionConverters.java @@ -32,24 +32,26 @@ public static Object convertConstant(Type type, Object value) { return null; } - return switch (type.typeId()) { - case STRING -> value.toString(); - case TIME -> DateTimeUtil.timeFromMicros((Long) value); - case DATE -> DateTimeUtil.dateFromDays((Integer) value); - case TIMESTAMP -> { + switch (type.typeId()) { + case STRING: + return value.toString(); + case TIME: + return DateTimeUtil.timeFromMicros((Long) value); + case DATE: + return DateTimeUtil.dateFromDays((Integer) value); + case TIMESTAMP: if (((Types.TimestampType) type).shouldAdjustToUTC()) { - yield DateTimeUtil.timestamptzFromMicros((Long) value); + return DateTimeUtil.timestamptzFromMicros((Long) value); } else { - yield DateTimeUtil.timestampFromMicros((Long) value); + return DateTimeUtil.timestampFromMicros((Long) value); } - } - case FIXED -> { + case FIXED: if (value instanceof GenericData.Fixed) { - yield ((GenericData.Fixed) value).bytes(); + return ((GenericData.Fixed) value).bytes(); } - yield value; - } - default -> value; - }; + return value; + default: + } + return value; } } diff --git a/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java b/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java index 6fe260239426..5f813f8db576 100644 --- a/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java +++ b/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java @@ -127,50 +127,67 @@ public ValueReader map(Types.MapType ignored, Schema map, ValueReader valu public ValueReader primitive(Type.PrimitiveType ignored, Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - return switch (logicalType.getName()) { - case "date" -> GenericReaders.dates(); - case "time-micros" -> GenericReaders.times(); - case "timestamp-micros" -> { + switch (logicalType.getName()) { + case "date": + return GenericReaders.dates(); + + case "time-micros": + return GenericReaders.times(); + + case "timestamp-micros": if (AvroSchemaUtil.isTimestamptz(primitive)) { - yield GenericReaders.timestamptz(); + return GenericReaders.timestamptz(); } - yield GenericReaders.timestamps(); - } - case "timestamp-nanos" -> { + return GenericReaders.timestamps(); + + case "timestamp-nanos": if (AvroSchemaUtil.isTimestamptz(primitive)) { - yield GenericReaders.timestamptzNanos(); + return GenericReaders.timestamptzNanos(); } - yield GenericReaders.timestampNanos(); - } - case "timestamp-millis" -> { + return GenericReaders.timestampNanos(); + + case "timestamp-millis": if (AvroSchemaUtil.isTimestamptz(primitive)) { - yield GenericReaders.timestamptzMillis(); + return GenericReaders.timestamptzMillis(); } - yield GenericReaders.timestampMillis(); - } - case "decimal" -> - ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - case "uuid" -> ValueReaders.uuids(); - default -> throw new IllegalArgumentException("Unknown logical type: " + logicalType); - }; + return GenericReaders.timestampMillis(); + + case "decimal": + return ValueReaders.decimal( + ValueReaders.decimalBytesReader(primitive), + ((LogicalTypes.Decimal) logicalType).getScale()); + + case "uuid": + return ValueReaders.uuids(); + + default: + throw new IllegalArgumentException("Unknown logical type: " + logicalType); + } } - return switch (primitive.getType()) { - case NULL -> ValueReaders.nulls(); - case BOOLEAN -> ValueReaders.booleans(); - case INT -> ValueReaders.ints(); - case LONG -> ValueReaders.longs(); - case FLOAT -> ValueReaders.floats(); - case DOUBLE -> ValueReaders.doubles(); - case STRING -> - // might want to use a binary-backed container like Utf8 - ValueReaders.strings(); - case FIXED -> ValueReaders.fixed(primitive.getFixedSize()); - case BYTES -> ValueReaders.byteBuffers(); - default -> throw new IllegalArgumentException("Unsupported type: " + primitive); - }; + switch (primitive.getType()) { + case NULL: + return ValueReaders.nulls(); + case BOOLEAN: + return ValueReaders.booleans(); + case INT: + return ValueReaders.ints(); + case LONG: + return ValueReaders.longs(); + case FLOAT: + return ValueReaders.floats(); + case DOUBLE: + return ValueReaders.doubles(); + case STRING: + // might want to use a binary-backed container like Utf8 + return ValueReaders.strings(); + case FIXED: + return ValueReaders.fixed(primitive.getFixedSize()); + case BYTES: + return ValueReaders.byteBuffers(); + default: + throw new IllegalArgumentException("Unsupported type: " + primitive); + } } } } diff --git a/core/src/main/java/org/apache/iceberg/data/avro/DataWriter.java b/core/src/main/java/org/apache/iceberg/data/avro/DataWriter.java index ffc8c10f92d1..397b38643b15 100644 --- a/core/src/main/java/org/apache/iceberg/data/avro/DataWriter.java +++ b/core/src/main/java/org/apache/iceberg/data/avro/DataWriter.java @@ -111,42 +111,59 @@ public ValueWriter variant( public ValueWriter primitive(Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - return switch (logicalType.getName()) { - case "date" -> GenericWriters.dates(); - case "time-micros" -> GenericWriters.times(); - case "timestamp-micros" -> { + switch (logicalType.getName()) { + case "date": + return GenericWriters.dates(); + + case "time-micros": + return GenericWriters.times(); + + case "timestamp-micros": if (AvroSchemaUtil.isTimestamptz(primitive)) { - yield GenericWriters.timestamptz(); + return GenericWriters.timestamptz(); } - yield GenericWriters.timestamps(); - } - case "timestamp-nanos" -> { + return GenericWriters.timestamps(); + + case "timestamp-nanos": if (AvroSchemaUtil.isTimestamptz(primitive)) { - yield GenericWriters.timestamptzNanos(); + return GenericWriters.timestamptzNanos(); } - yield GenericWriters.timestampNanos(); - } - case "decimal" -> { + return GenericWriters.timestampNanos(); + + case "decimal": LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; - yield ValueWriters.decimal(decimal.getPrecision(), decimal.getScale()); - } - case "uuid" -> ValueWriters.uuids(); - default -> throw new IllegalArgumentException("Unsupported logical type: " + logicalType); - }; + return ValueWriters.decimal(decimal.getPrecision(), decimal.getScale()); + + case "uuid": + return ValueWriters.uuids(); + + default: + throw new IllegalArgumentException("Unsupported logical type: " + logicalType); + } } - return switch (primitive.getType()) { - case NULL -> ValueWriters.nulls(); - case BOOLEAN -> ValueWriters.booleans(); - case INT -> ValueWriters.ints(); - case LONG -> ValueWriters.longs(); - case FLOAT -> ValueWriters.floats(); - case DOUBLE -> ValueWriters.doubles(); - case STRING -> ValueWriters.strings(); - case FIXED -> ValueWriters.fixed(primitive.getFixedSize()); - case BYTES -> ValueWriters.byteBuffers(); - default -> throw new IllegalArgumentException("Unsupported type: " + primitive); - }; + switch (primitive.getType()) { + case NULL: + return ValueWriters.nulls(); + case BOOLEAN: + return ValueWriters.booleans(); + case INT: + return ValueWriters.ints(); + case LONG: + return ValueWriters.longs(); + case FLOAT: + return ValueWriters.floats(); + case DOUBLE: + return ValueWriters.doubles(); + case STRING: + return ValueWriters.strings(); + case FIXED: + return ValueWriters.fixed(primitive.getFixedSize()); + case BYTES: + return ValueWriters.byteBuffers(); + default: + throw new IllegalArgumentException("Unsupported type: " + primitive); + } } } } diff --git a/core/src/main/java/org/apache/iceberg/data/avro/PlannedDataReader.java b/core/src/main/java/org/apache/iceberg/data/avro/PlannedDataReader.java index 54e3cee4d4a9..747907a2fb97 100644 --- a/core/src/main/java/org/apache/iceberg/data/avro/PlannedDataReader.java +++ b/core/src/main/java/org/apache/iceberg/data/avro/PlannedDataReader.java @@ -135,60 +135,73 @@ public ValueReader variant( public ValueReader primitive(Type partner, Schema primitive) { LogicalType logicalType = primitive.getLogicalType(); if (logicalType != null) { - return switch (logicalType.getName()) { - case "date" -> GenericReaders.dates(); - case "time-micros" -> GenericReaders.times(); - case "timestamp-micros" -> { + switch (logicalType.getName()) { + case "date": + return GenericReaders.dates(); + + case "time-micros": + return GenericReaders.times(); + + case "timestamp-micros": if (AvroSchemaUtil.isTimestamptz(primitive)) { - yield GenericReaders.timestamptz(); + return GenericReaders.timestamptz(); } - yield GenericReaders.timestamps(); - } - case "timestamp-nanos" -> { + return GenericReaders.timestamps(); + + case "timestamp-nanos": if (AvroSchemaUtil.isTimestamptz(primitive)) { - yield GenericReaders.timestamptzNanos(); + return GenericReaders.timestamptzNanos(); } - yield GenericReaders.timestampNanos(); - } - case "timestamp-millis" -> { + return GenericReaders.timestampNanos(); + + case "timestamp-millis": if (AvroSchemaUtil.isTimestamptz(primitive)) { - yield GenericReaders.timestamptzMillis(); + return GenericReaders.timestamptzMillis(); } - yield GenericReaders.timestampMillis(); - } - case "decimal" -> - ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - case "uuid" -> ValueReaders.uuids(); - default -> throw new IllegalArgumentException("Unknown logical type: " + logicalType); - }; + return GenericReaders.timestampMillis(); + + case "decimal": + return ValueReaders.decimal( + ValueReaders.decimalBytesReader(primitive), + ((LogicalTypes.Decimal) logicalType).getScale()); + + case "uuid": + return ValueReaders.uuids(); + + default: + throw new IllegalArgumentException("Unknown logical type: " + logicalType); + } } - return switch (primitive.getType()) { - case NULL -> ValueReaders.nulls(); - case BOOLEAN -> ValueReaders.booleans(); - case INT -> { + switch (primitive.getType()) { + case NULL: + return ValueReaders.nulls(); + case BOOLEAN: + return ValueReaders.booleans(); + case INT: if (partner != null && partner.typeId() == Type.TypeID.LONG) { - yield ValueReaders.intsAsLongs(); + return ValueReaders.intsAsLongs(); } - yield ValueReaders.ints(); - } - case LONG -> ValueReaders.longs(); - case FLOAT -> { + return ValueReaders.ints(); + case LONG: + return ValueReaders.longs(); + case FLOAT: if (partner != null && partner.typeId() == Type.TypeID.DOUBLE) { - yield ValueReaders.floatsAsDoubles(); + return ValueReaders.floatsAsDoubles(); } - yield ValueReaders.floats(); - } - case DOUBLE -> ValueReaders.doubles(); - case STRING -> - // might want to use a binary-backed container like Utf8 - ValueReaders.strings(); - case FIXED -> ValueReaders.fixed(primitive.getFixedSize()); - case BYTES -> ValueReaders.byteBuffers(); - default -> throw new IllegalArgumentException("Unsupported type: " + primitive); - }; + return ValueReaders.floats(); + case DOUBLE: + return ValueReaders.doubles(); + case STRING: + // might want to use a binary-backed container like Utf8 + return ValueReaders.strings(); + case FIXED: + return ValueReaders.fixed(primitive.getFixedSize()); + case BYTES: + return ValueReaders.byteBuffers(); + default: + throw new IllegalArgumentException("Unsupported type: " + primitive); + } } } } diff --git a/core/src/main/java/org/apache/iceberg/deletes/DeleteGranularity.java b/core/src/main/java/org/apache/iceberg/deletes/DeleteGranularity.java index 4ca4e5ae67c8..c225192fa121 100644 --- a/core/src/main/java/org/apache/iceberg/deletes/DeleteGranularity.java +++ b/core/src/main/java/org/apache/iceberg/deletes/DeleteGranularity.java @@ -48,10 +48,14 @@ public enum DeleteGranularity { @Override public String toString() { - return switch (this) { - case FILE -> "file"; - case PARTITION -> "partition"; - }; + switch (this) { + case FILE: + return "file"; + case PARTITION: + return "partition"; + default: + throw new IllegalArgumentException("Unknown delete granularity: " + this); + } } public static DeleteGranularity fromString(String valueAsString) { diff --git a/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java b/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java index 8bb006390fb9..c3b6cbaa9bff 100644 --- a/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java +++ b/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java @@ -81,21 +81,32 @@ public R row() { @Override @SuppressWarnings("unchecked") public T get(int colPos, Class javaClass) { - return switch (colPos) { - case 0 -> (T) path; - case 1 -> (T) (Long) pos; - case 2 -> (T) row; - default -> throw new IllegalArgumentException("No column at position " + colPos); - }; + switch (colPos) { + case 0: + return (T) path; + case 1: + return (T) (Long) pos; + case 2: + return (T) row; + default: + throw new IllegalArgumentException("No column at position " + colPos); + } } @Override public void set(int colPos, T value) { switch (colPos) { - case 0 -> this.path = (CharSequence) value; - case 1 -> this.pos = (Long) value; - case 2 -> this.row = (R) value; - default -> throw new IllegalArgumentException("No column at position " + colPos); + case 0: + this.path = (CharSequence) value; + break; + case 1: + this.pos = (Long) value; + break; + case 2: + this.row = (R) value; + break; + default: + throw new IllegalArgumentException("No column at position " + colPos); } } } diff --git a/core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java b/core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java index 50b6c15e78b0..ba954577aa74 100644 --- a/core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java +++ b/core/src/main/java/org/apache/iceberg/deletes/SortingPositionOnlyDeleteWriter.java @@ -101,11 +101,14 @@ public DeleteWriteResult result() { public void close() throws IOException { if (result == null) { switch (granularity) { - case FILE -> this.result = writeFileDeletes(); - case PARTITION -> this.result = writePartitionDeletes(); - default -> - throw new UnsupportedOperationException( - "Unsupported delete granularity: " + granularity); + case FILE: + this.result = writeFileDeletes(); + return; + case PARTITION: + this.result = writePartitionDeletes(); + return; + default: + throw new UnsupportedOperationException("Unsupported delete granularity: " + granularity); } } } diff --git a/core/src/main/java/org/apache/iceberg/encryption/StandardKeyMetadata.java b/core/src/main/java/org/apache/iceberg/encryption/StandardKeyMetadata.java index 062328a080f7..6ddea184d8c4 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/StandardKeyMetadata.java +++ b/core/src/main/java/org/apache/iceberg/encryption/StandardKeyMetadata.java @@ -144,23 +144,32 @@ public NativeEncryptionKeyMetadata copyWithLength(long length) { @Override public void put(int i, Object v) { switch (i) { - case 0 -> this.encryptionKey = (ByteBuffer) v; - case 1 -> this.aadPrefix = (ByteBuffer) v; - case 2 -> this.fileLength = (Long) v; - default -> { + case 0: + this.encryptionKey = (ByteBuffer) v; + return; + case 1: + this.aadPrefix = (ByteBuffer) v; + return; + case 2: + this.fileLength = (Long) v; + return; + default: // ignore the object, it must be from a newer version of the format - } } } @Override public Object get(int i) { - return switch (i) { - case 0 -> encryptionKey; - case 1 -> aadPrefix; - case 2 -> fileLength; - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + i); - }; + switch (i) { + case 0: + return encryptionKey; + case 1: + return aadPrefix; + case 2: + return fileLength; + default: + throw new UnsupportedOperationException("Unknown field ordinal: " + i); + } } @Override diff --git a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java index 16e0cdacd022..9bb5b7d05f0b 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java +++ b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java @@ -295,18 +295,20 @@ static Expression fromJson(JsonNode json, Schema schema) { } Expression.Operation op = fromType(type); - return switch (op) { - case NOT -> Expressions.not(fromJson(JsonUtil.get(CHILD, json), schema)); - case AND -> - Expressions.and( - fromJson(JsonUtil.get(LEFT, json), schema), - fromJson(JsonUtil.get(RIGHT, json), schema)); - case OR -> - Expressions.or( - fromJson(JsonUtil.get(LEFT, json), schema), - fromJson(JsonUtil.get(RIGHT, json), schema)); - default -> predicateFromJson(op, json, schema); - }; + switch (op) { + case NOT: + return Expressions.not(fromJson(JsonUtil.get(CHILD, json), schema)); + case AND: + return Expressions.and( + fromJson(JsonUtil.get(LEFT, json), schema), + fromJson(JsonUtil.get(RIGHT, json), schema)); + case OR: + return Expressions.or( + fromJson(JsonUtil.get(LEFT, json), schema), + fromJson(JsonUtil.get(RIGHT, json), schema)); + } + + return predicateFromJson(op, json, schema); } private static Expression.Operation fromType(String type) { @@ -326,25 +328,34 @@ private static UnboundPredicate predicateFromJson( convertValue = valueNode -> (T) ExpressionParser.asObject(valueNode); } - return switch (op) { - case IS_NULL, NOT_NULL, IS_NAN, NOT_NAN -> { + switch (op) { + case IS_NULL: + case NOT_NULL: + case IS_NAN: + case NOT_NAN: // unary predicates Preconditions.checkArgument( !node.has(VALUE), "Cannot parse %s predicate: has invalid value field", op); Preconditions.checkArgument( !node.has(VALUES), "Cannot parse %s predicate: has invalid values field", op); - yield Expressions.predicate(op, term); - } - case LT, LT_EQ, GT, GT_EQ, EQ, NOT_EQ, STARTS_WITH, NOT_STARTS_WITH -> { + return Expressions.predicate(op, term); + case LT: + case LT_EQ: + case GT: + case GT_EQ: + case EQ: + case NOT_EQ: + case STARTS_WITH: + case NOT_STARTS_WITH: // literal predicates Preconditions.checkArgument( node.has(VALUE), "Cannot parse %s predicate: missing value", op); Preconditions.checkArgument( !node.has(VALUES), "Cannot parse %s predicate: has invalid values field", op); T value = literal(JsonUtil.get(VALUE, node), convertValue); - yield Expressions.predicate(op, term, ImmutableList.of(value)); - } - case IN, NOT_IN -> { + return Expressions.predicate(op, term, ImmutableList.of(value)); + case IN: + case NOT_IN: // literal set predicates Preconditions.checkArgument( node.has(VALUES), "Cannot parse %s predicate: missing values", op); @@ -353,14 +364,14 @@ private static UnboundPredicate predicateFromJson( JsonNode valuesNode = JsonUtil.get(VALUES, node); Preconditions.checkArgument( valuesNode.isArray(), "Cannot parse literals from non-array: %s", valuesNode); - yield Expressions.predicate( + return Expressions.predicate( op, term, Iterables.transform( ((ArrayNode) valuesNode)::elements, valueNode -> literal(valueNode, convertValue))); - } - default -> throw new UnsupportedOperationException("Unsupported operation: " + op); - }; + default: + throw new UnsupportedOperationException("Unsupported operation: " + op); + } } private static T literal(JsonNode valueNode, Function toValue) { @@ -395,16 +406,17 @@ private static UnboundTerm term(JsonNode node) { return Expressions.ref(node.asText()); } else if (node.isObject()) { String type = JsonUtil.getString(TYPE, node); - return switch (type) { - case REFERENCE -> Expressions.ref(JsonUtil.getString(TERM, node)); - case TRANSFORM -> { + switch (type) { + case REFERENCE: + return Expressions.ref(JsonUtil.getString(TERM, node)); + case TRANSFORM: UnboundTerm child = term(JsonUtil.get(TERM, node)); String transform = JsonUtil.getString(TRANSFORM, node); - yield (UnboundTerm) + return (UnboundTerm) Expressions.transform(child.ref().name(), Transforms.fromString(transform)); - } - default -> throw new IllegalArgumentException("Cannot parse type as a reference: " + type); - }; + default: + throw new IllegalArgumentException("Cannot parse type as a reference: " + type); + } } throw new IllegalArgumentException( diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopMetricsContext.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopMetricsContext.java index de39fe106825..4e677c9b739c 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopMetricsContext.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopMetricsContext.java @@ -66,28 +66,24 @@ public void initialize(Map properties) { @Override @SuppressWarnings("unchecked") public Counter counter(String name, Class type, Unit unit) { - return switch (name) { - case READ_BYTES -> { + switch (name) { + case READ_BYTES: ValidationException.check(type == Long.class, "'%s' requires Long type", READ_BYTES); - yield (Counter) longCounter(statistics()::incrementBytesRead); - } - case READ_OPERATIONS -> { + return (Counter) longCounter(statistics()::incrementBytesRead); + case READ_OPERATIONS: ValidationException.check( type == Integer.class, "'%s' requires Integer type", READ_OPERATIONS); - yield (Counter) integerCounter(statistics()::incrementReadOps); - } - case WRITE_BYTES -> { + return (Counter) integerCounter(statistics()::incrementReadOps); + case WRITE_BYTES: ValidationException.check(type == Long.class, "'%s' requires Long type", WRITE_BYTES); - yield (Counter) longCounter(statistics()::incrementBytesWritten); - } - case WRITE_OPERATIONS -> { + return (Counter) longCounter(statistics()::incrementBytesWritten); + case WRITE_OPERATIONS: ValidationException.check( type == Integer.class, "'%s' requires Integer type", WRITE_OPERATIONS); - yield (Counter) integerCounter(statistics()::incrementWriteOps); - } - default -> - throw new IllegalArgumentException(String.format("Unsupported counter: '%s'", name)); - }; + return (Counter) integerCounter(statistics()::incrementWriteOps); + default: + throw new IllegalArgumentException(String.format("Unsupported counter: '%s'", name)); + } } private Counter longCounter(Consumer consumer) { @@ -129,17 +125,19 @@ public void increment(Integer amount) { */ @Override public org.apache.iceberg.metrics.Counter counter(String name, Unit unit) { - return switch (name) { - case READ_BYTES -> counter(statistics()::incrementBytesRead, statistics()::getBytesRead); - case READ_OPERATIONS -> - counter((long x) -> statistics.incrementReadOps((int) x), statistics()::getReadOps); - case WRITE_BYTES -> - counter(statistics()::incrementBytesWritten, statistics()::getBytesWritten); - case WRITE_OPERATIONS -> - counter((long x) -> statistics.incrementWriteOps((int) x), statistics()::getWriteOps); - default -> - throw new IllegalArgumentException(String.format("Unsupported counter: '%s'", name)); - }; + switch (name) { + case READ_BYTES: + return counter(statistics()::incrementBytesRead, statistics()::getBytesRead); + case READ_OPERATIONS: + return counter((long x) -> statistics.incrementReadOps((int) x), statistics()::getReadOps); + case WRITE_BYTES: + return counter(statistics()::incrementBytesWritten, statistics()::getBytesWritten); + case WRITE_OPERATIONS: + return counter( + (long x) -> statistics.incrementWriteOps((int) x), statistics()::getWriteOps); + default: + throw new IllegalArgumentException(String.format("Unsupported counter: '%s'", name)); + } } private org.apache.iceberg.metrics.Counter counter(LongConsumer consumer, LongSupplier supplier) { diff --git a/core/src/main/java/org/apache/iceberg/io/ClusteredPositionDeleteWriter.java b/core/src/main/java/org/apache/iceberg/io/ClusteredPositionDeleteWriter.java index ba67a8a2e5c8..c9d911894c77 100644 --- a/core/src/main/java/org/apache/iceberg/io/ClusteredPositionDeleteWriter.java +++ b/core/src/main/java/org/apache/iceberg/io/ClusteredPositionDeleteWriter.java @@ -70,10 +70,14 @@ public ClusteredPositionDeleteWriter( @Override protected FileWriter, DeleteWriteResult> newWriter( PartitionSpec spec, StructLike partition) { - return switch (granularity) { - case FILE -> new FileScopedPositionDeleteWriter<>(() -> newRollingWriter(spec, partition)); - case PARTITION -> newRollingWriter(spec, partition); - }; + switch (granularity) { + case FILE: + return new FileScopedPositionDeleteWriter<>(() -> newRollingWriter(spec, partition)); + case PARTITION: + return newRollingWriter(spec, partition); + default: + throw new UnsupportedOperationException("Unsupported delete granularity: " + granularity); + } } private RollingPositionDeleteWriter newRollingWriter( diff --git a/core/src/main/java/org/apache/iceberg/metrics/TimerResultParser.java b/core/src/main/java/org/apache/iceberg/metrics/TimerResultParser.java index d76432bd1f67..52235d030744 100644 --- a/core/src/main/java/org/apache/iceberg/metrics/TimerResultParser.java +++ b/core/src/main/java/org/apache/iceberg/metrics/TimerResultParser.java @@ -119,17 +119,23 @@ private static TimeUnit toTimeUnit(String timeUnit) { } private static ChronoUnit toChronoUnit(TimeUnit unit) { - return switch (unit) { - case NANOSECONDS -> ChronoUnit.NANOS; - case MICROSECONDS -> ChronoUnit.MICROS; - case MILLISECONDS -> ChronoUnit.MILLIS; - case SECONDS -> ChronoUnit.SECONDS; - case MINUTES -> ChronoUnit.MINUTES; - case HOURS -> ChronoUnit.HOURS; - case DAYS -> ChronoUnit.DAYS; - default -> - throw new IllegalArgumentException( - "Cannot determine chrono unit from time unit: " + unit); - }; + switch (unit) { + case NANOSECONDS: + return ChronoUnit.NANOS; + case MICROSECONDS: + return ChronoUnit.MICROS; + case MILLISECONDS: + return ChronoUnit.MILLIS; + case SECONDS: + return ChronoUnit.SECONDS; + case MINUTES: + return ChronoUnit.MINUTES; + case HOURS: + return ChronoUnit.HOURS; + case DAYS: + return ChronoUnit.DAYS; + default: + throw new IllegalArgumentException("Cannot determine chrono unit from time unit: " + unit); + } } } diff --git a/core/src/main/java/org/apache/iceberg/puffin/PuffinFormat.java b/core/src/main/java/org/apache/iceberg/puffin/PuffinFormat.java index f461d7b6fd8a..7a2ee61612a9 100644 --- a/core/src/main/java/org/apache/iceberg/puffin/PuffinFormat.java +++ b/core/src/main/java/org/apache/iceberg/puffin/PuffinFormat.java @@ -104,14 +104,17 @@ static int readIntegerLittleEndian(byte[] data, int offset) { } static ByteBuffer compress(PuffinCompressionCodec codec, ByteBuffer input) { - return switch (codec) { - case NONE -> input.duplicate(); - case LZ4 -> - // TODO requires LZ4 frame compressor, e.g. - // https://github.com/airlift/aircompressor/pull/142 - throw new UnsupportedOperationException("Unsupported codec: " + codec); - case ZSTD -> compress(new ZstdCompressor(), input); - }; + switch (codec) { + case NONE: + return input.duplicate(); + case LZ4: + // TODO requires LZ4 frame compressor, e.g. + // https://github.com/airlift/aircompressor/pull/142 + break; + case ZSTD: + return compress(new ZstdCompressor(), input); + } + throw new UnsupportedOperationException("Unsupported codec: " + codec); } private static ByteBuffer compress(Compressor compressor, ByteBuffer input) { @@ -122,14 +125,20 @@ private static ByteBuffer compress(Compressor compressor, ByteBuffer input) { } static ByteBuffer decompress(PuffinCompressionCodec codec, ByteBuffer input) { - return switch (codec) { - case NONE -> input.duplicate(); - case LZ4 -> - // TODO requires LZ4 frame decompressor, e.g. - // https://github.com/airlift/aircompressor/pull/142 - throw new UnsupportedOperationException("Unsupported codec: " + codec); - case ZSTD -> decompressZstd(input); - }; + switch (codec) { + case NONE: + return input.duplicate(); + + case LZ4: + // TODO requires LZ4 frame decompressor, e.g. + // https://github.com/airlift/aircompressor/pull/142 + break; + + case ZSTD: + return decompressZstd(input); + } + + throw new UnsupportedOperationException("Unsupported codec: " + codec); } private static ByteBuffer decompressZstd(ByteBuffer input) { diff --git a/core/src/main/java/org/apache/iceberg/puffin/PuffinReader.java b/core/src/main/java/org/apache/iceberg/puffin/PuffinReader.java index 1f9ac98fd9bc..e30b6e1ee6ef 100644 --- a/core/src/main/java/org/apache/iceberg/puffin/PuffinReader.java +++ b/core/src/main/java/org/apache/iceberg/puffin/PuffinReader.java @@ -71,11 +71,13 @@ public FileMetadata fileMetadata() throws IOException { PuffinCompressionCodec footerCompression = PuffinCompressionCodec.NONE; for (Flag flag : decodeFlags(footer, footerStructOffset)) { - footerCompression = - switch (flag) { - case FOOTER_PAYLOAD_COMPRESSED -> PuffinFormat.FOOTER_COMPRESSION_CODEC; - default -> throw new IllegalStateException("Unsupported flag: " + flag); - }; + switch (flag) { + case FOOTER_PAYLOAD_COMPRESSED: + footerCompression = PuffinFormat.FOOTER_COMPRESSION_CODEC; + break; + default: + throw new IllegalStateException("Unsupported flag: " + flag); + } } int footerPayloadSize = diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index b3e82b00ecfe..3a1e62260aae 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -518,18 +518,21 @@ public static LoadTableResponse loadTable( if (table instanceof BaseTable) { TableMetadata loadedMetadata = ((BaseTable) table).operations().current(); - TableMetadata metadata = - switch (mode) { - case ALL -> loadedMetadata; - case REFS -> - TableMetadata.buildFrom(loadedMetadata) - .withMetadataLocation(loadedMetadata.metadataFileLocation()) - .suppressHistoricalSnapshots() - .build(); - default -> - throw new IllegalArgumentException( - String.format("Invalid snapshot mode: %s", mode)); - }; + TableMetadata metadata; + switch (mode) { + case ALL: + metadata = loadedMetadata; + break; + case REFS: + metadata = + TableMetadata.buildFrom(loadedMetadata) + .withMetadataLocation(loadedMetadata.metadataFileLocation()) + .suppressHistoricalSnapshots() + .build(); + break; + default: + throw new IllegalArgumentException(String.format("Invalid snapshot mode: %s", mode)); + } return LoadTableResponse.builder().withTableMetadata(metadata).build(); } else if (table instanceof BaseMetadataTable) { diff --git a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java index 7a3fcf808d47..334bfde8abfc 100644 --- a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java @@ -122,12 +122,16 @@ private static class CommitErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404 -> throw new NoSuchTableException("%s", error.message()); - case 409 -> throw new CommitFailedException("Commit failed: %s", error.message()); - case 500, 502, 503, 504 -> - throw new CommitStateUnknownException( - new ServiceFailureException( - "Service failed: %s: %s", error.code(), error.message())); + case 404: + throw new NoSuchTableException("%s", error.message()); + case 409: + throw new CommitFailedException("Commit failed: %s", error.message()); + case 500: + case 502: + case 503: + case 504: + throw new CommitStateUnknownException( + new ServiceFailureException("Service failed: %s: %s", error.code(), error.message())); } super.accept(error); @@ -141,7 +145,7 @@ private static class TableErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404 -> { + case 404: if (NoSuchNamespaceException.class.getSimpleName().equals(error.type())) { throw new NoSuchNamespaceException("%s", error.message()); } else if (NotFoundException.class.getSimpleName().equals(error.type())) { @@ -149,8 +153,8 @@ public void accept(ErrorResponse error) { } else { throw new NoSuchTableException("%s", error.message()); } - } - case 409 -> throw new AlreadyExistsException("%s", error.message()); + case 409: + throw new AlreadyExistsException("%s", error.message()); } super.accept(error); @@ -164,8 +168,10 @@ private static class CreateTableErrorHandler extends CommitErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404 -> throw new NoSuchNamespaceException("%s", error.message()); - case 409 -> throw new AlreadyExistsException("%s", error.message()); + case 404: + throw new NoSuchNamespaceException("%s", error.message()); + case 409: + throw new AlreadyExistsException("%s", error.message()); } super.accept(error); @@ -219,12 +225,16 @@ private static class ViewCommitErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404 -> throw new NoSuchViewException("%s", error.message()); - case 409 -> throw new CommitFailedException("Commit failed: %s", error.message()); - case 500, 502, 503, 504 -> - throw new CommitStateUnknownException( - new ServiceFailureException( - "Service failed: %s: %s", error.code(), error.message())); + case 404: + throw new NoSuchViewException("%s", error.message()); + case 409: + throw new CommitFailedException("Commit failed: %s", error.message()); + case 500: + case 502: + case 503: + case 504: + throw new CommitStateUnknownException( + new ServiceFailureException("Service failed: %s: %s", error.code(), error.message())); } super.accept(error); @@ -238,14 +248,14 @@ private static class ViewErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 404 -> { + case 404: if (NoSuchNamespaceException.class.getSimpleName().equals(error.type())) { throw new NoSuchNamespaceException("%s", error.message()); } else { throw new NoSuchViewException("%s", error.message()); } - } - case 409 -> throw new AlreadyExistsException("%s", error.message()); + case 409: + throw new AlreadyExistsException("%s", error.message()); } super.accept(error); @@ -259,15 +269,17 @@ private static class NamespaceErrorHandler extends DefaultErrorHandler { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 400 -> { + case 400: if (NamespaceNotEmptyException.class.getSimpleName().equals(error.type())) { throw new NamespaceNotEmptyException("%s", error.message()); } throw new BadRequestException("Malformed request: %s", error.message()); - } - case 404 -> throw new NoSuchNamespaceException("%s", error.message()); - case 409 -> throw new AlreadyExistsException("%s", error.message()); - case 422 -> throw createRESTException(error); + case 404: + throw new NoSuchNamespaceException("%s", error.message()); + case 409: + throw new AlreadyExistsException("%s", error.message()); + case 422: + throw createRESTException(error); } super.accept(error); @@ -322,21 +334,24 @@ public ErrorResponse parseResponse(int code, String json) { @Override public void accept(ErrorResponse error) { switch (error.code()) { - case 400 -> { + case 400: if (IllegalArgumentException.class.getSimpleName().equals(error.type())) { throw new IllegalArgumentException(error.message()); } throw new BadRequestException("Malformed request: %s", error.message()); - } - case 401 -> throw new NotAuthorizedException("Not authorized: %s", error.message()); - case 403 -> throw new ForbiddenException("Forbidden: %s", error.message()); - case 405, 406 -> {} - case 500 -> - throw new ServiceFailureException( - "Server error: %s: %s", error.type(), error.message()); - case 501 -> throw new UnsupportedOperationException(error.message()); - case 503 -> - throw new ServiceUnavailableException("Service unavailable: %s", error.message()); + case 401: + throw new NotAuthorizedException("Not authorized: %s", error.message()); + case 403: + throw new ForbiddenException("Forbidden: %s", error.message()); + case 405: + case 406: + break; + case 500: + throw new ServiceFailureException("Server error: %s: %s", error.type(), error.message()); + case 501: + throw new UnsupportedOperationException(error.message()); + case 503: + throw new ServiceUnavailableException("Service unavailable: %s", error.message()); } throw createRESTException(error); @@ -360,16 +375,16 @@ public ErrorResponse parseResponse(int code, String json) { public void accept(ErrorResponse error) { if (error.type() != null) { switch (error.type()) { - case OAuth2Properties.INVALID_CLIENT_ERROR -> - throw new NotAuthorizedException( - "Not authorized: %s: %s", error.type(), error.message()); - case OAuth2Properties.INVALID_REQUEST_ERROR, - OAuth2Properties.INVALID_GRANT_ERROR, - OAuth2Properties.UNAUTHORIZED_CLIENT_ERROR, - OAuth2Properties.UNSUPPORTED_GRANT_TYPE_ERROR, - OAuth2Properties.INVALID_SCOPE_ERROR -> - throw new BadRequestException( - "Malformed request: %s: %s", error.type(), error.message()); + case OAuth2Properties.INVALID_CLIENT_ERROR: + throw new NotAuthorizedException( + "Not authorized: %s: %s", error.type(), error.message()); + case OAuth2Properties.INVALID_REQUEST_ERROR: + case OAuth2Properties.INVALID_GRANT_ERROR: + case OAuth2Properties.UNAUTHORIZED_CLIENT_ERROR: + case OAuth2Properties.UNSUPPORTED_GRANT_TYPE_ERROR: + case OAuth2Properties.INVALID_SCOPE_ERROR: + throw new BadRequestException( + "Malformed request: %s: %s", error.type(), error.message()); } } throw createRESTException(error); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java b/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java index f228c3ab733e..be763d30fef1 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java @@ -160,7 +160,7 @@ public void commit(TableMetadata base, TableMetadata metadata) { List requirements; List updates; switch (updateType) { - case CREATE -> { + case CREATE: Preconditions.checkState( base == null, "Invalid base metadata for create transaction, expected null: %s", base); updates = @@ -170,8 +170,9 @@ public void commit(TableMetadata base, TableMetadata metadata) { .build(); requirements = UpdateRequirements.forCreateTable(updates); errorHandler = ErrorHandlers.createTableErrorHandler(); - } - case REPLACE -> { + break; + + case REPLACE: Preconditions.checkState(base != null, "Invalid base metadata: null"); updates = ImmutableList.builder() @@ -181,16 +182,18 @@ public void commit(TableMetadata base, TableMetadata metadata) { // use the original replace base metadata because the transaction will refresh requirements = UpdateRequirements.forReplaceTable(replaceBase, updates); errorHandler = ErrorHandlers.tableCommitHandler(); - } - case SIMPLE -> { + break; + + case SIMPLE: Preconditions.checkState(base != null, "Invalid base metadata: null"); updates = metadata.changes(); requirements = UpdateRequirements.forUpdateTable(base, updates); errorHandler = ErrorHandlers.tableCommitHandler(); - } - default -> - throw new UnsupportedOperationException( - String.format("Update type %s is not supported", updateType)); + break; + + default: + throw new UnsupportedOperationException( + String.format("Update type %s is not supported", updateType)); } UpdateTableRequest request = new UpdateTableRequest(requirements, updates); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java b/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java index 0256a0083798..9fa273ca169f 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java @@ -212,18 +212,18 @@ private CloseableIterable planTableScan(PlanTableScanRequest planT this.scanFileIO = !response.credentials().isEmpty() ? scanFileIO(response.credentials()) : table().io(); - return switch (planStatus) { - case COMPLETED -> scanTasksIterable(response.planTasks(), response.fileScanTasks()); - case SUBMITTED -> { + switch (planStatus) { + case COMPLETED: + return scanTasksIterable(response.planTasks(), response.fileScanTasks()); + case SUBMITTED: Endpoint.check(supportedEndpoints, Endpoint.V1_FETCH_TABLE_SCAN_PLAN); - yield fetchPlanningResult(); - } - case FAILED -> - throw new IllegalStateException(failureMessage(planId, response.errorResponse())); - default -> - throw new IllegalStateException( - String.format("Invalid planStatus: %s for planId: %s", planStatus, planId)); - }; + return fetchPlanningResult(); + case FAILED: + throw new IllegalStateException(failureMessage(planId, response.errorResponse())); + default: + throw new IllegalStateException( + String.format("Invalid planStatus: %s for planId: %s", planStatus, planId)); + } } private FileIO scanFileIO(List storageCredentials) { @@ -282,21 +282,24 @@ private CloseableIterable fetchPlanningResult() { parserContext); switch (response.planStatus()) { - case COMPLETED -> result.set(response); - case SUBMITTED -> throw new NotCompleteException(); - case FAILED -> - throw new IllegalStateException(failureMessage(id, response.errorResponse())); - case CANCELLED -> - throw new IllegalStateException( - String.format( - Locale.ROOT, "Remote scan planning cancelled for planId: %s", id)); - default -> - throw new IllegalStateException( - String.format( - Locale.ROOT, - "Invalid planStatus: %s for planId: %s", - response.planStatus(), - id)); + case COMPLETED: + result.set(response); + break; + case SUBMITTED: + throw new NotCompleteException(); + case FAILED: + throw new IllegalStateException(failureMessage(id, response.errorResponse())); + case CANCELLED: + throw new IllegalStateException( + String.format( + Locale.ROOT, "Remote scan planning cancelled for planId: %s", id)); + default: + throw new IllegalStateException( + String.format( + Locale.ROOT, + "Invalid planStatus: %s for planId: %s", + response.planStatus(), + id)); } }); } catch (NotCompleteException e) { diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java b/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java index f8de625d69bc..4e48118561f7 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java @@ -84,15 +84,26 @@ public static AuthManager loadAuthManager(String name, Map prope delegate = loadAuthManager(name, newProperties); } - String impl = - switch (authType.toLowerCase(Locale.ROOT)) { - case AuthProperties.AUTH_TYPE_NONE -> AuthProperties.AUTH_MANAGER_IMPL_NONE; - case AuthProperties.AUTH_TYPE_BASIC -> AuthProperties.AUTH_MANAGER_IMPL_BASIC; - case AuthProperties.AUTH_TYPE_SIGV4 -> AuthProperties.AUTH_MANAGER_IMPL_SIGV4; - case AuthProperties.AUTH_TYPE_GOOGLE -> AuthProperties.AUTH_MANAGER_IMPL_GOOGLE; - case AuthProperties.AUTH_TYPE_OAUTH2 -> AuthProperties.AUTH_MANAGER_IMPL_OAUTH2; - default -> authType; - }; + String impl; + switch (authType.toLowerCase(Locale.ROOT)) { + case AuthProperties.AUTH_TYPE_NONE: + impl = AuthProperties.AUTH_MANAGER_IMPL_NONE; + break; + case AuthProperties.AUTH_TYPE_BASIC: + impl = AuthProperties.AUTH_MANAGER_IMPL_BASIC; + break; + case AuthProperties.AUTH_TYPE_SIGV4: + impl = AuthProperties.AUTH_MANAGER_IMPL_SIGV4; + break; + case AuthProperties.AUTH_TYPE_GOOGLE: + impl = AuthProperties.AUTH_MANAGER_IMPL_GOOGLE; + break; + case AuthProperties.AUTH_TYPE_OAUTH2: + impl = AuthProperties.AUTH_MANAGER_IMPL_OAUTH2; + break; + default: + impl = authType; + } LOG.info("Loading AuthManager implementation: {}", impl); DynConstructors.Ctor ctor; diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java index 511782fe5a8f..a9504a7a4e99 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java @@ -290,17 +290,17 @@ private static Map tokenExchangeRequest( private static Pair parseCredential(String credential) { Preconditions.checkNotNull(credential, "Invalid credential: null"); List parts = CREDENTIAL_SPLITTER.splitToList(credential); - return switch (parts.size()) { - case 2 -> - // client ID and client secret - Pair.of(parts.get(0), parts.get(1)); - case 1 -> - // client secret - Pair.of(null, parts.get(0)); - default -> - // this should never happen because the credential splitter is limited to 2 - throw new IllegalArgumentException("Invalid credential: " + credential); - }; + switch (parts.size()) { + case 2: + // client ID and client secret + return Pair.of(parts.get(0), parts.get(1)); + case 1: + // client secret + return Pair.of(null, parts.get(0)); + default: + // this should never happen because the credential splitter is limited to 2 + throw new IllegalArgumentException("Invalid credential: " + credential); + } } private static Map clientCredentialsRequest( diff --git a/core/src/main/java/org/apache/iceberg/schema/SchemaWithPartnerVisitor.java b/core/src/main/java/org/apache/iceberg/schema/SchemaWithPartnerVisitor.java index 861658f1a872..694bfb2f6242 100644 --- a/core/src/main/java/org/apache/iceberg/schema/SchemaWithPartnerVisitor.java +++ b/core/src/main/java/org/apache/iceberg/schema/SchemaWithPartnerVisitor.java @@ -47,8 +47,8 @@ public static T visit( public static T visit( Type type, P partner, SchemaWithPartnerVisitor visitor, PartnerAccessors

accessors) { - return switch (type.typeId()) { - case STRUCT -> { + switch (type.typeId()) { + case STRUCT: Types.StructType struct = type.asNestedType().asStructType(); List results = Lists.newArrayListWithExpectedSize(struct.fields().size()); for (Types.NestedField field : struct.fields()) { @@ -65,9 +65,9 @@ public static T visit( } results.add(visitor.field(field, fieldPartner, result)); } - yield visitor.struct(struct, partner, results); - } - case LIST -> { + return visitor.struct(struct, partner, results); + + case LIST: Types.ListType list = type.asNestedType().asListType(); T elementResult; @@ -80,9 +80,9 @@ public static T visit( visitor.afterListElement(elementField, partnerElement); } - yield visitor.list(list, partner, elementResult); - } - case MAP -> { + return visitor.list(list, partner, elementResult); + + case MAP: Types.MapType map = type.asNestedType().asMapType(); T keyResult; T valueResult; @@ -105,11 +105,14 @@ public static T visit( visitor.afterMapValue(valueField, valuePartner); } - yield visitor.map(map, partner, keyResult, valueResult); - } - case VARIANT -> visitor.variant(type.asVariantType(), partner); - default -> visitor.primitive(type.asPrimitiveType(), partner); - }; + return visitor.map(map, partner, keyResult, valueResult); + + case VARIANT: + return visitor.variant(type.asVariantType(), partner); + + default: + return visitor.primitive(type.asPrimitiveType(), partner); + } } public void beforeField(Types.NestedField field, P partnerField) {} diff --git a/core/src/main/java/org/apache/iceberg/variants/PrimitiveWrapper.java b/core/src/main/java/org/apache/iceberg/variants/PrimitiveWrapper.java index 3ae9a3dba04b..6fd211156b00 100644 --- a/core/src/main/java/org/apache/iceberg/variants/PrimitiveWrapper.java +++ b/core/src/main/java/org/apache/iceberg/variants/PrimitiveWrapper.java @@ -82,108 +82,113 @@ public T get() { @Override public int sizeInBytes() { - return switch (type()) { - case NULL, BOOLEAN_TRUE, BOOLEAN_FALSE -> 1; // 1 header only - case INT8 -> 2; // 1 header + 1 value - case INT16 -> 3; // 1 header + 2 value - case INT32, DATE, FLOAT -> 5; // 1 header + 4 value - case INT64, DOUBLE, TIMESTAMPTZ, TIMESTAMPNTZ, TIMESTAMPTZ_NANOS, TIMESTAMPNTZ_NANOS, TIME -> - 9; // 1 header + 8 value - case DECIMAL4 -> 6; // 1 header + 1 scale + 4 unscaled value - case DECIMAL8 -> 10; // 1 header + 1 scale + 8 unscaled value - case DECIMAL16 -> 18; // 1 header + 1 scale + 16 unscaled value - case BINARY -> 5 + ((ByteBuffer) value).remaining(); // 1 header + 4 length + value length - case STRING -> { + switch (type()) { + case NULL: + case BOOLEAN_TRUE: + case BOOLEAN_FALSE: + return 1; // 1 header only + case INT8: + return 2; // 1 header + 1 value + case INT16: + return 3; // 1 header + 2 value + case INT32: + case DATE: + case FLOAT: + return 5; // 1 header + 4 value + case INT64: + case DOUBLE: + case TIMESTAMPTZ: + case TIMESTAMPNTZ: + case TIMESTAMPTZ_NANOS: + case TIMESTAMPNTZ_NANOS: + case TIME: + return 9; // 1 header + 8 value + case DECIMAL4: + return 6; // 1 header + 1 scale + 4 unscaled value + case DECIMAL8: + return 10; // 1 header + 1 scale + 8 unscaled value + case DECIMAL16: + return 18; // 1 header + 1 scale + 16 unscaled value + case BINARY: + return 5 + ((ByteBuffer) value).remaining(); // 1 header + 4 length + value length + case STRING: if (null == buffer) { this.buffer = ByteBuffer.wrap(((String) value).getBytes(StandardCharsets.UTF_8)); } if (buffer.remaining() <= MAX_SHORT_STRING_LENGTH) { - yield 1 + buffer.remaining(); // 1 header + value length + return 1 + buffer.remaining(); // 1 header + value length } - yield 5 + buffer.remaining(); // 1 header + 4 length + value length - } - case UUID -> 1 + 16; // 1 header + 16 length - default -> throw new UnsupportedOperationException("Unsupported primitive type: " + type()); - }; + return 5 + buffer.remaining(); // 1 header + 4 length + value length + case UUID: + return 1 + 16; // 1 header + 16 length + } + + throw new UnsupportedOperationException("Unsupported primitive type: " + type()); } @Override public int writeTo(ByteBuffer outBuffer, int offset) { Preconditions.checkArgument( outBuffer.order() == ByteOrder.LITTLE_ENDIAN, "Invalid byte order: big endian"); - return switch (type()) { - case NULL -> { + switch (type()) { + case NULL: outBuffer.put(offset, NULL_HEADER); - yield 1; - } - case BOOLEAN_TRUE -> { + return 1; + case BOOLEAN_TRUE: outBuffer.put(offset, TRUE_HEADER); - yield 1; - } - case BOOLEAN_FALSE -> { + return 1; + case BOOLEAN_FALSE: outBuffer.put(offset, FALSE_HEADER); - yield 1; - } - case INT8 -> { + return 1; + case INT8: outBuffer.put(offset, INT8_HEADER); outBuffer.put(offset + 1, (Byte) value); - yield 2; - } - case INT16 -> { + return 2; + case INT16: outBuffer.put(offset, INT16_HEADER); outBuffer.putShort(offset + 1, (Short) value); - yield 3; - } - case INT32 -> { + return 3; + case INT32: outBuffer.put(offset, INT32_HEADER); outBuffer.putInt(offset + 1, (Integer) value); - yield 5; - } - case INT64 -> { + return 5; + case INT64: outBuffer.put(offset, INT64_HEADER); outBuffer.putLong(offset + 1, (Long) value); - yield 9; - } - case FLOAT -> { + return 9; + case FLOAT: outBuffer.put(offset, FLOAT_HEADER); outBuffer.putFloat(offset + 1, (Float) value); - yield 5; - } - case DOUBLE -> { + return 5; + case DOUBLE: outBuffer.put(offset, DOUBLE_HEADER); outBuffer.putDouble(offset + 1, (Double) value); - yield 9; - } - case DATE -> { + return 9; + case DATE: outBuffer.put(offset, DATE_HEADER); outBuffer.putInt(offset + 1, (Integer) value); - yield 5; - } - case TIMESTAMPTZ -> { + return 5; + case TIMESTAMPTZ: outBuffer.put(offset, TIMESTAMPTZ_HEADER); outBuffer.putLong(offset + 1, (Long) value); - yield 9; - } - case TIMESTAMPNTZ -> { + return 9; + case TIMESTAMPNTZ: outBuffer.put(offset, TIMESTAMPNTZ_HEADER); outBuffer.putLong(offset + 1, (Long) value); - yield 9; - } - case DECIMAL4 -> { + return 9; + case DECIMAL4: BigDecimal decimal4 = (BigDecimal) value; outBuffer.put(offset, DECIMAL4_HEADER); outBuffer.put(offset + 1, (byte) decimal4.scale()); outBuffer.putInt(offset + 2, decimal4.unscaledValue().intValueExact()); - yield 6; - } - case DECIMAL8 -> { + return 6; + case DECIMAL8: BigDecimal decimal8 = (BigDecimal) value; outBuffer.put(offset, DECIMAL8_HEADER); outBuffer.put(offset + 1, (byte) decimal8.scale()); outBuffer.putLong(offset + 2, decimal8.unscaledValue().longValueExact()); - yield 10; - } - case DECIMAL16 -> { + return 10; + case DECIMAL16: BigDecimal decimal16 = (BigDecimal) value; byte padding = (byte) (decimal16.signum() < 0 ? 0xFF : 0x00); byte[] bytes = decimal16.unscaledValue().toByteArray(); @@ -198,53 +203,47 @@ public int writeTo(ByteBuffer outBuffer, int offset) { outBuffer.put(offset + 2 + i, padding); } } - yield 18; - } - case BINARY -> { + return 18; + case BINARY: ByteBuffer binary = (ByteBuffer) value; outBuffer.put(offset, BINARY_HEADER); outBuffer.putInt(offset + 1, binary.remaining()); outBuffer.put(offset + 5, binary, binary.position(), binary.remaining()); - yield 5 + binary.remaining(); - } - case STRING -> { + return 5 + binary.remaining(); + case STRING: if (null == buffer) { this.buffer = ByteBuffer.wrap(((String) value).getBytes(StandardCharsets.UTF_8)); } if (buffer.remaining() <= MAX_SHORT_STRING_LENGTH) { outBuffer.put(offset, VariantUtil.shortStringHeader(buffer.remaining())); outBuffer.put(offset + 1, buffer, buffer.position(), buffer.remaining()); - yield 1 + buffer.remaining(); + return 1 + buffer.remaining(); } else { outBuffer.put(offset, STRING_HEADER); outBuffer.putInt(offset + 1, buffer.remaining()); outBuffer.put(offset + 5, buffer, buffer.position(), buffer.remaining()); - yield 5 + buffer.remaining(); + return 5 + buffer.remaining(); } - } - case TIME -> { + case TIME: outBuffer.put(offset, TIME_HEADER); outBuffer.putLong(offset + 1, (Long) value); - yield 9; - } - case TIMESTAMPTZ_NANOS -> { + return 9; + case TIMESTAMPTZ_NANOS: outBuffer.put(offset, TIMESTAMPTZ_NANOS_HEADER); outBuffer.putLong(offset + 1, (Long) value); - yield 9; - } - case TIMESTAMPNTZ_NANOS -> { + return 9; + case TIMESTAMPNTZ_NANOS: outBuffer.put(offset, TIMESTAMPNTZ_NANOS_HEADER); outBuffer.putLong(offset + 1, (Long) value); - yield 9; - } - case UUID -> { + return 9; + case UUID: outBuffer.put(offset, UUID_HEADER); ByteBuffer uuidBuffer = UUIDUtil.convertToByteBuffer((UUID) value); outBuffer.put(offset + 1, uuidBuffer, uuidBuffer.position(), uuidBuffer.remaining()); - yield 17; - } - default -> throw new UnsupportedOperationException("Unsupported primitive type: " + type()); - }; + return 17; + } + + throw new UnsupportedOperationException("Unsupported primitive type: " + type()); } @Override diff --git a/core/src/main/java/org/apache/iceberg/variants/VariantVisitor.java b/core/src/main/java/org/apache/iceberg/variants/VariantVisitor.java index c89e7d03849c..3bee300f20a9 100644 --- a/core/src/main/java/org/apache/iceberg/variants/VariantVisitor.java +++ b/core/src/main/java/org/apache/iceberg/variants/VariantVisitor.java @@ -47,8 +47,8 @@ public static R visit(Variant variant, VariantVisitor visitor) { } public static R visit(VariantValue value, VariantVisitor visitor) { - return switch (value.type()) { - case ARRAY -> { + switch (value.type()) { + case ARRAY: VariantArray array = value.asArray(); List elementResults = Lists.newArrayList(); for (int index = 0; index < array.numElements(); index += 1) { @@ -60,9 +60,9 @@ public static R visit(VariantValue value, VariantVisitor visitor) { } } - yield visitor.array(array, elementResults); - } - case OBJECT -> { + return visitor.array(array, elementResults); + + case OBJECT: VariantObject object = value.asObject(); List fieldNames = Lists.newArrayList(); List fieldResults = Lists.newArrayList(); @@ -76,9 +76,10 @@ public static R visit(VariantValue value, VariantVisitor visitor) { } } - yield visitor.object(object, fieldNames, fieldResults); - } - default -> visitor.primitive(value.asPrimitive()); - }; + return visitor.object(object, fieldNames, fieldResults); + + default: + return visitor.primitive(value.asPrimitive()); + } } } diff --git a/core/src/main/java/org/apache/iceberg/view/ViewRepresentationParser.java b/core/src/main/java/org/apache/iceberg/view/ViewRepresentationParser.java index d02dbc6ad01f..79d50701ea51 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewRepresentationParser.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewRepresentationParser.java @@ -34,12 +34,14 @@ static void toJson(ViewRepresentation representation, JsonGenerator generator) throws IOException { Preconditions.checkArgument(representation != null, "Invalid view representation: null"); switch (representation.type().toLowerCase(Locale.ENGLISH)) { - case ViewRepresentation.Type.SQL -> - SQLViewRepresentationParser.toJson((SQLViewRepresentation) representation, generator); - default -> - throw new UnsupportedOperationException( - String.format( - "Cannot serialize unsupported view representation: %s", representation.type())); + case ViewRepresentation.Type.SQL: + SQLViewRepresentationParser.toJson((SQLViewRepresentation) representation, generator); + break; + + default: + throw new UnsupportedOperationException( + String.format( + "Cannot serialize unsupported view representation: %s", representation.type())); } } @@ -56,9 +58,12 @@ static ViewRepresentation fromJson(JsonNode node) { Preconditions.checkArgument( node.isObject(), "Cannot parse view representation from non-object: %s", node); String type = JsonUtil.getString(TYPE, node).toLowerCase(Locale.ENGLISH); - return switch (type) { - case ViewRepresentation.Type.SQL -> SQLViewRepresentationParser.fromJson(node); - default -> ImmutableUnknownViewRepresentation.builder().type(type).build(); - }; + switch (type) { + case ViewRepresentation.Type.SQL: + return SQLViewRepresentationParser.fromJson(node); + + default: + return ImmutableUnknownViewRepresentation.builder().type(type).build(); + } } } From 1ba42d46eb868dfd2f5a12ad4caee0ea8f0606a0 Mon Sep 17 00:00:00 2001 From: Thomas Powell Date: Wed, 24 Jun 2026 22:40:46 +0100 Subject: [PATCH 37/73] Remove encryption key from keysById first to avoid removeIf (#16860) Co-authored-by: Thomas Powell --- core/src/main/java/org/apache/iceberg/TableMetadata.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index b811d8c42f15..07f6d3342727 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -1519,10 +1519,10 @@ public Builder addEncryptionKey(EncryptedKey key) { } public Builder removeEncryptionKey(String keyId) { - boolean removed = encryptionKeys.removeIf(key -> key.keyId().equals(keyId)); - keysById.remove(keyId); + EncryptedKey removedKey = keysById.remove(keyId); - if (removed) { + if (removedKey != null) { + encryptionKeys.remove(removedKey); changes.add(new MetadataUpdate.RemoveEncryptionKey(keyId)); } From d3daeef03146d817bbd763e9ea330293722ac88a Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Thu, 25 Jun 2026 00:46:28 +0200 Subject: [PATCH 38/73] Build: Align Jackson versions to fix CVE (#16954) --- .../kafka-connect-runtime.trivyignore | 29 +++++++++++++++++++ .../trivyignore/spark-runtime-3.5.trivyignore | 27 +++++++++++++++++ .github/workflows/cve-scan.yml | 5 ++++ azure-bundle/build.gradle | 3 ++ azure-bundle/runtime-deps.txt | 8 ++--- gcp-bundle/build.gradle | 3 ++ gcp-bundle/runtime-deps.txt | 10 +++---- gradle/libs.versions.toml | 5 ++-- spark/v4.0/build.gradle | 6 ++-- spark/v4.0/spark-runtime/runtime-deps.txt | 4 +-- spark/v4.1/build.gradle | 6 ++-- spark/v4.1/spark-runtime/runtime-deps.txt | 4 +-- 12 files changed, 88 insertions(+), 22 deletions(-) create mode 100644 .github/trivyignore/kafka-connect-runtime.trivyignore create mode 100644 .github/trivyignore/spark-runtime-3.5.trivyignore diff --git a/.github/trivyignore/kafka-connect-runtime.trivyignore b/.github/trivyignore/kafka-connect-runtime.trivyignore new file mode 100644 index 000000000000..2da8c6d2f484 --- /dev/null +++ b/.github/trivyignore/kafka-connect-runtime.trivyignore @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Trivy ignore file for the Kafka Connect runtime distribution. +# +# The Kafka Connect runtime's own jackson-databind is already aligned to the +# project version (2.22.x). The remaining finding comes from parquet-jackson, +# which ships its own shaded copy of jackson-databind (relocated under +# shaded.parquet.*). That copy cannot be upgraded independently of Apache +# Parquet, and 1.17.1 still bundles a vulnerable version. +# +# jackson-databind CVEs shaded into parquet-jackson, only fixed in jackson +# >= 2.21.4 / 2.18.8. +CVE-2026-54512 +CVE-2026-54513 diff --git a/.github/trivyignore/spark-runtime-3.5.trivyignore b/.github/trivyignore/spark-runtime-3.5.trivyignore new file mode 100644 index 000000000000..a8f4671634c5 --- /dev/null +++ b/.github/trivyignore/spark-runtime-3.5.trivyignore @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Trivy ignore file for the Spark 3.5 runtime bundle. +# +# The Spark 3.5 runtime bundles the Jackson version provided by Spark 3.5 +# (jackson 2.15.x), which cannot be safely upgraded without breaking +# compatibility with Spark 3.5. Newer Spark lines (4.0, 4.1) align Jackson to a +# fixed version instead of ignoring the finding. +# +# jackson-databind CVEs that are only fixed in jackson >= 2.18.8. +CVE-2026-54512 +CVE-2026-54513 diff --git a/.github/workflows/cve-scan.yml b/.github/workflows/cve-scan.yml index 22f3401b64de..043d2378bd43 100644 --- a/.github/workflows/cve-scan.yml +++ b/.github/workflows/cve-scan.yml @@ -31,6 +31,7 @@ on: - '**' - '!.github/**' - '.github/workflows/cve-scan.yml' + - '.github/trivyignore/**' - '!.baseline/**' - '!.palantir/**' - '!.gitignore' @@ -54,6 +55,7 @@ on: - '**' - '!.github/**' - '.github/workflows/cve-scan.yml' + - '.github/trivyignore/**' - '!.baseline/**' - '!.palantir/**' - '!.gitignore' @@ -113,6 +115,7 @@ jobs: :iceberg-kafka-connect:iceberg-kafka-connect-runtime:distZip scan-path: kafka-connect/kafka-connect-runtime/build/distributions unpack: true + trivyignores: .github/trivyignore/kafka-connect-runtime.trivyignore - distribution: aws-bundle build-task: :iceberg-aws-bundle:shadowJar scan-path: aws-bundle/build/libs @@ -131,6 +134,7 @@ jobs: :iceberg-spark:iceberg-spark-runtime-3.5_2.12:shadowJar scan-path: spark/v3.5/spark-runtime/build/libs unpack: false + trivyignores: .github/trivyignore/spark-runtime-3.5.trivyignore - distribution: spark-runtime-4.0_2.13 build-task: >- -DsparkVersions=4.0 @@ -214,6 +218,7 @@ jobs: scan-ref: '/tmp/cve-scan' scanners: 'vuln' severity: 'HIGH,CRITICAL' + trivyignores: ${{ matrix.trivyignores }} limit-severities-for-sarif: true # Block PRs on CVE findings; on main/release branches report without failing exit-code: ${{ github.event_name == 'pull_request' && '1' || '0' }} diff --git a/azure-bundle/build.gradle b/azure-bundle/build.gradle index fde8adbfc539..8d0fe41a5789 100644 --- a/azure-bundle/build.gradle +++ b/azure-bundle/build.gradle @@ -31,6 +31,9 @@ project(":iceberg-azure-bundle") { dependencies { implementation platform(libs.azuresdk.bom) + // Align the shaded Jackson to the project version; the Azure SDK BOM otherwise + // pulls in an older jackson-databind with known CVEs. + implementation platform(libs.jackson.bom) implementation "com.azure:azure-storage-file-datalake" implementation "com.azure:azure-security-keyvault-keys" implementation "com.azure:azure-identity" diff --git a/azure-bundle/runtime-deps.txt b/azure-bundle/runtime-deps.txt index 2caed26153e8..adb6044657ee 100644 --- a/azure-bundle/runtime-deps.txt +++ b/azure-bundle/runtime-deps.txt @@ -8,10 +8,10 @@ com.azure:azure-storage-common:12.33 com.azure:azure-storage-file-datalake:12.27 com.azure:azure-storage-internal-avro:12.19 com.azure:azure-xml:1.2 -com.fasterxml.jackson.core:jackson-annotations:2.18 -com.fasterxml.jackson.core:jackson-core:2.18 -com.fasterxml.jackson.core:jackson-databind:2.18 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18 +com.fasterxml.jackson.core:jackson-annotations:2.22 +com.fasterxml.jackson.core:jackson-core:2.22 +com.fasterxml.jackson.core:jackson-databind:2.22 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22 com.microsoft.azure:msal4j-persistence-extension:1.3 com.microsoft.azure:msal4j:1.23 io.netty:netty-buffer:4.2 diff --git a/gcp-bundle/build.gradle b/gcp-bundle/build.gradle index 9c4907bcdaa5..39b230745b9b 100644 --- a/gcp-bundle/build.gradle +++ b/gcp-bundle/build.gradle @@ -32,6 +32,9 @@ project(":iceberg-gcp-bundle") { dependencies { implementation platform(libs.google.libraries.bom) + // Align the shaded Jackson to the project version; the Google libraries BOM + // otherwise pulls in an older jackson-databind with known CVEs. + implementation platform(libs.jackson.bom) implementation "com.google.cloud:google-cloud-storage" implementation "com.google.cloud:google-cloud-bigquery" implementation "com.google.cloud:google-cloud-core" diff --git a/gcp-bundle/runtime-deps.txt b/gcp-bundle/runtime-deps.txt index 48d68619d892..84cc4fe3c0f3 100644 --- a/gcp-bundle/runtime-deps.txt +++ b/gcp-bundle/runtime-deps.txt @@ -1,8 +1,8 @@ -com.fasterxml.jackson.core:jackson-annotations:2.18 -com.fasterxml.jackson.core:jackson-core:2.18 -com.fasterxml.jackson.core:jackson-databind:2.18 -com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.18 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18 +com.fasterxml.jackson.core:jackson-annotations:2.22 +com.fasterxml.jackson.core:jackson-core:2.22 +com.fasterxml.jackson.core:jackson-databind:2.22 +com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.22 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22 com.fasterxml.woodstox:woodstox-core:7.0 com.github.ben-manes.caffeine:caffeine:3.2 com.google.android:annotations:4.1 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b859567560a0..d10badc2e4af 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -60,8 +60,9 @@ hive2 = { strictly = "2.3.10"} # see rich version usage explanation above immutables-value = "2.12.2" jackson-annotations = "2.22" jackson-bom = "2.22.0" -jackson214 = { strictly = "2.14.2"} jackson215 = { strictly = "2.15.2"} # see rich version usage explanation above +jackson218 = { strictly = "2.18.8"} +jackson221 = { strictly = "2.21.4"} jakarta-el-api = "3.0.3" jakarta-servlet-api = "6.1.0" jaxb-api = "2.3.1" @@ -157,8 +158,6 @@ jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jac jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version.ref = "jackson-bom" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson-bom" } jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "jackson-annotations" } -jackson214-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson214" } -jackson215-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson215" } jaxb-api = { module = "javax.xml.bind:jaxb-api", version.ref = "jaxb-api" } jaxb-runtime = { module = "org.glassfish.jaxb:jaxb-runtime", version.ref = "jaxb-runtime" } kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafka" } diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle index 3707e01e4865..8557115cedd1 100644 --- a/spark/v4.0/build.gradle +++ b/spark/v4.0/build.gradle @@ -30,9 +30,9 @@ configure(sparkProjects) { configurations { all { resolutionStrategy { - force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson215.get()}" - force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson215.get()}" - force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson215.get()}" + force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson218.get()}" + force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson218.get()}" + force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson218.get()}" } } } diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt index 18c17ad30177..97841db98914 100644 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -1,6 +1,6 @@ com.fasterxml.jackson.core:jackson-annotations:2.22 -com.fasterxml.jackson.core:jackson-core:2.15 -com.fasterxml.jackson.core:jackson-databind:2.15 +com.fasterxml.jackson.core:jackson-core:2.18 +com.fasterxml.jackson.core:jackson-databind:2.18 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22 com.github.ben-manes.caffeine:caffeine:2.9 com.google.errorprone:error_prone_annotations:2.10 diff --git a/spark/v4.1/build.gradle b/spark/v4.1/build.gradle index e6455fa34f88..36e7f2672262 100644 --- a/spark/v4.1/build.gradle +++ b/spark/v4.1/build.gradle @@ -30,9 +30,9 @@ configure(sparkProjects) { configurations { all { resolutionStrategy { - force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson215.get()}" - force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson215.get()}" - force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson215.get()}" + force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson221.get()}" + force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson221.get()}" + force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson221.get()}" } } } diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt index 18c17ad30177..db0e53357807 100644 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -1,6 +1,6 @@ com.fasterxml.jackson.core:jackson-annotations:2.22 -com.fasterxml.jackson.core:jackson-core:2.15 -com.fasterxml.jackson.core:jackson-databind:2.15 +com.fasterxml.jackson.core:jackson-core:2.21 +com.fasterxml.jackson.core:jackson-databind:2.21 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22 com.github.ben-manes.caffeine:caffeine:2.9 com.google.errorprone:error_prone_annotations:2.10 From 2a6c556c0c883c04c6d3fbd68e6aeda36b91e0aa Mon Sep 17 00:00:00 2001 From: gaborkaszab Date: Thu, 25 Jun 2026 01:26:58 +0200 Subject: [PATCH 39/73] Core: Rename TrackedFile writer_format_version to format_version (#16952) --- .../java/org/apache/iceberg/TrackedFile.java | 10 +- .../apache/iceberg/TrackedFileBuilder.java | 19 +-- .../org/apache/iceberg/TrackedFileStruct.java | 141 ++++++------------ .../org/apache/iceberg/TestTrackedFile.java | 2 +- .../iceberg/TestTrackedFileAdapters.java | 8 +- .../iceberg/TestTrackedFileBuilder.java | 50 +++---- .../apache/iceberg/TestTrackedFileStruct.java | 20 +-- 7 files changed, 100 insertions(+), 150 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 7dd7e1b3b48d..5b9cdd9ab46c 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -35,9 +35,9 @@ interface TrackedFile { "content_type", Types.IntegerType.get(), "Type of content: 0=DATA, 2=EQUALITY_DELETES, 3=DATA_MANIFEST, 4=DELETE_MANIFEST"); - Types.NestedField WRITER_FORMAT_VERSION = + Types.NestedField FORMAT_VERSION = Types.NestedField.required( - 157, "writer_format_version", Types.IntegerType.get(), "Writer format version"); + 157, "format_version", Types.IntegerType.get(), "Format version of this file"); Types.NestedField LOCATION = Types.NestedField.required(100, "location", Types.StringType.get(), "Location of the file"); Types.NestedField FILE_FORMAT = @@ -100,7 +100,7 @@ static Types.StructType schemaWithContentStats( return Types.StructType.of( TRACKING, CONTENT_TYPE, - WRITER_FORMAT_VERSION, + FORMAT_VERSION, LOCATION, FILE_FORMAT, RECORD_COUNT, @@ -123,8 +123,8 @@ static Types.StructType schemaWithContentStats( /** Returns the type of content stored by this entry. */ FileContent contentType(); - /** Returns the version of the writer that wrote this file */ - int writerFormatVersion(); + /** Returns the format version of this file. */ + int formatVersion(); /** Returns the location of the file. */ String location(); diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java b/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java index cfffd0b9c8ca..d7aa28c3290f 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java @@ -27,7 +27,7 @@ class TrackedFileBuilder { private final FileContent contentType; // Required fields - private Integer writerFormatVersion = null; + private Integer formatVersion = null; private String location = null; private FileFormat fileFormat = null; private Long recordCount = null; @@ -129,7 +129,7 @@ private static TrackedFile terminal(TrackedFile source, Tracking tracking) { return new TrackedFileStruct( tracking, source.contentType(), - source.writerFormatVersion(), + source.formatVersion(), source.location(), source.fileFormat(), (PartitionData) source.partition(), @@ -153,7 +153,7 @@ private TrackedFileBuilder(FileContent contentType, long snapshotId) { private TrackedFileBuilder(TrackedFile source, long snapshotId) { this.contentType = source.contentType(); this.snapshotId = snapshotId; - this.writerFormatVersion = source.writerFormatVersion(); + this.formatVersion = source.formatVersion(); this.location = source.location(); this.fileFormat = source.fileFormat(); this.recordCount = source.recordCount(); @@ -170,12 +170,10 @@ private TrackedFileBuilder(TrackedFile source, long snapshotId) { this.sourceTracking = source.tracking(); } - TrackedFileBuilder writerFormatVersion(int newWriterFormatVersion) { + TrackedFileBuilder formatVersion(int newFormatVersion) { Preconditions.checkArgument( - newWriterFormatVersion >= 0, - "Invalid writer format version: %s (must be >= 0)", - newWriterFormatVersion); - this.writerFormatVersion = newWriterFormatVersion; + newFormatVersion >= 0, "Invalid format version: %s (must be >= 0)", newFormatVersion); + this.formatVersion = newFormatVersion; return this; } @@ -311,8 +309,7 @@ private static boolean isLeafManifest(FileContent contentType) { } TrackedFile build() { - Preconditions.checkArgument( - writerFormatVersion != null, "Missing required field: writer format version"); + Preconditions.checkArgument(formatVersion != null, "Missing required field: format version"); Preconditions.checkArgument(location != null, "Missing required field: location"); Preconditions.checkArgument(fileFormat != null, "Missing required field: file format"); Preconditions.checkArgument(recordCount != null, "Missing required field: record count"); @@ -346,7 +343,7 @@ TrackedFile build() { return new TrackedFileStruct( trackingBuilder.build(), contentType, - writerFormatVersion, + formatVersion, location, fileFormat, partitionData, diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 9a44e8045cbb..958ddfbbc4cf 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -45,7 +45,7 @@ public PartitionData copy() { Types.StructType.of( TrackedFile.TRACKING, TrackedFile.CONTENT_TYPE, - TrackedFile.WRITER_FORMAT_VERSION, + TrackedFile.FORMAT_VERSION, TrackedFile.LOCATION, TrackedFile.FILE_FORMAT, TrackedFile.RECORD_COUNT, @@ -69,7 +69,7 @@ public PartitionData copy() { TrackedFile.EQUALITY_IDS); private FileContent contentType = null; - private int writerFormatVersion = -1; + private int formatVersion = -1; private String location = null; private FileFormat fileFormat = null; private Tracking tracking = null; @@ -105,7 +105,7 @@ public PartitionData copy() { TrackedFileStruct( Tracking tracking, FileContent contentType, - int writerFormatVersion, + int formatVersion, String location, FileFormat fileFormat, PartitionData partition, @@ -122,7 +122,7 @@ public PartitionData copy() { super(BASE_TYPE.fields().size()); this.tracking = tracking; this.contentType = contentType; - this.writerFormatVersion = writerFormatVersion; + this.formatVersion = formatVersion; this.location = location; this.fileFormat = fileFormat; this.recordCount = recordCount; @@ -145,7 +145,7 @@ public PartitionData copy() { private TrackedFileStruct(TrackedFileStruct toCopy, boolean withStats, Set statsIds) { super(toCopy); this.contentType = toCopy.contentType; - this.writerFormatVersion = toCopy.writerFormatVersion; + this.formatVersion = toCopy.formatVersion; this.location = toCopy.location; this.fileFormat = toCopy.fileFormat; this.recordCount = toCopy.recordCount; @@ -189,8 +189,8 @@ public FileContent contentType() { } @Override - public int writerFormatVersion() { - return writerFormatVersion; + public int formatVersion() { + return formatVersion; } @Override @@ -274,98 +274,51 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - switch (pos) { - case 0: - return tracking; - case 1: - return contentType != null ? contentType.id() : null; - case 2: - return writerFormatVersion; - case 3: - return location; - case 4: - return fileFormat != null ? fileFormat.toString() : null; - case 5: - return recordCount; - case 6: - return fileSizeInBytes; - case 7: - return specId; - case 8: - return partitionData; - case 9: - return contentStats; - case 10: - return sortOrderId; - case 11: - return deletionVector; - case 12: - return manifestInfo; - case 13: - return keyMetadata(); - case 14: - return splitOffsets(); - case 15: - return equalityIds(); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + return switch (pos) { + case 0 -> tracking; + case 1 -> contentType != null ? contentType.id() : null; + case 2 -> formatVersion; + case 3 -> location; + case 4 -> fileFormat != null ? fileFormat.toString() : null; + case 5 -> recordCount; + case 6 -> fileSizeInBytes; + case 7 -> specId; + case 8 -> partitionData; + case 9 -> contentStats; + case 10 -> sortOrderId; + case 11 -> deletionVector; + case 12 -> manifestInfo; + case 13 -> keyMetadata(); + case 14 -> splitOffsets(); + case 15 -> equalityIds(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0: - this.tracking = (Tracking) value; - break; - case 1: - this.contentType = FileContent.fromId((Integer) value); - break; - case 2: - this.writerFormatVersion = (int) value; - break; - case 3: - // always coerce to String for Serializable - this.location = value.toString(); - break; - case 4: - this.fileFormat = FileFormat.fromString(value.toString()); - break; - case 5: - this.recordCount = (long) value; - break; - case 6: - this.fileSizeInBytes = (long) value; - break; - case 7: - this.specId = (Integer) value; - break; - case 8: - this.partitionData = (PartitionData) value; - break; - case 9: - this.contentStats = (ContentStats) value; - break; - case 10: - this.sortOrderId = (Integer) value; - break; - case 11: - this.deletionVector = (DeletionVector) value; - break; - case 12: - this.manifestInfo = (ManifestInfo) value; - break; - case 13: - this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); - break; - case 14: - this.splitOffsets = ArrayUtil.toLongArray((List) value); - break; - case 15: - this.equalityIds = ArrayUtil.toIntArray((List) value); - break; - default: + case 0 -> this.tracking = (Tracking) value; + case 1 -> this.contentType = FileContent.fromId((Integer) value); + case 2 -> this.formatVersion = (int) value; + case 3 -> + // always coerce to String for Serializable + this.location = value.toString(); + case 4 -> this.fileFormat = FileFormat.fromString(value.toString()); + case 5 -> this.recordCount = (long) value; + case 6 -> this.fileSizeInBytes = (long) value; + case 7 -> this.specId = (Integer) value; + case 8 -> this.partitionData = (PartitionData) value; + case 9 -> this.contentStats = (ContentStats) value; + case 10 -> this.sortOrderId = (Integer) value; + case 11 -> this.deletionVector = (DeletionVector) value; + case 12 -> this.manifestInfo = (ManifestInfo) value; + case 13 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + case 14 -> this.splitOffsets = ArrayUtil.toLongArray((List) value); + case 15 -> this.equalityIds = ArrayUtil.toIntArray((List) value); + default -> { // ignore the object, it must be from a newer version of the format + } } } @@ -373,7 +326,7 @@ protected void internalSet(int pos, T value) { public String toString() { return MoreObjects.toStringHelper(this) .add("content", contentType != null ? contentType.lowerCaseName() : null) - .add("writer_format_version", writerFormatVersion) + .add("format_version", formatVersion) .add("location", location) .add("file_format", fileFormat) .add("record_count", recordCount) diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java index 2c4bf93e4945..170c01ef7dc4 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java @@ -47,7 +47,7 @@ public void schemaWithContentStatsFieldOrder() { .containsExactly( "tracking", "content_type", - "writer_format_version", + "format_version", "location", "file_format", "record_count", diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index dc8f26dce8a8..f4d5675ee94a 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -34,7 +34,7 @@ class TestTrackedFileAdapters { - private static final int WRITER_FORMAT_VERSION = 4; + private static final int FORMAT_VERSION_V4 = 4; private static final String MANIFEST_LOCATION = "s3://bucket/table/manifest.parquet"; private static final String DATA_FILE_LOCATION = "s3://bucket/data/file.parquet"; private static final String DV_LOCATION = "s3://bucket/puffin/dv-file.bin"; @@ -80,7 +80,7 @@ class TestTrackedFileAdapters { void testDataFileAdapterDelegation() { TrackedFile file = TrackedFileBuilder.data(42L) - .writerFormatVersion(WRITER_FORMAT_VERSION) + .formatVersion(FORMAT_VERSION_V4) .location(DATA_FILE_LOCATION) .fileFormat(FileFormat.PARQUET) .partition(PARTITION) @@ -140,7 +140,7 @@ void testDataFileAdapterRejectsNonDataContent(FileContent contentType) { void testEqualityDeleteFileAdapterDelegation() { TrackedFile file = TrackedFileBuilder.equalityDelete(42L) - .writerFormatVersion(WRITER_FORMAT_VERSION) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/eq-delete.avro") .fileFormat(FileFormat.AVRO) .partition(PARTITION) @@ -210,7 +210,7 @@ void testDVDeleteFileAdapterDelegation() { TrackedFile file = TrackedFileBuilder.data(42L) - .writerFormatVersion(WRITER_FORMAT_VERSION) + .formatVersion(FORMAT_VERSION_V4) .location(DATA_FILE_LOCATION) .fileFormat(FileFormat.PARQUET) .partition(PARTITION) diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java index 9e96923f2fc0..d6b0701fdfd7 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java @@ -32,7 +32,7 @@ import org.junit.jupiter.params.provider.MethodSource; public class TestTrackedFileBuilder { - private static final int WRITER_FORMAT_VERSION_V4 = 4; + private static final int FORMAT_VERSION_V4 = 4; private static final Schema TABLE_SCHEMA = new Schema( optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get())); @@ -87,7 +87,7 @@ public class TestTrackedFileBuilder { private static Stream missingRequiredFieldCases() { return Stream.of( - Arguments.of("writerFormatVersion", "Missing required field: writer format version"), + Arguments.of("formatVersion", "Missing required field: format version"), Arguments.of("location", "Missing required field: location"), Arguments.of("fileFormat", "Missing required field: file format"), Arguments.of("recordCount", "Missing required field: record count"), @@ -123,8 +123,8 @@ public void missingRequiredFields(String missingField, String expectedMessage) { private TrackedFileBuilder builderWithMissingRequiredField( TrackedFileBuilder builder, String missingField) { - if (!"writerFormatVersion".equals(missingField)) { - builder.writerFormatVersion(WRITER_FORMAT_VERSION_V4); + if (!"formatVersion".equals(missingField)) { + builder.formatVersion(FORMAT_VERSION_V4); } if (!"location".equals(missingField)) { builder.location("s3://bucket/data/file"); @@ -150,7 +150,7 @@ public void missingFieldsForManifests(TrackedFileBuilder builder, FileContent co assertThatThrownBy( () -> builder - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/manifest.avro") .fileFormat(FileFormat.AVRO) .recordCount(420L) @@ -166,7 +166,7 @@ public void missingEqualityIdsForEqualityDeletes() { assertThatThrownBy( () -> TrackedFileBuilder.equalityDelete(50L) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/eq_delete.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(2000L) @@ -353,9 +353,9 @@ public void invalidNullInputs() { @Test public void invalidNegativeInputs() { - assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).writerFormatVersion(-1)) + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).formatVersion(-1)) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid writer format version: -1 (must be >= 0)"); + .hasMessage("Invalid format version: -1 (must be >= 0)"); assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).recordCount(-1)) .isInstanceOf(IllegalArgumentException.class) @@ -378,7 +378,7 @@ public void invalidNegativeInputs() { public void buildDataFileWithRequiredFieldsOnly() { TrackedFile trackedFile = TrackedFileBuilder.data(50L) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/file.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(2000L) @@ -386,7 +386,7 @@ public void buildDataFileWithRequiredFieldsOnly() { .partition(PARTITION_DATA) .build(); - assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA); assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/file.parquet"); assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); @@ -412,7 +412,7 @@ public void buildDataFileWithRequiredFieldsOnly() { public void buildDataFileWithAllFields() { TrackedFile trackedFile = TrackedFileBuilder.data(50L) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/file.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(2000L) @@ -426,7 +426,7 @@ public void buildDataFileWithAllFields() { .splitOffsets(SPLIT_OFFSETS) .build(); - assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA); assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/file.parquet"); assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); @@ -453,7 +453,7 @@ public void buildDataFileWithAllFields() { public void buildEqualityDeleteFileWithRequiredFieldsOnly() { TrackedFile trackedFile = TrackedFileBuilder.equalityDelete(50L) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/eq_delete.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(2000L) @@ -462,7 +462,7 @@ public void buildEqualityDeleteFileWithRequiredFieldsOnly() { .equalityIds(ImmutableList.of(1)) .build(); - assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(trackedFile.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/eq_delete.parquet"); assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); @@ -487,7 +487,7 @@ public void buildEqualityDeleteFileWithRequiredFieldsOnly() { public void buildEqualityDeleteFileWithAllFields() { TrackedFile trackedFile = TrackedFileBuilder.equalityDelete(50L) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/eq_delete.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(2000L) @@ -501,7 +501,7 @@ public void buildEqualityDeleteFileWithAllFields() { .equalityIds(ImmutableList.of(1, 2)) .build(); - assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(trackedFile.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/eq_delete.parquet"); assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); @@ -535,7 +535,7 @@ public void buildManifestWithRequiredFieldsOnly( TrackedFileBuilder builder, FileContent contentType) { TrackedFile trackedFile = builder - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/manifest.avro") .fileFormat(FileFormat.AVRO) .recordCount(420L) @@ -544,7 +544,7 @@ public void buildManifestWithRequiredFieldsOnly( .manifestInfo(MANIFEST_INFO) .build(); - assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(trackedFile.contentType()).isEqualTo(contentType); assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/manifest.avro"); assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.AVRO); @@ -570,7 +570,7 @@ public void buildManifestWithRequiredFieldsOnly( public void buildManifestWithAllFields(TrackedFileBuilder builder, FileContent contentType) { TrackedFile trackedFile = builder - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/manifest.avro") .fileFormat(FileFormat.AVRO) .recordCount(420L) @@ -582,7 +582,7 @@ public void buildManifestWithAllFields(TrackedFileBuilder builder, FileContent c .manifestInfo(MANIFEST_INFO) .build(); - assertThat(trackedFile.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(trackedFile.contentType()).isEqualTo(contentType); assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/manifest.avro"); assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.AVRO); @@ -770,7 +770,7 @@ public void replacedFromManifestSourceFails(TrackedFile source, FileContent cont private static TrackedFile sourceData(long snapshotId) { return TrackedFileBuilder.data(snapshotId) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/file.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(2000L) @@ -787,7 +787,7 @@ private static TrackedFile sourceData(long snapshotId) { private static TrackedFile sourceEqualityDelete(long snapshotId) { return TrackedFileBuilder.equalityDelete(snapshotId) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/eq_delete.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(2000L) @@ -799,7 +799,7 @@ private static TrackedFile sourceEqualityDelete(long snapshotId) { private static TrackedFile sourceDataManifest(long snapshotId) { return TrackedFileBuilder.dataManifest(snapshotId) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/data_manifest.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(420L) @@ -811,7 +811,7 @@ private static TrackedFile sourceDataManifest(long snapshotId) { private static TrackedFile sourceDeleteManifest(long snapshotId) { return TrackedFileBuilder.deleteManifest(snapshotId) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/delete_manifest.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(100L) @@ -835,7 +835,7 @@ private static TrackedFile entryWithInheritedSeqNums(TrackedFile entry, long seq * here, because based on the entry's status it is either carried over or not. */ private static void verifyFieldsAreFromSource(TrackedFile entry, TrackedFile source) { - assertThat(entry.writerFormatVersion()).isEqualTo(source.writerFormatVersion()); + assertThat(entry.formatVersion()).isEqualTo(source.formatVersion()); assertThat(entry.location()).isEqualTo(source.location()); assertThat(entry.fileFormat()).isEqualTo(source.fileFormat()); assertThat(entry.recordCount()).isEqualTo(source.recordCount()); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 4f5452e31455..8e0d5c06824b 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; class TestTrackedFileStruct { - private static final int WRITER_FORMAT_VERSION_V4 = 4; + private static final int FORMAT_VERSION_V4 = 4; private static final Types.StructType PARTITION_TYPE = Types.StructType.of( Types.NestedField.optional(1000, "id_bucket", Types.IntegerType.get()), @@ -41,8 +41,8 @@ class TestTrackedFileStruct { TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()).fields(); private static final int TRACKING_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.TRACKING); private static final int CONTENT_TYPE_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.CONTENT_TYPE); - private static final int WRITER_FORMAT_VERSION_ORDINAL = - SCHEMA_FIELDS.indexOf(TrackedFile.WRITER_FORMAT_VERSION); + private static final int FORMAT_VERSION_ORDINAL = + SCHEMA_FIELDS.indexOf(TrackedFile.FORMAT_VERSION); private static final int LOCATION_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.LOCATION); private static final int FILE_FORMAT_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.FILE_FORMAT); private static final int RECORD_COUNT_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.RECORD_COUNT); @@ -100,7 +100,7 @@ void testFieldAccess() { file.set(TRACKING_ORDINAL, tracking); file.set(CONTENT_TYPE_ORDINAL, FileContent.EQUALITY_DELETES.id()); - file.set(WRITER_FORMAT_VERSION_ORDINAL, WRITER_FORMAT_VERSION_V4); + file.set(FORMAT_VERSION_ORDINAL, FORMAT_VERSION_V4); file.set(LOCATION_ORDINAL, "s3://bucket/data/eq-delete.avro"); file.set(FILE_FORMAT_ORDINAL, "avro"); file.set(RECORD_COUNT_ORDINAL, 50L); @@ -117,7 +117,7 @@ void testFieldAccess() { assertThat(file.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(file.tracking().snapshotId()).isEqualTo(42L); assertThat(file.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); - assertThat(file.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(file.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(file.location()).isEqualTo("s3://bucket/data/eq-delete.avro"); assertThat(file.fileFormat()).isEqualTo(FileFormat.AVRO); assertThat(file.recordCount()).isEqualTo(50L); @@ -209,7 +209,7 @@ void testCopy() { assertThat(copy.sortOrderId()).isEqualTo(1); assertThat(copy.recordCount()).isEqualTo(100L); assertThat(copy.fileSizeInBytes()).isEqualTo(1024L); - assertThat(copy.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(copy.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(copy.keyMetadata()).isNotNull(); assertThat(copy.splitOffsets()).containsExactly(50L); assertThat(copy.equalityIds()).isNull(); @@ -332,7 +332,7 @@ void testJavaSerializationRoundTrip() throws IOException, ClassNotFoundException TrackedFileStruct deserialized = TestHelpers.roundTripSerialize(file); assertThat(deserialized.contentType()).isEqualTo(FileContent.DATA); - assertThat(deserialized.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(deserialized.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(deserialized.location()).isEqualTo("s3://bucket/data/file.parquet"); assertThat(deserialized.fileFormat()).isEqualTo(FileFormat.PARQUET); assertThat(deserialized.recordCount()).isEqualTo(100L); @@ -356,7 +356,7 @@ void testKryoSerializationRoundTrip() throws IOException { TrackedFileStruct deserialized = TestHelpers.KryoHelpers.roundTripSerialize(file); assertThat(deserialized.contentType()).isEqualTo(FileContent.DATA); - assertThat(deserialized.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(deserialized.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(deserialized.location()).isEqualTo("s3://bucket/data/file.parquet"); assertThat(deserialized.fileFormat()).isEqualTo(FileFormat.PARQUET); assertThat(deserialized.recordCount()).isEqualTo(100L); @@ -385,7 +385,7 @@ static TrackedFileStruct createFullTrackedFile() { TrackedFileStruct file = (TrackedFileStruct) TrackedFileBuilder.data(42L) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/file.parquet") .fileFormat(FileFormat.PARQUET) .partition(newPartition(7, "music")) @@ -462,7 +462,7 @@ static TrackedFileStruct createTrackedFileWithStats() { return (TrackedFileStruct) TrackedFileBuilder.data(0L) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/data/file.parquet") .fileFormat(FileFormat.PARQUET) .partition(new PartitionData(Types.StructType.of())) From b27930e1827b7ff00d76b36c2037ac93d2bb3bed Mon Sep 17 00:00:00 2001 From: Swapna Marru Date: Wed, 24 Jun 2026 22:06:30 -0700 Subject: [PATCH 40/73] Flink: SQL: Pass only white-listed Catalog properties for Table LIKE (#16728) --- .../apache/iceberg/flink/FlinkCatalog.java | 10 +++----- .../iceberg/flink/FlinkCatalogFactory.java | 1 - .../flink/FlinkCreateTableOptions.java | 23 +++---------------- .../flink/FlinkDynamicTableFactory.java | 11 ++++----- .../apache/iceberg/flink/CatalogTestBase.java | 1 + .../iceberg/flink/TestFlinkCatalogTable.java | 11 +++------ .../flink/source/TestIcebergSourceSql.java | 15 +++++++++--- 7 files changed, 27 insertions(+), 45 deletions(-) diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index a56c4e0ca6ed..aed9060e8ce1 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -99,7 +99,6 @@ public class FlinkCatalog extends AbstractCatalog { private final Namespace baseNamespace; private final SupportsNamespaces asNamespaceCatalog; private final Closeable closeable; - private final Map catalogProps; private final boolean cacheEnabled; public FlinkCatalog( @@ -107,12 +106,10 @@ public FlinkCatalog( String defaultDatabase, Namespace baseNamespace, CatalogLoader catalogLoader, - Map catalogProps, boolean cacheEnabled, long cacheExpirationIntervalMs) { super(catalogName, defaultDatabase); this.catalogLoader = catalogLoader; - this.catalogProps = catalogProps; this.baseNamespace = baseNamespace; this.cacheEnabled = cacheEnabled; @@ -339,13 +336,12 @@ public CatalogTable getTable(ObjectPath tablePath) Table table = loadIcebergTable(tablePath); // Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table. - // Inorder to create such table in non iceberg catalog, we need to send across catalog - // properties also. // As Flink API accepts only Map for props, here we are serializing catalog - // props as json string to distinguish between catalog and table properties in createTable. + // name, database, table as json string to distinguish between catalog info + // and table properties in createTable. String srcCatalogProps = FlinkCreateTableOptions.toJson( - getName(), tablePath.getDatabaseName(), tablePath.getObjectName(), catalogProps); + getName(), tablePath.getDatabaseName(), tablePath.getObjectName()); Map tableProps = table.properties(); if (tableProps.containsKey(FlinkCreateTableOptions.CONNECTOR_PROPS_KEY) diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java index 33cbc92ddeec..c1889dc6bd03 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java @@ -170,7 +170,6 @@ protected Catalog createCatalog( defaultDatabase, baseNamespace, catalogLoader, - properties, cacheEnabled, cacheExpirationIntervalMs); } diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java index 0612260bfe7d..067b42bba954 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java @@ -18,7 +18,6 @@ */ package org.apache.iceberg.flink; -import java.util.Map; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.iceberg.util.JsonUtil; @@ -27,14 +26,11 @@ class FlinkCreateTableOptions { private final String catalogName; private final String catalogDb; private final String catalogTable; - private final Map catalogProps; - private FlinkCreateTableOptions( - String catalogName, String catalogDb, String catalogTable, Map props) { + private FlinkCreateTableOptions(String catalogName, String catalogDb, String catalogTable) { this.catalogName = catalogName; this.catalogDb = catalogDb; this.catalogTable = catalogTable; - this.catalogProps = props; } public static final ConfigOption CATALOG_NAME = @@ -61,12 +57,6 @@ private FlinkCreateTableOptions( .noDefaultValue() .withDescription("Table name managed in the underlying iceberg catalog and database."); - public static final ConfigOption> CATALOG_PROPS = - ConfigOptions.key("catalog-props") - .mapType() - .noDefaultValue() - .withDescription("Properties for the underlying catalog for iceberg table."); - public static final ConfigOption USE_DYNAMIC_ICEBERG_SINK = ConfigOptions.key("use-dynamic-iceberg-sink") .booleanType() @@ -89,15 +79,13 @@ private FlinkCreateTableOptions( public static final String CONNECTOR_PROPS_KEY = "connector"; public static final String LOCATION_KEY = "location"; - static String toJson( - String catalogName, String catalogDb, String catalogTable, Map catalogProps) { + static String toJson(String catalogName, String catalogDb, String catalogTable) { return JsonUtil.generate( gen -> { gen.writeStartObject(); gen.writeStringField(CATALOG_NAME.key(), catalogName); gen.writeStringField(CATALOG_DATABASE.key(), catalogDb); gen.writeStringField(CATALOG_TABLE.key(), catalogTable); - JsonUtil.writeStringMap(CATALOG_PROPS.key(), catalogProps, gen); gen.writeEndObject(); }, false); @@ -110,9 +98,8 @@ static FlinkCreateTableOptions fromJson(String createTableOptions) { String catalogName = JsonUtil.getString(CATALOG_NAME.key(), node); String catalogDb = JsonUtil.getString(CATALOG_DATABASE.key(), node); String catalogTable = JsonUtil.getString(CATALOG_TABLE.key(), node); - Map catalogProps = JsonUtil.getStringMap(CATALOG_PROPS.key(), node); - return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable, catalogProps); + return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable); }); } @@ -127,8 +114,4 @@ String catalogDb() { String catalogTable() { return catalogTable; } - - Map catalogProps() { - return catalogProps; - } } diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java index b2e2e33b9291..6306ee7a0a1c 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java @@ -237,13 +237,13 @@ private static TableLoader createTableLoader( } /** - * Merges source catalog properties with connector properties. Iceberg Catalog properties are - * serialized as json in FlinkCatalog#getTable to be able to isolate catalog props from iceberg - * table props, Here, we flatten and merge them back to use to create catalog. + * Merges source catalog properties (catalog name, database, table) with connector properties. + * Source catalog name, database, table are serialized as json in FlinkCatalog#getTable to be able + * to isolate them from iceberg table props, Here, we flatten and merge them back. * * @param tableProps the existing table properties - * @return a map of merged properties, with source catalog properties taking precedence when keys - * conflict + * @return a map of merged properties, defaulting to source catalog name, database and table + * unless overridden. */ private static Map mergeSrcCatalogProps(Map tableProps) { String srcCatalogProps = tableProps.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY); @@ -257,7 +257,6 @@ private static Map mergeSrcCatalogProps(Map tabl FlinkCreateTableOptions.CATALOG_DATABASE.key(), createTableOptions.catalogDb()); mergedProps.put( FlinkCreateTableOptions.CATALOG_TABLE.key(), createTableOptions.catalogTable()); - mergedProps.putAll(createTableOptions.catalogProps()); tableProps.forEach( (k, v) -> { diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java index 062ff68d5d85..e45abe25f919 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java @@ -89,6 +89,7 @@ public void before() { config.put(CatalogProperties.URI, getURI(hiveConf)); } config.put(CatalogProperties.WAREHOUSE_LOCATION, String.format("file://%s", warehouseRoot())); + config.put("extra-catalog-prop", "extra-value"); this.flinkDatabase = catalogName + "." + DATABASE; this.icebergNamespace = diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index 020663a7de02..aa5638809e8d 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -34,7 +34,6 @@ import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.catalog.CatalogTable; -import org.apache.flink.table.catalog.CommonCatalogOptions; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.iceberg.BaseTable; @@ -229,19 +228,15 @@ public void testCreateTableLikeInFlinkCatalog() throws TableNotExistException { .column("id", DataTypes.BIGINT()) .build()); - // `type` option is filtered out by Flink - // https://github.com/apache/flink/blob/edc3d68736de73665440f4313ddcfd9142d8d42b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/FactoryUtil.java#L378 - Map filteredOptions = Maps.newHashMap(config); - filteredOptions.remove(CommonCatalogOptions.CATALOG_TYPE.key()); - - String srcCatalogProps = - FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl", filteredOptions); + String srcCatalogProps = FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl"); Map options = catalogTable.getOptions(); assertThat(options) .containsEntry( FlinkCreateTableOptions.CONNECTOR_PROPS_KEY, FlinkDynamicTableFactory.FACTORY_IDENTIFIER) .containsEntry(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY, srcCatalogProps); + assertThat(options.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY)) + .doesNotContain("extra-catalog-prop", "extra-value"); } @TestTemplate diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java index 0cdaf8371cbd..6cfc18868af0 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java @@ -179,7 +179,10 @@ public void testReadFlinkDynamicTable() throws Exception { List expected = generateExpectedRecords(false); SqlHelpers.sql( getTableEnv(), - "create table `default_catalog`.`default_database`.flink_table LIKE iceberg_catalog.`default`.%s", + "create table `default_catalog`.`default_database`.flink_table " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); // Read from table in flink catalog @@ -199,8 +202,11 @@ public void testWatermarkInvalidConfig() { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); assertThatThrownBy(() -> SqlHelpers.sql(getStreamingTableEnv(), "SELECT * FROM %s", flinkTable)) @@ -218,8 +224,11 @@ public void testWatermarkValidConfig() throws Exception { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) WITH ('watermark-column'='t1') LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s', 'watermark-column'='t1') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); TestHelpers.assertRecordsWithOrder( From 3c604fb6073d1c11095f0a6708c8dca5731430aa Mon Sep 17 00:00:00 2001 From: Thomas Thornton Date: Thu, 25 Jun 2026 01:24:34 -0700 Subject: [PATCH 41/73] Kafka Connect: Fix avro schema conversion for UUID (#16828) * Kafka Connect: Fix avro schema conversion for UUID Signed-off-by: Thomas Thornton * Kafka Connect: Fix schema conversion for FIXED type UUID Signed-off-by: Thomas Thornton * kafka connect: map UUID logical type on string to UUIDType Signed-off-by: Thomas Thornton * kafka connect: restore missing @Test decorator to testEmptyListAndMapConvert Signed-off-by: Thomas Thornton --------- Signed-off-by: Thomas Thornton --- .../iceberg/avro/TestSchemaConversions.java | 6 ++++++ .../apache/iceberg/connect/data/SchemaUtils.java | 5 +++++ .../connect/data/TestRecordConverter.java | 16 ++++++++++++++++ .../iceberg/connect/data/TestSchemaUtils.java | 9 +++++++++ 4 files changed, 36 insertions(+) diff --git a/core/src/test/java/org/apache/iceberg/avro/TestSchemaConversions.java b/core/src/test/java/org/apache/iceberg/avro/TestSchemaConversions.java index 0f822d3e4b64..f934e0101b73 100644 --- a/core/src/test/java/org/apache/iceberg/avro/TestSchemaConversions.java +++ b/core/src/test/java/org/apache/iceberg/avro/TestSchemaConversions.java @@ -94,6 +94,12 @@ public void testPrimitiveTypes() { } } + @Test + public void testAvroToIcebergUUIDTypeOnString() { + Schema uuidStringSchema = LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)); + assertThat(AvroSchemaUtil.convert(uuidStringSchema)).isEqualTo(Types.UUIDType.get()); + } + @Test public void testAvroToIcebergTimestampTypeWithoutAdjustToUTC() { // Not included in the primitives test because there is not a way to round trip the diff --git a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java index b0dd56b45d67..89d7878172cb 100644 --- a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java +++ b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java @@ -55,6 +55,7 @@ import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.types.Types.TimeType; import org.apache.iceberg.types.Types.TimestampType; +import org.apache.iceberg.types.Types.UUIDType; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; import org.apache.kafka.connect.data.Date; @@ -286,6 +287,10 @@ Type toIcebergType(Schema valueSchema) { .collect(Collectors.toList()); return StructType.of(structFields); case STRING: + if ("uuid".equals(valueSchema.name())) { + return UUIDType.get(); + } + return StringType.get(); default: return StringType.get(); } diff --git a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java index 1030e5839ba1..50a1ae6ad1c4 100644 --- a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java +++ b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java @@ -233,6 +233,22 @@ public void testMapConvert() { assertRecordValues(record); } + @Test + public void testUUIDStringConversion() { + Table table = mock(Table.class); + when(table.schema()) + .thenReturn(new org.apache.iceberg.Schema(NestedField.required(1, "uuid", UUIDType.get()))); + + Schema connectSchema = + SchemaBuilder.struct().field("uuid", SchemaBuilder.string().name("uuid").build()).build(); + Struct data = new Struct(connectSchema).put("uuid", UUID_VAL.toString()); + + RecordConverter converter = new RecordConverter(table, config); + Record record = converter.convert(data); + + assertThat(record.getField("uuid")).isEqualTo(UUID_VAL); + } + @Test public void testEmptyListAndMapConvert() { Table table = mock(Table.class); diff --git a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSchemaUtils.java b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSchemaUtils.java index bde2452128b9..9443ed467696 100644 --- a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSchemaUtils.java +++ b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSchemaUtils.java @@ -63,6 +63,7 @@ import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.types.Types.TimeType; import org.apache.iceberg.types.Types.TimestampType; +import org.apache.iceberg.types.Types.UUIDType; import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Decimal; import org.apache.kafka.connect.data.Schema; @@ -331,4 +332,12 @@ public void testInferIcebergTypeEmpty() { assertThat(SchemaUtils.inferIcebergType(ImmutableMap.of("nested", ImmutableMap.of()), config)) .isNull(); } + + @Test + public void testToIcebergTypeUUIDLogicalTypeOnString() { + IcebergSinkConfig config = mock(IcebergSinkConfig.class); + + Schema uuidSchema = SchemaBuilder.string().name("uuid").build(); + assertThat(SchemaUtils.toIcebergType(uuidSchema, config)).isInstanceOf(UUIDType.class); + } } From 33bd1019a3544ad005979c64b73ee0a8da2d50e4 Mon Sep 17 00:00:00 2001 From: Robin Moffatt Date: Thu, 25 Jun 2026 12:27:38 +0100 Subject: [PATCH 42/73] Flink: Improve error message for unsupported table kinds in createTable (#16079) --- .../apache/iceberg/flink/FlinkCatalog.java | 7 ++++- .../iceberg/flink/TestFlinkCatalogTable.java | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index aed9060e8ce1..1d1505a28c05 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -423,7 +423,12 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig + "create table without 'connector'='iceberg' related properties in an iceberg table."); } - Preconditions.checkArgument(table instanceof ResolvedCatalogTable, "table should be resolved"); + Preconditions.checkArgument( + table instanceof ResolvedCatalogTable, + "Expected a ResolvedCatalogTable but got: %s. " + + "Iceberg Flink catalog only supports resolved catalog tables " + + "(Materialized tables and other table kinds are not supported).", + table == null ? "null" : table.getClass().getName()); createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists); } diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index aa5638809e8d..30daf6d55ef2 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -33,7 +33,9 @@ import org.apache.flink.table.api.Schema.UnresolvedPrimaryKey; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.CatalogMaterializedTable; import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.iceberg.BaseTable; @@ -745,6 +747,31 @@ private void validateTableFiles(Table tbl, DataFile... expectedFiles) { assertThat(actualFilePaths).as("Files should match").isEqualTo(expectedFilePaths); } + @TestTemplate + public void testCreateMaterializedTableIsUnsupported() { + CatalogMaterializedTable materializedTable = + CatalogMaterializedTable.newBuilder() + .schema( + org.apache.flink.table.api.Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .build()) + .definitionQuery("SELECT id FROM tl") + .freshness(IntervalFreshness.ofMinute("5")) + .logicalRefreshMode(CatalogMaterializedTable.LogicalRefreshMode.AUTOMATIC) + .refreshMode(CatalogMaterializedTable.RefreshMode.CONTINUOUS) + .refreshStatus(CatalogMaterializedTable.RefreshStatus.INITIALIZING) + .build(); + + assertThatThrownBy( + () -> + getTableEnv() + .getCatalog(catalogName) + .get() + .createTable(new ObjectPath(DATABASE, "mt_table"), materializedTable, false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Materialized tables and other table kinds are not supported"); + } + private Table table(String name) { return validationCatalog.loadTable(TableIdentifier.of(icebergNamespace, name)); } From d1f789e517cd93589968325e60fab86306982cce Mon Sep 17 00:00:00 2001 From: Yujiang Zhong <42907416+zhongyujiang@users.noreply.github.com> Date: Thu, 25 Jun 2026 20:09:01 +0800 Subject: [PATCH 43/73] Flink: Backport fix file offset mismatch in DataIterator.seek() when files are skipped (#16950) Backports #16929 --- .../iceberg/flink/source/DataIterator.java | 4 +- .../flink/source/reader/ReaderUtil.java | 16 +- ...stArrayPoolDataIteratorBatcherRowData.java | 170 ++++++++++++++++++ .../iceberg/flink/source/DataIterator.java | 4 +- .../flink/source/reader/ReaderUtil.java | 16 +- ...stArrayPoolDataIteratorBatcherRowData.java | 170 ++++++++++++++++++ 6 files changed, 372 insertions(+), 8 deletions(-) diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java index 3beda960cec8..2cb7667a489a 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java @@ -86,6 +86,7 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask); for (long i = 0L; i < startingFileOffset; ++i) { tasks.next(); + fileOffset += 1; } updateCurrentIterator(); @@ -103,9 +104,6 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask)); } } - - fileOffset = startingFileOffset; - recordOffset = startingRecordOffset; } @Override diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java index 3b094ba02298..5a6787001170 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java @@ -57,6 +57,21 @@ private ReaderUtil() {} public static FileScanTask createFileTask( List records, File file, FileFormat fileFormat, Schema schema) throws IOException { + return createFileTask( + records, + file, + fileFormat, + schema, + ResidualEvaluator.unpartitioned(Expressions.alwaysTrue())); + } + + public static FileScanTask createFileTask( + List records, + File file, + FileFormat fileFormat, + Schema schema, + ResidualEvaluator residuals) + throws IOException { DataWriter writer = new GenericFileWriterFactory.Builder() .dataSchema(schema) @@ -69,7 +84,6 @@ public static FileScanTask createFileTask( DataFile dataFile = writer.toDataFile(); - ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.alwaysTrue()); return new BaseFileScanTask( dataFile, null, diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java index 4f6d182bb3c1..994227f52bcd 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -32,13 +33,18 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.flink.FlinkConfigOptions; import org.apache.iceberg.flink.TestFixtures; import org.apache.iceberg.flink.TestHelpers; import org.apache.iceberg.flink.source.DataIterator; import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -354,4 +360,168 @@ public void testMultipleFilesWithSeekPosition() throws Exception { assertThat(recordBatchIterator).isExhausted(); } + + @Test + public void testDataIteratorWithResidualFilter() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "file-1"); + record0.setField("id", 0L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "file-2"); + record1.setField("id", 1L); + record1.setField("dt", "-"); + + List fileRecords0 = ImmutableList.of(record0); + List fileRecords1 = ImmutableList.of(record1); + + validateFileOffsetWithResidualFilter(fileRecords0, fileRecords1, Expressions.alwaysTrue()); + + validateFileOffsetWithResidualFilter( + fileRecords0, fileRecords1, Expressions.greaterThan("id", 0)); + } + + private void validateFileOffsetWithResidualFilter( + List fileRecords0, List fileRecords1, Expression residualFilter) + throws IOException { + ResidualEvaluator residualEvaluator = ResidualEvaluator.unpartitioned(residualFilter); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + fileRecords0, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + fileRecords1, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + while (dataIterator.hasNext()) { + assertThat(dataIterator.fileOffset()).isEqualTo(dataIterator.next().getLong(1)); + } + } + + @Test + public void testInitializationWithHeadFilesSkipped() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "a"); + record0.setField("id", 1L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "a"); + record1.setField("id", 10L); + record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + assertThat(dataIterator.fileOffset()) + .as("File offset should be 1 because file 0 should be skipped") + .isEqualTo(1); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, record1, dataIterator.next()); + } + + @Test + void testSeekResumesCorrectlyAfterHeadFilesSkipped() throws IOException { + GenericRecord filteredRecord = GenericRecord.create(TestFixtures.SCHEMA); + filteredRecord.setField("data", "a"); + filteredRecord.setField("id", 1L); + filteredRecord.setField("dt", "-"); + + GenericRecord f1Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f1Record0.setField("data", "b"); + f1Record0.setField("id", 10L); + f1Record0.setField("dt", "-"); + + GenericRecord f2Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record0.setField("data", "d"); + f2Record0.setField("id", 20L); + f2Record0.setField("dt", "-"); + + GenericRecord f2Record1 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record1.setField("data", "e"); + f2Record1.setField("id", 21L); + f2Record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(filteredRecord), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(f1Record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask2 = + ReaderUtil.createFileTask( + ImmutableList.of(f2Record0, f2Record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = + new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1, fileTask2)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + dataIterator.seek(0, 0); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, f1Record0, dataIterator.next()); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record0, dataIterator.next()); + + int checkpointFileOffset = dataIterator.fileOffset(); + long checkpointRecordOffset = dataIterator.recordOffset(); + + DataIterator restoredIterator = ReaderUtil.createDataIterator(combinedTask); + restoredIterator.seek(checkpointFileOffset, checkpointRecordOffset); + + assertThat(restoredIterator.hasNext()).isTrue(); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record1, restoredIterator.next()); + assertThat(restoredIterator.hasNext()).isFalse(); + } } diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java index 3beda960cec8..2cb7667a489a 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java @@ -86,6 +86,7 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask); for (long i = 0L; i < startingFileOffset; ++i) { tasks.next(); + fileOffset += 1; } updateCurrentIterator(); @@ -103,9 +104,6 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask)); } } - - fileOffset = startingFileOffset; - recordOffset = startingRecordOffset; } @Override diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java index 3b094ba02298..5a6787001170 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java @@ -57,6 +57,21 @@ private ReaderUtil() {} public static FileScanTask createFileTask( List records, File file, FileFormat fileFormat, Schema schema) throws IOException { + return createFileTask( + records, + file, + fileFormat, + schema, + ResidualEvaluator.unpartitioned(Expressions.alwaysTrue())); + } + + public static FileScanTask createFileTask( + List records, + File file, + FileFormat fileFormat, + Schema schema, + ResidualEvaluator residuals) + throws IOException { DataWriter writer = new GenericFileWriterFactory.Builder() .dataSchema(schema) @@ -69,7 +84,6 @@ public static FileScanTask createFileTask( DataFile dataFile = writer.toDataFile(); - ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.alwaysTrue()); return new BaseFileScanTask( dataFile, null, diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java index 4f6d182bb3c1..994227f52bcd 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -32,13 +33,18 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.flink.FlinkConfigOptions; import org.apache.iceberg.flink.TestFixtures; import org.apache.iceberg.flink.TestHelpers; import org.apache.iceberg.flink.source.DataIterator; import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -354,4 +360,168 @@ public void testMultipleFilesWithSeekPosition() throws Exception { assertThat(recordBatchIterator).isExhausted(); } + + @Test + public void testDataIteratorWithResidualFilter() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "file-1"); + record0.setField("id", 0L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "file-2"); + record1.setField("id", 1L); + record1.setField("dt", "-"); + + List fileRecords0 = ImmutableList.of(record0); + List fileRecords1 = ImmutableList.of(record1); + + validateFileOffsetWithResidualFilter(fileRecords0, fileRecords1, Expressions.alwaysTrue()); + + validateFileOffsetWithResidualFilter( + fileRecords0, fileRecords1, Expressions.greaterThan("id", 0)); + } + + private void validateFileOffsetWithResidualFilter( + List fileRecords0, List fileRecords1, Expression residualFilter) + throws IOException { + ResidualEvaluator residualEvaluator = ResidualEvaluator.unpartitioned(residualFilter); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + fileRecords0, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + fileRecords1, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + while (dataIterator.hasNext()) { + assertThat(dataIterator.fileOffset()).isEqualTo(dataIterator.next().getLong(1)); + } + } + + @Test + public void testInitializationWithHeadFilesSkipped() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "a"); + record0.setField("id", 1L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "a"); + record1.setField("id", 10L); + record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + assertThat(dataIterator.fileOffset()) + .as("File offset should be 1 because file 0 should be skipped") + .isEqualTo(1); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, record1, dataIterator.next()); + } + + @Test + void testSeekResumesCorrectlyAfterHeadFilesSkipped() throws IOException { + GenericRecord filteredRecord = GenericRecord.create(TestFixtures.SCHEMA); + filteredRecord.setField("data", "a"); + filteredRecord.setField("id", 1L); + filteredRecord.setField("dt", "-"); + + GenericRecord f1Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f1Record0.setField("data", "b"); + f1Record0.setField("id", 10L); + f1Record0.setField("dt", "-"); + + GenericRecord f2Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record0.setField("data", "d"); + f2Record0.setField("id", 20L); + f2Record0.setField("dt", "-"); + + GenericRecord f2Record1 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record1.setField("data", "e"); + f2Record1.setField("id", 21L); + f2Record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(filteredRecord), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(f1Record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask2 = + ReaderUtil.createFileTask( + ImmutableList.of(f2Record0, f2Record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = + new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1, fileTask2)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + dataIterator.seek(0, 0); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, f1Record0, dataIterator.next()); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record0, dataIterator.next()); + + int checkpointFileOffset = dataIterator.fileOffset(); + long checkpointRecordOffset = dataIterator.recordOffset(); + + DataIterator restoredIterator = ReaderUtil.createDataIterator(combinedTask); + restoredIterator.seek(checkpointFileOffset, checkpointRecordOffset); + + assertThat(restoredIterator.hasNext()).isTrue(); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record1, restoredIterator.next()); + assertThat(restoredIterator.hasNext()).isFalse(); + } } From b99389ba82cd573e17683818e00b6b052a5ef345 Mon Sep 17 00:00:00 2001 From: gaborkaszab Date: Thu, 25 Jun 2026 15:34:52 +0200 Subject: [PATCH 44/73] Core: Set scan planning mode when initializing client (#15903) --- .../iceberg/rest/RESTSessionCatalog.java | 41 ++++++---- .../iceberg/rest/TestRESTScanPlanning.java | 81 +++++++++++++++++++ 2 files changed, 106 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index dbba1becade1..1d8e9da6d43c 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -72,6 +72,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.rest.RESTCatalogProperties.ScanPlanningMode; import org.apache.iceberg.rest.RESTCatalogProperties.SnapshotMode; import org.apache.iceberg.rest.RESTTableCache.TableWithETag; import org.apache.iceberg.rest.auth.AuthManager; @@ -174,6 +175,7 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog private Set endpoints; private Supplier> mutationHeaders = Map::of; private String namespaceSeparator = null; + private ScanPlanningMode clientScanPlanningMode = null; private RESTTableCache tableCache; @@ -291,6 +293,10 @@ public void initialize(String name, Map unresolved) { RESTCatalogProperties.NAMESPACE_SEPARATOR, RESTCatalogProperties.NAMESPACE_SEPARATOR_DEFAULT); + String scanPlanningModeConfig = mergedProps.get(RESTCatalogProperties.SCAN_PLANNING_MODE); + this.clientScanPlanningMode = + scanPlanningModeConfig == null ? null : ScanPlanningMode.fromString(scanPlanningModeConfig); + this.tableCache = createTableCache(mergedProps); this.closeables.addCloseable(this.tableCache); @@ -603,31 +609,34 @@ private RESTTable restTableForScanPlanning( TableIdentifier finalIdentifier, RESTClient restClient, Map tableConf) { - // Get client-side and server-side scan planning modes - String planningModeClientConfig = properties().get(RESTCatalogProperties.SCAN_PLANNING_MODE); String planningModeServerConfig = tableConf.get(RESTCatalogProperties.SCAN_PLANNING_MODE); - - // Warn if client and server configs conflict; server config takes precedence - if (planningModeClientConfig != null - && planningModeServerConfig != null - && !planningModeClientConfig.equalsIgnoreCase(planningModeServerConfig)) { + ScanPlanningMode serverScanPlanningMode = + planningModeServerConfig == null + ? null + : ScanPlanningMode.fromString(planningModeServerConfig); + + // Warn if client and server configs conflict + if (clientScanPlanningMode != null + && serverScanPlanningMode != null + && clientScanPlanningMode != serverScanPlanningMode) { LOG.warn( "Scan planning mode mismatch for table {}: client config={}, server config={}. " + "Server config will take precedence.", finalIdentifier, - planningModeClientConfig, + clientScanPlanningMode.modeName(), planningModeServerConfig); } - // Determine effective mode: prefer server config if present, otherwise use client config - String effectiveModeConfig = - planningModeServerConfig != null ? planningModeServerConfig : planningModeClientConfig; - RESTCatalogProperties.ScanPlanningMode effectiveMode = - effectiveModeConfig != null - ? RESTCatalogProperties.ScanPlanningMode.fromString(effectiveModeConfig) - : RESTCatalogProperties.SCAN_PLANNING_MODE_DEFAULT; + // Determine effective mode: prefer server config if present, otherwise use client config, + // fall back to default if both are null + ScanPlanningMode effectiveMode = RESTCatalogProperties.SCAN_PLANNING_MODE_DEFAULT; + if (serverScanPlanningMode != null) { + effectiveMode = serverScanPlanningMode; + } else if (clientScanPlanningMode != null) { + effectiveMode = clientScanPlanningMode; + } - if (effectiveMode == RESTCatalogProperties.ScanPlanningMode.SERVER) { + if (effectiveMode == ScanPlanningMode.SERVER) { Preconditions.checkState( endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN), "Server requires server-side scan planning for table %s but does not support endpoint %s", diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java index 9b42d445f585..117bf7fac24b 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java @@ -75,6 +75,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -1576,4 +1577,84 @@ public void clientRequestsClientAndServerReturnsNothing() { assertThat(table).isNotInstanceOf(RESTTable.class); assertThat(table).isInstanceOf(BaseTable.class); } + + @Test + public void defaultPlanningModeWhenNoneSpecified() { + CatalogWithAdapter catalogWithAdapter = catalogWithModes(null, null); + catalogWithAdapter.catalog.createNamespace(NS); + + Table table = + catalogWithAdapter + .catalog + .buildTable(TableIdentifier.of(NS, "default_mode_test"), SCHEMA) + .create(); + + assertThat(table).isNotInstanceOf(RESTTable.class).isInstanceOf(BaseTable.class); + } + + @Test + public void invalidPlanningModeConfiguredForClient() { + assertThatThrownBy( + () -> + catalogWithModes( + "invalid_mode", RESTCatalogProperties.ScanPlanningMode.CLIENT.modeName())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid scan planning mode: invalid_mode"); + } + + @Test + public void invalidPlanningModeConfiguredForServer() { + CatalogWithAdapter catalogWithAdapter = + catalogWithModes(RESTCatalogProperties.ScanPlanningMode.CLIENT.modeName(), "invalid_mode"); + catalogWithAdapter.catalog.createNamespace(NS); + + assertThatThrownBy( + () -> + catalogWithAdapter + .catalog + .buildTable(TableIdentifier.of(NS, "invalid_server_mode_test"), SCHEMA) + .create()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid scan planning mode: invalid_mode"); + } + + @ParameterizedTest + @ValueSource( + strings = {"client", "CLIENT", "Client", "cLiEnT", "server", "SERVER", "Server", "sErVeR"}) + public void planningModeWithDifferentCasesOnClient(String planningMode) { + CatalogWithAdapter catalogWithAdapter = catalogWithModes(planningMode, null); + catalogWithAdapter.catalog.createNamespace(NS); + + Table table = + catalogWithAdapter + .catalog + .buildTable(TableIdentifier.of(NS, "client_case_test_" + planningMode), SCHEMA) + .create(); + + verifyTableTypeForPlanningMode(planningMode, table); + } + + @ParameterizedTest + @ValueSource( + strings = {"client", "CLIENT", "Client", "cLiEnT", "server", "SERVER", "Server", "sErVeR"}) + public void planningModeWithDifferentCasesOnServer(String serverMode) { + CatalogWithAdapter catalogWithAdapter = catalogWithModes(null, serverMode); + catalogWithAdapter.catalog.createNamespace(NS); + + Table table = + catalogWithAdapter + .catalog + .buildTable(TableIdentifier.of(NS, "server_case_test_" + serverMode), SCHEMA) + .create(); + + verifyTableTypeForPlanningMode(serverMode, table); + } + + private void verifyTableTypeForPlanningMode(String planingMode, Table table) { + if (planingMode.equalsIgnoreCase("client")) { + assertThat(table).isNotInstanceOf(RESTTable.class).isInstanceOf(BaseTable.class); + } else { + assertThat(table).isInstanceOf(RESTTable.class); + } + } } From 6c5f9f242750fe2cade46e5dbd1a90d011741ea6 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Thu, 25 Jun 2026 22:43:41 +0200 Subject: [PATCH 45/73] Flink: Add equality delete conversion API and integration tests (#16948) * Flink: Add equality delete conversion API and integration tests Flink's upsert mode writes equality deletes, which are cheap to produce but force every reader to join the data against the delete files (merge-on-read). ConvertEqualityDeletes is a maintenance task that rewrites them into deletion vectors on a target branch, so reads apply deletes by position. The writer keeps appending equality deletes to a staging branch while the conversion runs in the background. Example: ``` TableMaintenance.forTable(env, tableLoader) .add( ConvertEqualityDeletes.builder() .stagingBranch("staging") .targetBranch("main") .equalityFieldColumns(ImmutableList.of("id")) .scheduleOnEqDeleteFileCount(1) .append(); ``` The table must be V3 for deletion vectors, the equality field columns must match the writer's, and every partition column must be an equality field, because the converter keys rows on equality values alone and would otherwise resolve a delete against the wrong partition. The conversion task has checks in place to ensure this. * fixup! Add runtime check to ensure spec fields are included in equality fields * fixup! Add requested test for an unknown equality field column * fixup! Increase test timeout to prevent flakiness in CI --- .../api/ConvertEqualityDeletes.java | 286 ++++ .../operator/EqualityConvertCommitter.java | 3 +- .../operator/EqualityConvertPlanner.java | 20 + .../api/TestConvertEqualityDeletes.java | 1333 +++++++++++++++++ .../api/TestConvertEqualityDeletesE2E.java | 170 +++ .../maintenance/api/TestMaintenanceE2E.java | 33 + .../operator/TestEqualityConvertPlanner.java | 40 + 7 files changed, 1884 insertions(+), 1 deletion(-) create mode 100644 flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java create mode 100644 flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java create mode 100644 flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java new file mode 100644 index 000000000000..f51df3338d02 --- /dev/null +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.flink.annotation.Experimental; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.BroadcastStream; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.flink.maintenance.operator.DVPosition; +import org.apache.iceberg.flink.maintenance.operator.DVWriteResult; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertDVWriter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPKIndex; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlan; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlanner; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertReader; +import org.apache.iceberg.flink.maintenance.operator.IndexCommand; +import org.apache.iceberg.flink.maintenance.operator.ReadCommand; +import org.apache.iceberg.flink.maintenance.operator.SerializedEqualityValues; +import org.apache.iceberg.flink.maintenance.operator.TaskResultAggregator; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types.NestedField; + +/** + * Creates the equality delete to DV conversion data stream. Runs a single iteration of the + * conversion for every {@link Trigger} event. + * + *

The pipeline reads equality delete files from a staging branch, converts them to deletion + * vectors (DVs) using a primary key index stored in Flink state, and commits the data files and DVs + * to the target branch. + * + *

The conversion is split into parallel stages: + * + *

    + *
  1. Planner (p=1): scans staging branch, emits file-level ReadCommands with phase timestamps + *
  2. Reader (p=N): reads files, emits row-level IndexCommands + *
  3. PKIndex (p=N): maintains PK index shards, resolves equality deletes to DV positions + *
  4. DVWriter (p=N, keyed by data file path): buffers positions per file, writes Puffin DVs + * inline + *
  5. Committer (p=1): commits data files and DVs to the target branch + *
+ * + *

Mutual exclusion with concurrent maintenance tasks (e.g. compaction) is enforced by the Flink + * maintenance framework lock. + */ +@Experimental +public class ConvertEqualityDeletes { + static final String PLANNER_TASK_NAME = "EqConvert Planner"; + static final String READER_TASK_NAME = "EqConvert Reader"; + static final String PK_INDEX_TASK_NAME = "EqConvert PKIndex"; + static final String DV_WRITER_TASK_NAME = "EqConvert DVWriter"; + static final String UPSTREAM_ABORT_TASK_NAME = "EqConvert UpstreamAbort"; + static final String COMMIT_TASK_NAME = "EqConvert Commit"; + static final String AGGREGATOR_TASK_NAME = "EqConvert Aggregator"; + + private ConvertEqualityDeletes() {} + + public static Builder builder() { + return new Builder(); + } + + public static class Builder extends MaintenanceTaskBuilder { + private String stagingBranch; + private String targetBranch = SnapshotRef.MAIN_BRANCH; + private List equalityFieldColumns = Collections.emptyList(); + + @Override + String maintenanceTaskName() { + return "ConvertEqualityDeletes"; + } + + /** Sets the staging branch name that holds the equality delete files and data files. */ + public Builder stagingBranch(String newStagingBranch) { + this.stagingBranch = newStagingBranch; + return this; + } + + /** + * Sets the target branch where converted data files and DVs are committed. Defaults to the main + * branch. + */ + public Builder targetBranch(String newTargetBranch) { + this.targetBranch = newTargetBranch; + return this; + } + + /** + * Sets the equality field columns used by the worker index. Required. Must match the equality + * field columns the writer uses for staging eq-delete files. Mirrors {@link + * org.apache.iceberg.flink.sink.IcebergSink.Builder#equalityFieldColumns}. + * + *

The partition source columns of an equality delete's spec must be a subset of these + * columns. Writes via Flink's IcebergSink already ensure this. + */ + public Builder equalityFieldColumns(List columns) { + Preconditions.checkNotNull(columns, "equalityFieldColumns must not be null"); + Preconditions.checkArgument(!columns.isEmpty(), "equalityFieldColumns must not be empty"); + this.equalityFieldColumns = ImmutableList.copyOf(columns); + return this; + } + + @Override + DataStream append(DataStream trigger) { + Preconditions.checkNotNull(stagingBranch, "stagingBranch must be set"); + Preconditions.checkArgument( + !equalityFieldColumns.isEmpty(), "equalityFieldColumns must be set on the builder"); + Set eqFieldIds = resolveEqualityFieldIds(); + + // Planner (p=1): emits ReadCommands with phase timestamps and watermarks + SingleOutputStreamOperator planned = + setSlotSharingGroup( + trigger + .transform( + operatorName(PLANNER_TASK_NAME), + TypeInformation.of(ReadCommand.class), + new EqualityConvertPlanner( + tableName(), + taskName(), + tableLoader(), + stagingBranch, + targetBranch, + eqFieldIds)) + .uid(PLANNER_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Reader (p=N): reads files, emits IndexCommands + SingleOutputStreamOperator index = + setSlotSharingGroup( + planned + .rebalance() + .process( + new EqualityConvertReader( + tableLoader(), eqFieldIds, stagingBranch.equals(targetBranch))) + .name(operatorName(READER_TASK_NAME)) + .uid(READER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Broadcast from the planner to the PKIndex to clear the entire index + BroadcastStream clearIndexBroadcast = + planned + .getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM) + .broadcast(EqualityConvertPKIndex.CLEAR_BROADCAST_DESCRIPTOR); + + // PKIndex (p=N): keyed by full PK, phase-aware buffering. + SingleOutputStreamOperator dvPositions = + setSlotSharingGroup( + index + .keyBy(IndexCommand::key, TypeInformation.of(SerializedEqualityValues.class)) + .connect(clearIndexBroadcast) + .process(new EqualityConvertPKIndex(stagingBranch.equals(targetBranch))) + .name(operatorName(PK_INDEX_TASK_NAME)) + .uid(PK_INDEX_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Reader-side abort signals bypass the PKIndex and feed the DVWriter directly, so a reader + // failure can short-circuit the cycle without waiting on a keyed shuffle. This is not a full + // short-circuit: the abort is keyed by data file path (empty for ABORT), so only one resolver + // subtask observes it; the others still write their buffered DVs, which the committer then + // drops. + DataStream readerAborts = + index.getSideOutput(EqualityConvertReader.READER_ABORT_STREAM); + DataStream dvPositionsWithAborts = dvPositions.union(readerAborts); + + // Metadata side output from planner + DataStream metadata = + planned.getSideOutput(EqualityConvertPlanner.METADATA_STREAM); + + // DVWriter (p=N, keyed by data file path): groups positions per file, writes Puffin DV + // files inline, emits a DVWriteResult per cycle. Plan metadata broadcast so every subtask + // sees it. + SingleOutputStreamOperator resolved = + setSlotSharingGroup( + dvPositionsWithAborts + .keyBy(DVPosition::dataFilePath) + .connect(metadata.broadcast()) + .transform( + operatorName(DV_WRITER_TASK_NAME), + TypeInformation.of(DVWriteResult.class), + new EqualityConvertDVWriter( + tableName(), taskName(), tableLoader(), targetBranch)) + .uid(DV_WRITER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Upstream errors become abort signals so a partial read never commits. The same error side + // outputs also feed the aggregator below to surface the exception in TaskResult; the two + // consumers serve different purposes and must both exist. + DataStream upstreamAborts = + setSlotSharingGroup( + index + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .map(e -> DVWriteResult.ABORT) + .returns(TypeInformation.of(DVWriteResult.class)) + .name(operatorName(UPSTREAM_ABORT_TASK_NAME)) + .uid(UPSTREAM_ABORT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Committer (p=1): commits data files + DVs to main. + SingleOutputStreamOperator committed = + setSlotSharingGroup( + resolved + .union(upstreamAborts) + .connect(metadata) + .transform( + operatorName(COMMIT_TASK_NAME), + TypeInformation.of(Trigger.class), + new EqualityConvertCommitter( + tableName(), taskName(), tableLoader(), stagingBranch, targetBranch)) + .uid(COMMIT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Aggregator (p=1): collects errors and emits TaskResult. + return setSlotSharingGroup( + committed + .connect( + planned + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(index.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(resolved.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(committed.getSideOutput(TaskResultAggregator.ERROR_STREAM))) + .transform( + operatorName(AGGREGATOR_TASK_NAME), + TypeInformation.of(TaskResult.class), + new TaskResultAggregator(tableName(), taskName(), index())) + .uid(AGGREGATOR_TASK_NAME + uidSuffix()) + .forceNonParallel()); + } + + private Set resolveEqualityFieldIds() { + if (!tableLoader().isOpen()) { + tableLoader().open(); + } + + Table table = tableLoader().loadTable(); + int formatVersion = TableUtil.formatVersion(table); + Preconditions.checkArgument( + formatVersion >= 3, + "ConvertEqualityDeletes requires table format version >= 3 (DVs), " + + "but table '%s' is version %s", + tableName(), + formatVersion); + + Schema schema = table.schema(); + List fieldIds = Lists.newArrayListWithCapacity(equalityFieldColumns.size()); + for (String column : equalityFieldColumns) { + NestedField field = schema.findField(column); + Preconditions.checkArgument( + field != null, + "Equality field column '%s' not found in table schema %s", + column, + schema); + fieldIds.add(field.fieldId()); + } + + return ImmutableSet.copyOf(fieldIds); + } + } +} diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java index c83b06a55abf..b3e8fb4d8938 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -71,7 +71,8 @@ public class EqualityConvertCommitter extends AbstractStreamOperator private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); - static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = "equality-convert-staging-snapshot"; + public static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = + "equality-convert-staging-snapshot"; private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java index c3c1785290cf..89e5510dd848 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -43,6 +43,7 @@ import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileContent; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; @@ -515,6 +516,7 @@ private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { deleteFile.location(), deleteFieldIds, eqFieldIds); + validateDeleteSpecPartitionColumns(stagingSnapshot, deleteFile); eqDeleteFiles.add(deleteFile); } else if (ContentFileUtil.isDV(deleteFile)) { stagingDVFiles.add(deleteFile); @@ -531,6 +533,24 @@ private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); } + private void validateDeleteSpecPartitionColumns(Snapshot stagingSnapshot, DeleteFile deleteFile) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + for (PartitionField field : spec.fields()) { + Preconditions.checkState( + eqFieldIds.contains(field.sourceId()), + "Staging snapshot %s on branch '%s' contains an equality delete file %s under spec %s, " + + "which partitions by field '%s' (source id %s) that is not an equality field %s. " + + "Partition columns must be a subset of the equality fields.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + spec.specId(), + field.name(), + field.sourceId(), + eqFieldIds); + } + } + /** Files added by one staging snapshot, classified for cycle emission. */ private record StagingInputs( List newDataFiles, diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java new file mode 100644 index 000000000000..1c2f230315be --- /dev/null +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java @@ -0,0 +1,1333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.COMMIT_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.PLANNER_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ERROR_COUNTER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.stream.StreamSupport; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.graph.StreamGraphGenerator; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.MetricsReporterFactoryForTests; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructLikeSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestConvertEqualityDeletes extends MaintenanceTaskTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void testRejectsFormatVersion2() { + createTableWithDelete(2); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("format version >= 3"); + } + + @Test + void testRejectsUnknownEqualityFieldColumns() { + createTableWithDelete(3); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("nonexistent")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Equality field column 'nonexistent' not found in table schema"); + } + + @Test + void testConvertEqualityDeletesToDVs() throws Exception { + Table table = createTableWithDelete(3); + + // Insert initial data to main + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + assertThat(dataFileCount(table)).isEqualTo(3); + + // Create staging branch from current main state + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Write a new data file (simulating insert of id=4) + DataFile newDataFile = writeDataFile(table, createRecord(4, "d")); + + // Write an equality delete for id=2 (simulating delete of row "b") + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + + // Commit both to the staging branch + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Verify staging branch has the eq delete + long stagingEqDeleteCount = + table.snapshot(STAGING_BRANCH).deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(stagingEqDeleteCount).isEqualTo(1); + + // Wire the ConvertEqualityDeletes maintenance task + appendConvertTask(); + + // Run the maintenance task + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Verify main branch now has 4 data files (3 original + 1 new) + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(4); + + // Verify main branch has exactly one DV (id=2 deleted from its single-row data file) + long mainDvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .count(); + assertThat(mainDvCount).isEqualTo(1); + + // Verify no equality deletes on main + long mainEqDeleteCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(mainEqDeleteCount).isEqualTo(0); + + // Verify data correctness: id=2 should be deleted, id=4 should be added + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(3, "c"), createRecord(4, "d"))); + } + + @Test + void testMetrics() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot: a new data file (id=3) plus an eq-delete (id=1). The conversion commits + // exactly one data file and one DV to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Only metrics named on TableMaintenanceMetrics flow through the test reporter. Among the + // converter operators only the planner and committer own an ERROR_COUNTER; the parallel reader, + // PK index, and DV writer report failures through ERROR_STREAM instead. The committer also + // counts the data files it adds. Operator-specific counters (reindexCount, addedDvNum, ...) are + // asserted by the operator unit tests. A -1 expected value means "present, value not checked". + MetricsReporterFactoryForTests.assertCounters( + new ImmutableMap.Builder, Long>() + .put(errorKey(PLANNER_TASK_NAME), 0L) + .put(errorKey(COMMIT_TASK_NAME), 0L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_NUM_METRIC), 1L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_SIZE_METRIC), -1L) + .build()); + } + + private static List errorKey(String taskName) { + return metricKey(taskName, ERROR_COUNTER); + } + + private static List metricKey(String taskName, String metric) { + return ImmutableList.of(taskName + "[0]", DUMMY_TABLE_NAME, DUMMY_TASK_NAME, "0", metric); + } + + @Test + void testNoOpWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + // Create staging branch with no new files + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + // Should complete successfully with no changes + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(1); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testMultipleEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + insert(table, 4, "d"); + insert(table, 5, "e"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Delete id=1 and id=4 via equality deletes on staging + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + DeleteFile eqDelete2 = writeEqualityDelete(table, 4, "d"); + + table + .newRowDelta() + .addDeletes(eqDelete1) + .addDeletes(eqDelete2) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"), createRecord(5, "e"))); + } + + @Test + void testDuplicateKeyAcrossDataFiles() throws Exception { + Table table = createTableWithDelete(3); + + // Two data files with the same key (id=1, data="a") + insert(table, 1, "a"); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Eq delete for id=1 should produce DVs for both data files containing id=1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only id=2 should remain + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } + + @Test + void testMultiSnapshotStagingWithPerSnapshotScoping() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: delete id=1 from main + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: re-insert id=1 (new data file) + DataFile newDataFile = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (delete id=1) + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: processes S2 (re-insert id=1) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // id=1 should still exist: the delete from S1 removed the original, + // but S2 re-inserted it. Per-snapshot scoping ensures S1's delete + // doesn't affect S2's data. + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertThenDeleteAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: insert id=1 (data-only, no eq deletes) + DataFile insertS1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(insertS1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: eq delete the row written in S1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (insert id=1), commits data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: processes S2 (eq delete id=1), must find id=1 on main + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertUpdateDeleteInsertUpdateChain() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: insert K=1, V=A + DataFile s1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(s1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: update K=1 to V=B (eq delete + insert in same commit) + DataFile s2 = writeDataFile(table, createRecord(1, "b")); + DeleteFile e2 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(s2).addDeletes(e2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S3: delete K=1 + DeleteFile e3 = writeEqualityDelete(table, 1, "b"); + table.newRowDelta().addDeletes(e3).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S4: insert K=1, V=C + DataFile s4 = writeDataFile(table, createRecord(1, "c")); + table.newAppend().appendFile(s4).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S5: update K=1 to V=D (eq delete + insert in same commit) + DataFile s5 = writeDataFile(table, createRecord(1, "d")); + DeleteFile e5 = writeEqualityDelete(table, 1, "c"); + table.newRowDelta().addRows(s5).addDeletes(e5).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time = System.currentTimeMillis(); + + // Cycle 1: S1 inserts K=1, V=A + long time1 = time; + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: S2 updates K=1 from V=A to V=B (eq delete + insert) + long time2 = time + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "b"))); + + // Cycle 3: S3 deletes K=1 + long time3 = time + 2; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.exceptions()).isEmpty(); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 4: S4 inserts K=1, V=C + long time4 = time + 3; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.exceptions()).isEmpty(); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "c"))); + + // Cycle 5: S5 updates K=1 from V=C to V=D (eq delete + insert) + long time5 = time + 4; + infra.source().sendRecord(Trigger.create(time5, 0), time5); + TaskResult result5 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result5.exceptions()).isEmpty(); + assertThat(result5.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "d"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testParallelInsertOfToBeDeletedKeySurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Main holds the original (1, "a"); the staging eq-delete below removes this copy. + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot updates id=1 in place: re-insert (1, "a") plus an eq-delete (1, "a") in + // the same commit. The re-insert shares the equality key with the delete and carries the + // delete's sequence, so it must survive: the delete only removes the lower-sequence main copy. + // At parallelism > 1 the staging-data ADD can reach the index before the eq-delete resolves. + // Event-time phase ordering is what keeps the re-insert from being accidentally deleted. + DataFile reinsert = writeDataFile(table, createRecord(1, "a")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(reinsert).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(STAGING_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testDVMergeAcrossConversionCycles() throws Exception { + Table table = createTableWithDelete(3); + + // Single data file with 3 rows so DV merge applies to the same file + insert( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"), createRecord(3, "c"))); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1 setup: eq delete for id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: convert eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2 setup: eq delete for id=2 (committed while job is running) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 2: convert eq delete for id=2, should merge DV with existing + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + + // Verify: main has DVs, no equality deletes + assertNoEqualityDeletesOnMain(table, 0); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConversionCorrectAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + + // Three separate data files + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Compact file2 and file3 on main into one file (leave file1 + its DV untouched) + Set allDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + allDataFiles.add(df.copy()); + } + } + } + + // Find file1 (contains id=1) by checking which file has a DV against it + Set dvReferencedFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + dvReferencedFiles.add(df.referencedDataFile()); + } + } + } + } + + Set filesToCompact = Sets.newHashSet(); + for (DataFile df : allDataFiles) { + if (!dvReferencedFiles.contains(df.location())) { + filesToCompact.add(df); + } + } + + assertThat(filesToCompact).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : filesToCompact) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: delete id=2 (should target the compacted file) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConvertEqualityDeletesPartitionedTable() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Insert data into two partitions + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 2, "b"); + insertPartitioned(table, 3, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Partition-scoped equality delete for id=1 in partition data="a" + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // id=1 deleted from partition "a", id=2 in partition "b" and id=3 in partition "a" remain + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "a"))); + } + + @Test + void testEqualityDeleteIsScopedToItsPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Same PK (id=1) exists in two partitions. An eq delete in one partition must not delete + // rows in the other. + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 1, "b"); + insertPartitioned(table, 2, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only (1, "a") is deleted; (1, "b") remains because the eq delete was scoped to partition + // "a", and (2, "a") remains because its equality field values don't match. + assertRecords(table, ImmutableList.of(createRecord(1, "b"), createRecord(2, "a"))); + } + + @Test + void testStagingPositionDeleteMergedIntoConversionDV() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: write a data file with two rows (id=1 at pos 0, id=2 at pos 1). + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: eq delete matches row 0 (will produce a conversion DV at pos 0) + a position + // delete DV referencing the same data file at pos 1. Both DVs target the same data + // file and must be merged into a single DV (V3 invariant). + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table + .newRowDelta() + .addDeletes(eqDelete) + .addDeletes(stagingDV) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: commits S1's data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: converts S2's eq delete to DV, merges with staging DV + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Both rows from S1 must be masked: pos 0 by the conversion DV, pos 1 by the staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant): the resolver must have folded the + // staging DV's positions into the conversion DV. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingDataFilesOnlyNoEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging has only a new data file, no equality deletes + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(2); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"))); + } + + @Test + void testReindexAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"))); + + // External commit: insert id=3 directly to main (not via staging) + insert(table, 3, "c"); + + // Cycle 2: delete id=2 (should reindex because of external commit) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testReindexEvictsGhostKeyAfterExternalDataFileRemoval() throws Exception { + // CoW removal case: an external commit removes a data file, leaving a stale + // PK in the worker's index. A later staging eq-delete for that PK must NOT produce a DV + // referencing the removed file. The external commit advances main, so the next cycle reindexes + // and the worker clears the ghost key before resolving the delete. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.refresh(); + DataFile file1 = table.currentSnapshot().addedDataFiles(table.io()).iterator().next().copy(); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=2. Bootstraps the worker index from main (id=1 -> file1, id=2 -> file2). + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // External CoW-style removal: drop file1 (id=1) from main without re-adding the row. The + // worker's index still holds the ghost id=1 -> file1 until the next reindex clears it. + table.newDelete().deleteFile(file1).commit(); + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 2: stage an eq-delete for the removed key id=1. Without ghost eviction the worker + // would emit a DV position against the now-absent file1. + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + // No deletion vector may reference the removed file1. + table.refresh(); + assertThat(dvReferencedDataFiles(table)).doesNotContain(file1.location()); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + private static Set dvReferencedDataFiles(Table table) { + Set referenced = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + referenced.add(df.referencedDataFile()); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return referenced; + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + Schema eqDeleteSchema = table.schema(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + eqDeleteSchema); + } + + private DeleteFile writePartitionedEqualityDelete(Table table, Integer id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + partition, + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private static long dataFileCount(Table table) { + table.refresh(); + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile ignored : reader) { + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return count; + } + + @Test + void testStagingEqualsTargetBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Write eq delete directly to main (no separate staging branch) + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DataFile newData = writeDataFile(table, createRecord(3, "c")); + table.newRowDelta().addRows(newData).addDeletes(eqDelete).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: process the eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + long dataFilesAfterCycle1 = dataFileCount(table); + // Expect 3: two from insert() + one from the writer's rowDelta.addRows(newData). + // When stagingBranch == targetBranch, the committer must NOT re-add newData via + // rowDelta.addRows(...) — that would duplicate (count=4). + assertThat(dataFilesAfterCycle1).isEqualTo(3); + + // Cycle 2: no-op (converter's own commit must be skipped) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle1); + + // New eq delete for id=2 committed directly to main between cycles + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + DataFile newData2 = writeDataFile(table, createRecord(4, "d")); + table.newRowDelta().addRows(newData2).addDeletes(eqDelete2).commit(); + table.refresh(); + + // Cycle 3: process the new eq delete + long time3 = time2 + 1; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + long dataFilesAfterCycle3 = dataFileCount(table); + + // Cycle 4: no-op again + long time4 = time3 + 1; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle3); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchColdStartCatchUp() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + // Writer commits three eq-deletes to main BEFORE the converter starts. + // Cold start must pick up the unconverted history, not just the head snapshot. + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 2, "b")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 3, "c")).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One unconverted snapshot per cycle, oldest first. After three cycles every eq-delete + // commit has its own committer commit carrying the marker. + for (int cycle = 1; cycle <= 3; cycle++) { + long ts = System.currentTimeMillis() + cycle; + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.success()).isTrue(); + } + + table.refresh(); + long convertedCount = + StreamSupport.stream(table.snapshots().spliterator(), false) + .filter(s -> s.summary().containsKey(COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .count(); + assertThat(convertedCount).isEqualTo(3); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchReinsertAfterDeleteSurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Shared branch: insert id=1, eq-delete id=1, then re-insert id=1. The re-insert has a higher + // sequence than the delete and must survive the conversion (sequence-aware resolution). + insert(table, 1, "a"); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newAppend().appendFile(writeDataFile(table, createRecord(1, "a"))).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One cycle converts the eq-delete: the original row is deleted, the newer re-insert is + // not (its sequence is at or above the delete's). + long ts = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.exceptions()).isEmpty(); + assertThat(result.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchMergesStagingDvIntoSingleDv() throws Exception { + Table table = createTableWithDelete(3); + + // One data file with two rows: id=1 at pos 0, id=2 at pos 1, committed to main. + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).commit(); + table.refresh(); + + // Same-branch commit: an eq-delete for id=1 (resolves to a conversion DV at pos 0) plus a + // writer DV at pos 1 on the same data file. The resolver folds the staging DV into the + // conversion DV; on a shared branch the committer must remove the superseded staging DV. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table.newRowDelta().addDeletes(eqDelete).addDeletes(stagingDV).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Both rows masked: pos 0 by the conversion DV, pos 1 by the merged-in staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant). Without removing the rewritten staging DV on a + // shared branch, the data file would carry two DVs. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } + + @Test + void testReaderErrorSkipsCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + long mainSnapshotBeforeStaging = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging data file + eq delete file, both referenced by the staging commit. + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + File eqDeleteLocalFile = new File(eqDelete.location().replace("file:", "")); + + // Delete the eq delete file; the committer must abort rather than committing data without its + // DV. + assertThat(eqDeleteLocalFile.delete()).isTrue(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result1.success()).isFalse(); + assertThat(result1.exceptions()).isNotEmpty(); + + table.refresh(); + // Main must not have advanced (no commit happened). + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotBeforeStaging); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Restore the eq delete file content by rewriting an identical delete, and retry: + // the planner must re-process the same staging snapshot (cursor didn't advance on failure). + DeleteFile recreated = + FileHelpers.writeDeleteFile( + table, + Files.localOutput(eqDeleteLocalFile), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(1, "a")), + table.schema()); + assertThat(recreated.location()).isEqualTo(eqDelete.location()); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Staging data file committed with DV for id=1: should see id=2 only. + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } finally { + closeJobClient(jobClient); + } + } + + private void appendConvertTask() { + appendConvertTask(STAGING_BRANCH); + } + + private void appendConvertTask(String stagingBranch) { + ConvertEqualityDeletes.builder() + .stagingBranch(stagingBranch) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1) + .sinkTo(infra.sink()); + } + + private static void assertRecords(Table table, List expected) throws IOException { + table.refresh(); + Types.StructType type = SimpleDataUtil.SCHEMA.asStruct(); + + StructLikeSet expectedSet = StructLikeSet.create(type); + expectedSet.addAll(expected); + + try (CloseableIterable iterable = + IcebergGenerics.read(table) + .useSnapshot(table.currentSnapshot().snapshotId()) + .project(SimpleDataUtil.SCHEMA) + .build()) { + StructLikeSet actualSet = StructLikeSet.create(type); + for (Record record : iterable) { + actualSet.add(record); + } + + assertThat(actualSet).isEqualTo(expectedSet); + } + } + + private static void assertNoEqualityDeletesOnMain(Table table, long expectedEqDeleteCount) { + long mainEqDeleteCount = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile f : reader) { + if (f.content() == FileContent.EQUALITY_DELETES) { + mainEqDeleteCount++; + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + assertThat(mainEqDeleteCount).isEqualTo(expectedEqDeleteCount); + } +} diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java new file mode 100644 index 000000000000..f6d1fd9464e2 --- /dev/null +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.util.ContentFileUtil; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * End-to-end test for {@link ConvertEqualityDeletes} wired through the {@link TableMaintenance} + * framework. Verifies that the converter actually runs and commits a DV when the framework triggers + * it, exercising the full operator graph including the framework's monitor source, trigger manager, + * and lock remover. + */ +class TestConvertEqualityDeletesE2E extends OperatorTestBase { + private static final String STAGING_BRANCH = "staging"; + + @TempDir private Path tempDir; + private StreamExecutionEnvironment env; + + @BeforeEach + public void beforeEach() { + this.env = StreamExecutionEnvironment.getExecutionEnvironment(); + } + + @ParameterizedTest + @ValueSource(strings = {STAGING_BRANCH, SnapshotRef.MAIN_BRANCH}) + void testConvertEqualityDeletesE2E(String stagingBranch) throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // When staging is a separate branch, fork it from main first. + if (!stagingBranch.equals(SnapshotRef.MAIN_BRANCH)) { + table.manageSnapshots().createBranch(stagingBranch).commit(); + table.refresh(); + } + + // Commit a new data file + eq delete to staging. This pre-job snapshot exercises both the + // "new data file" and "eq delete" paths in one cycle. + DataFile newData = writeDataFile(table, 3, "c"); + DeleteFile firstDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newData).addDeletes(firstDelete).toBranch(stagingBranch).commit(); + table.refresh(); + assertThat(dvCountOnMain(table)).isZero(); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("ConvertEqualityDeletesE2EUID-" + stagingBranch) + .rateLimit(Duration.ofMillis(50)) + .lockCheckDelay(Duration.ofMillis(50)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnInterval(Duration.ofMillis(100)) + .stagingBranch(stagingBranch) + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Cycle 1: row 1 deleted by the converted DV; row 3 added on staging and committed to main. + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(1)); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: commit a second staging snapshot while the job is still running. The framework's + // next interval-trigger should pick it up and produce a second DV against the data file + // holding id=2. + table.refresh(); + DeleteFile secondDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(secondDelete).toBranch(stagingBranch).commit(); + + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(2)); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + private DataFile writeDataFile(Table table, Integer id, String data) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(SimpleDataUtil.createRecord(id, data))); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(SimpleDataUtil.createRecord(id, data)), + table.schema()); + } + + private static long dvCountOnMain(Table table) throws IOException { + table.refresh(); + if (table.currentSnapshot() == null) { + return 0; + } + + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (ContentFileUtil.isDV(file)) { + count++; + } + } + } + } + + return count; + } +} diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java index fe8457167a1f..f786f1cdb29d 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java @@ -24,8 +24,11 @@ import java.time.Duration; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -116,4 +119,34 @@ void testE2eUseCoordinator() throws Exception { closeJobClient(jobClient); } } + + @Test + void testE2eConvertEqualityDeletes() throws Exception { + // Converter requires V3 (DV support); replace the V2 table created in @BeforeEach. + dropTable(); + createTable(3, FileFormat.PARQUET); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("E2eConvertEqualityDeletesUID") + .rateLimit(Duration.ofMinutes(10)) + .lockCheckDelay(Duration.ofSeconds(10)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnEqDeleteFileCount(1) + .stagingBranch("staging") + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Just make sure that we are able to instantiate the flow + assertThat(jobClient).isNotNull(); + } finally { + closeJobClient(jobClient); + } + } } diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java index 82860a0de881..1ee5c5d0d5fd 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -110,6 +110,33 @@ void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { } } + @Test + void failsWhenStagingEqDeleteSpecPartitionsByNonEqualityColumn() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeIdOnlyPartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + harness.open(); + sendTrigger(harness); + + List> errOutput = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + + assertThat(errOutput).hasSize(1); + assertThat(errOutput.get(0).getValue()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Partition columns must be a subset of the equality fields."); + } + } + @Test void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { // When stagingBranch == targetBranch, the writer commits new data files directly to main. @@ -1063,6 +1090,19 @@ private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOExcep idOnly); } + private DeleteFile writeIdOnlyPartitionedEqualityDelete(Table table, int id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, Files.localOutput(file), partition, Lists.newArrayList(record), idOnly); + } + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) throws IOException { File file = File.createTempFile("junit", null, tempDir.toFile()); From cfebc8ed16fcdee7a28ad92635adbb744b0258bc Mon Sep 17 00:00:00 2001 From: Robin Moffatt Date: Thu, 25 Jun 2026 21:46:20 +0100 Subject: [PATCH 46/73] Flink: Backport improved createTable error message to 1.20 and 2.0 (#16959) Backports #16079 to the Flink 1.20 and 2.0 modules. FlinkCatalog#createTable now reports the actual table class and explains that the Iceberg Flink catalog only supports resolved catalog tables, instead of the terse "table should be resolved", and adds a test asserting the precondition rejects a CatalogMaterializedTable. --- .../apache/iceberg/flink/FlinkCatalog.java | 7 ++++- .../iceberg/flink/TestFlinkCatalogTable.java | 27 +++++++++++++++++++ .../apache/iceberg/flink/FlinkCatalog.java | 7 ++++- .../iceberg/flink/TestFlinkCatalogTable.java | 27 +++++++++++++++++++ 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index a56c4e0ca6ed..d07c8b90e494 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -427,7 +427,12 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig + "create table without 'connector'='iceberg' related properties in an iceberg table."); } - Preconditions.checkArgument(table instanceof ResolvedCatalogTable, "table should be resolved"); + Preconditions.checkArgument( + table instanceof ResolvedCatalogTable, + "Expected a ResolvedCatalogTable but got: %s. " + + "Iceberg Flink catalog only supports resolved catalog tables " + + "(Materialized tables and other table kinds are not supported).", + table == null ? "null" : table.getClass().getName()); createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists); } diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index b8c357bbb64a..c3bf9818ccdd 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -33,7 +33,9 @@ import org.apache.flink.table.api.Schema.UnresolvedPrimaryKey; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.CatalogMaterializedTable; import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.iceberg.BaseTable; @@ -743,6 +745,31 @@ private void validateTableFiles(Table tbl, DataFile... expectedFiles) { assertThat(actualFilePaths).as("Files should match").isEqualTo(expectedFilePaths); } + @TestTemplate + public void testCreateMaterializedTableIsUnsupported() { + CatalogMaterializedTable materializedTable = + CatalogMaterializedTable.newBuilder() + .schema( + org.apache.flink.table.api.Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .build()) + .definitionQuery("SELECT id FROM tl") + .freshness(IntervalFreshness.ofMinute("5")) + .logicalRefreshMode(CatalogMaterializedTable.LogicalRefreshMode.AUTOMATIC) + .refreshMode(CatalogMaterializedTable.RefreshMode.CONTINUOUS) + .refreshStatus(CatalogMaterializedTable.RefreshStatus.INITIALIZING) + .build(); + + assertThatThrownBy( + () -> + getTableEnv() + .getCatalog(catalogName) + .get() + .createTable(new ObjectPath(DATABASE, "mt_table"), materializedTable, false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Materialized tables and other table kinds are not supported"); + } + private Table table(String name) { return validationCatalog.loadTable(TableIdentifier.of(icebergNamespace, name)); } diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index a56c4e0ca6ed..d07c8b90e494 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -427,7 +427,12 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig + "create table without 'connector'='iceberg' related properties in an iceberg table."); } - Preconditions.checkArgument(table instanceof ResolvedCatalogTable, "table should be resolved"); + Preconditions.checkArgument( + table instanceof ResolvedCatalogTable, + "Expected a ResolvedCatalogTable but got: %s. " + + "Iceberg Flink catalog only supports resolved catalog tables " + + "(Materialized tables and other table kinds are not supported).", + table == null ? "null" : table.getClass().getName()); createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists); } diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index 020663a7de02..4dd1d6e40be3 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -33,8 +33,10 @@ import org.apache.flink.table.api.Schema.UnresolvedPrimaryKey; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.CatalogMaterializedTable; import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.catalog.CommonCatalogOptions; +import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.iceberg.BaseTable; @@ -750,6 +752,31 @@ private void validateTableFiles(Table tbl, DataFile... expectedFiles) { assertThat(actualFilePaths).as("Files should match").isEqualTo(expectedFilePaths); } + @TestTemplate + public void testCreateMaterializedTableIsUnsupported() { + CatalogMaterializedTable materializedTable = + CatalogMaterializedTable.newBuilder() + .schema( + org.apache.flink.table.api.Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .build()) + .definitionQuery("SELECT id FROM tl") + .freshness(IntervalFreshness.ofMinute("5")) + .logicalRefreshMode(CatalogMaterializedTable.LogicalRefreshMode.AUTOMATIC) + .refreshMode(CatalogMaterializedTable.RefreshMode.CONTINUOUS) + .refreshStatus(CatalogMaterializedTable.RefreshStatus.INITIALIZING) + .build(); + + assertThatThrownBy( + () -> + getTableEnv() + .getCatalog(catalogName) + .get() + .createTable(new ObjectPath(DATABASE, "mt_table"), materializedTable, false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Materialized tables and other table kinds are not supported"); + } + private Table table(String name) { return validationCatalog.loadTable(TableIdentifier.of(icebergNamespace, name)); } From 66150ef442be414c9580fc1c73962f1f9719d441 Mon Sep 17 00:00:00 2001 From: Swapna Marru Date: Thu, 25 Jun 2026 22:36:29 -0700 Subject: [PATCH 47/73] Flink: Backport Pass only white-listed Catalog properties to 1.20,2.0 (#16966) Backports #16728 --- .../apache/iceberg/flink/FlinkCatalog.java | 10 +++----- .../iceberg/flink/FlinkCatalogFactory.java | 1 - .../flink/FlinkCreateTableOptions.java | 23 +++---------------- .../flink/FlinkDynamicTableFactory.java | 11 ++++----- .../apache/iceberg/flink/CatalogTestBase.java | 1 + .../iceberg/flink/TestFlinkCatalogTable.java | 4 +++- .../flink/source/TestIcebergSourceSql.java | 15 +++++++++--- .../apache/iceberg/flink/FlinkCatalog.java | 10 +++----- .../iceberg/flink/FlinkCatalogFactory.java | 1 - .../flink/FlinkCreateTableOptions.java | 23 +++---------------- .../flink/FlinkDynamicTableFactory.java | 11 ++++----- .../apache/iceberg/flink/CatalogTestBase.java | 1 + .../iceberg/flink/TestFlinkCatalogTable.java | 11 +++------ .../flink/source/TestIcebergSourceSql.java | 15 +++++++++--- 14 files changed, 54 insertions(+), 83 deletions(-) diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index d07c8b90e494..1d1505a28c05 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -99,7 +99,6 @@ public class FlinkCatalog extends AbstractCatalog { private final Namespace baseNamespace; private final SupportsNamespaces asNamespaceCatalog; private final Closeable closeable; - private final Map catalogProps; private final boolean cacheEnabled; public FlinkCatalog( @@ -107,12 +106,10 @@ public FlinkCatalog( String defaultDatabase, Namespace baseNamespace, CatalogLoader catalogLoader, - Map catalogProps, boolean cacheEnabled, long cacheExpirationIntervalMs) { super(catalogName, defaultDatabase); this.catalogLoader = catalogLoader; - this.catalogProps = catalogProps; this.baseNamespace = baseNamespace; this.cacheEnabled = cacheEnabled; @@ -339,13 +336,12 @@ public CatalogTable getTable(ObjectPath tablePath) Table table = loadIcebergTable(tablePath); // Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table. - // Inorder to create such table in non iceberg catalog, we need to send across catalog - // properties also. // As Flink API accepts only Map for props, here we are serializing catalog - // props as json string to distinguish between catalog and table properties in createTable. + // name, database, table as json string to distinguish between catalog info + // and table properties in createTable. String srcCatalogProps = FlinkCreateTableOptions.toJson( - getName(), tablePath.getDatabaseName(), tablePath.getObjectName(), catalogProps); + getName(), tablePath.getDatabaseName(), tablePath.getObjectName()); Map tableProps = table.properties(); if (tableProps.containsKey(FlinkCreateTableOptions.CONNECTOR_PROPS_KEY) diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java index dd065617bd88..fe4008a13ce5 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java @@ -168,7 +168,6 @@ protected Catalog createCatalog( defaultDatabase, baseNamespace, catalogLoader, - properties, cacheEnabled, cacheExpirationIntervalMs); } diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java index 0612260bfe7d..067b42bba954 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java @@ -18,7 +18,6 @@ */ package org.apache.iceberg.flink; -import java.util.Map; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.iceberg.util.JsonUtil; @@ -27,14 +26,11 @@ class FlinkCreateTableOptions { private final String catalogName; private final String catalogDb; private final String catalogTable; - private final Map catalogProps; - private FlinkCreateTableOptions( - String catalogName, String catalogDb, String catalogTable, Map props) { + private FlinkCreateTableOptions(String catalogName, String catalogDb, String catalogTable) { this.catalogName = catalogName; this.catalogDb = catalogDb; this.catalogTable = catalogTable; - this.catalogProps = props; } public static final ConfigOption CATALOG_NAME = @@ -61,12 +57,6 @@ private FlinkCreateTableOptions( .noDefaultValue() .withDescription("Table name managed in the underlying iceberg catalog and database."); - public static final ConfigOption> CATALOG_PROPS = - ConfigOptions.key("catalog-props") - .mapType() - .noDefaultValue() - .withDescription("Properties for the underlying catalog for iceberg table."); - public static final ConfigOption USE_DYNAMIC_ICEBERG_SINK = ConfigOptions.key("use-dynamic-iceberg-sink") .booleanType() @@ -89,15 +79,13 @@ private FlinkCreateTableOptions( public static final String CONNECTOR_PROPS_KEY = "connector"; public static final String LOCATION_KEY = "location"; - static String toJson( - String catalogName, String catalogDb, String catalogTable, Map catalogProps) { + static String toJson(String catalogName, String catalogDb, String catalogTable) { return JsonUtil.generate( gen -> { gen.writeStartObject(); gen.writeStringField(CATALOG_NAME.key(), catalogName); gen.writeStringField(CATALOG_DATABASE.key(), catalogDb); gen.writeStringField(CATALOG_TABLE.key(), catalogTable); - JsonUtil.writeStringMap(CATALOG_PROPS.key(), catalogProps, gen); gen.writeEndObject(); }, false); @@ -110,9 +98,8 @@ static FlinkCreateTableOptions fromJson(String createTableOptions) { String catalogName = JsonUtil.getString(CATALOG_NAME.key(), node); String catalogDb = JsonUtil.getString(CATALOG_DATABASE.key(), node); String catalogTable = JsonUtil.getString(CATALOG_TABLE.key(), node); - Map catalogProps = JsonUtil.getStringMap(CATALOG_PROPS.key(), node); - return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable, catalogProps); + return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable); }); } @@ -127,8 +114,4 @@ String catalogDb() { String catalogTable() { return catalogTable; } - - Map catalogProps() { - return catalogProps; - } } diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java index b2e2e33b9291..6306ee7a0a1c 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java @@ -237,13 +237,13 @@ private static TableLoader createTableLoader( } /** - * Merges source catalog properties with connector properties. Iceberg Catalog properties are - * serialized as json in FlinkCatalog#getTable to be able to isolate catalog props from iceberg - * table props, Here, we flatten and merge them back to use to create catalog. + * Merges source catalog properties (catalog name, database, table) with connector properties. + * Source catalog name, database, table are serialized as json in FlinkCatalog#getTable to be able + * to isolate them from iceberg table props, Here, we flatten and merge them back. * * @param tableProps the existing table properties - * @return a map of merged properties, with source catalog properties taking precedence when keys - * conflict + * @return a map of merged properties, defaulting to source catalog name, database and table + * unless overridden. */ private static Map mergeSrcCatalogProps(Map tableProps) { String srcCatalogProps = tableProps.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY); @@ -257,7 +257,6 @@ private static Map mergeSrcCatalogProps(Map tabl FlinkCreateTableOptions.CATALOG_DATABASE.key(), createTableOptions.catalogDb()); mergedProps.put( FlinkCreateTableOptions.CATALOG_TABLE.key(), createTableOptions.catalogTable()); - mergedProps.putAll(createTableOptions.catalogProps()); tableProps.forEach( (k, v) -> { diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java index 062ff68d5d85..e45abe25f919 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java @@ -89,6 +89,7 @@ public void before() { config.put(CatalogProperties.URI, getURI(hiveConf)); } config.put(CatalogProperties.WAREHOUSE_LOCATION, String.format("file://%s", warehouseRoot())); + config.put("extra-catalog-prop", "extra-value"); this.flinkDatabase = catalogName + "." + DATABASE; this.icebergNamespace = diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index c3bf9818ccdd..b4a8965e9bd4 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -230,13 +230,15 @@ public void testCreateTableLikeInFlinkCatalog() throws TableNotExistException { .column("id", DataTypes.BIGINT()) .build()); - String srcCatalogProps = FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl", config); + String srcCatalogProps = FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl"); Map options = catalogTable.getOptions(); assertThat(options) .containsEntry( FlinkCreateTableOptions.CONNECTOR_PROPS_KEY, FlinkDynamicTableFactory.FACTORY_IDENTIFIER) .containsEntry(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY, srcCatalogProps); + assertThat(options.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY)) + .doesNotContain("extra-catalog-prop", "extra-value"); } @TestTemplate diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java index 0cdaf8371cbd..6cfc18868af0 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java @@ -179,7 +179,10 @@ public void testReadFlinkDynamicTable() throws Exception { List expected = generateExpectedRecords(false); SqlHelpers.sql( getTableEnv(), - "create table `default_catalog`.`default_database`.flink_table LIKE iceberg_catalog.`default`.%s", + "create table `default_catalog`.`default_database`.flink_table " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); // Read from table in flink catalog @@ -199,8 +202,11 @@ public void testWatermarkInvalidConfig() { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); assertThatThrownBy(() -> SqlHelpers.sql(getStreamingTableEnv(), "SELECT * FROM %s", flinkTable)) @@ -218,8 +224,11 @@ public void testWatermarkValidConfig() throws Exception { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) WITH ('watermark-column'='t1') LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s', 'watermark-column'='t1') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); TestHelpers.assertRecordsWithOrder( diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index d07c8b90e494..1d1505a28c05 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -99,7 +99,6 @@ public class FlinkCatalog extends AbstractCatalog { private final Namespace baseNamespace; private final SupportsNamespaces asNamespaceCatalog; private final Closeable closeable; - private final Map catalogProps; private final boolean cacheEnabled; public FlinkCatalog( @@ -107,12 +106,10 @@ public FlinkCatalog( String defaultDatabase, Namespace baseNamespace, CatalogLoader catalogLoader, - Map catalogProps, boolean cacheEnabled, long cacheExpirationIntervalMs) { super(catalogName, defaultDatabase); this.catalogLoader = catalogLoader; - this.catalogProps = catalogProps; this.baseNamespace = baseNamespace; this.cacheEnabled = cacheEnabled; @@ -339,13 +336,12 @@ public CatalogTable getTable(ObjectPath tablePath) Table table = loadIcebergTable(tablePath); // Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table. - // Inorder to create such table in non iceberg catalog, we need to send across catalog - // properties also. // As Flink API accepts only Map for props, here we are serializing catalog - // props as json string to distinguish between catalog and table properties in createTable. + // name, database, table as json string to distinguish between catalog info + // and table properties in createTable. String srcCatalogProps = FlinkCreateTableOptions.toJson( - getName(), tablePath.getDatabaseName(), tablePath.getObjectName(), catalogProps); + getName(), tablePath.getDatabaseName(), tablePath.getObjectName()); Map tableProps = table.properties(); if (tableProps.containsKey(FlinkCreateTableOptions.CONNECTOR_PROPS_KEY) diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java index 33cbc92ddeec..c1889dc6bd03 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java @@ -170,7 +170,6 @@ protected Catalog createCatalog( defaultDatabase, baseNamespace, catalogLoader, - properties, cacheEnabled, cacheExpirationIntervalMs); } diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java index 0612260bfe7d..067b42bba954 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java @@ -18,7 +18,6 @@ */ package org.apache.iceberg.flink; -import java.util.Map; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.iceberg.util.JsonUtil; @@ -27,14 +26,11 @@ class FlinkCreateTableOptions { private final String catalogName; private final String catalogDb; private final String catalogTable; - private final Map catalogProps; - private FlinkCreateTableOptions( - String catalogName, String catalogDb, String catalogTable, Map props) { + private FlinkCreateTableOptions(String catalogName, String catalogDb, String catalogTable) { this.catalogName = catalogName; this.catalogDb = catalogDb; this.catalogTable = catalogTable; - this.catalogProps = props; } public static final ConfigOption CATALOG_NAME = @@ -61,12 +57,6 @@ private FlinkCreateTableOptions( .noDefaultValue() .withDescription("Table name managed in the underlying iceberg catalog and database."); - public static final ConfigOption> CATALOG_PROPS = - ConfigOptions.key("catalog-props") - .mapType() - .noDefaultValue() - .withDescription("Properties for the underlying catalog for iceberg table."); - public static final ConfigOption USE_DYNAMIC_ICEBERG_SINK = ConfigOptions.key("use-dynamic-iceberg-sink") .booleanType() @@ -89,15 +79,13 @@ private FlinkCreateTableOptions( public static final String CONNECTOR_PROPS_KEY = "connector"; public static final String LOCATION_KEY = "location"; - static String toJson( - String catalogName, String catalogDb, String catalogTable, Map catalogProps) { + static String toJson(String catalogName, String catalogDb, String catalogTable) { return JsonUtil.generate( gen -> { gen.writeStartObject(); gen.writeStringField(CATALOG_NAME.key(), catalogName); gen.writeStringField(CATALOG_DATABASE.key(), catalogDb); gen.writeStringField(CATALOG_TABLE.key(), catalogTable); - JsonUtil.writeStringMap(CATALOG_PROPS.key(), catalogProps, gen); gen.writeEndObject(); }, false); @@ -110,9 +98,8 @@ static FlinkCreateTableOptions fromJson(String createTableOptions) { String catalogName = JsonUtil.getString(CATALOG_NAME.key(), node); String catalogDb = JsonUtil.getString(CATALOG_DATABASE.key(), node); String catalogTable = JsonUtil.getString(CATALOG_TABLE.key(), node); - Map catalogProps = JsonUtil.getStringMap(CATALOG_PROPS.key(), node); - return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable, catalogProps); + return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable); }); } @@ -127,8 +114,4 @@ String catalogDb() { String catalogTable() { return catalogTable; } - - Map catalogProps() { - return catalogProps; - } } diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java index b2e2e33b9291..6306ee7a0a1c 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java @@ -237,13 +237,13 @@ private static TableLoader createTableLoader( } /** - * Merges source catalog properties with connector properties. Iceberg Catalog properties are - * serialized as json in FlinkCatalog#getTable to be able to isolate catalog props from iceberg - * table props, Here, we flatten and merge them back to use to create catalog. + * Merges source catalog properties (catalog name, database, table) with connector properties. + * Source catalog name, database, table are serialized as json in FlinkCatalog#getTable to be able + * to isolate them from iceberg table props, Here, we flatten and merge them back. * * @param tableProps the existing table properties - * @return a map of merged properties, with source catalog properties taking precedence when keys - * conflict + * @return a map of merged properties, defaulting to source catalog name, database and table + * unless overridden. */ private static Map mergeSrcCatalogProps(Map tableProps) { String srcCatalogProps = tableProps.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY); @@ -257,7 +257,6 @@ private static Map mergeSrcCatalogProps(Map tabl FlinkCreateTableOptions.CATALOG_DATABASE.key(), createTableOptions.catalogDb()); mergedProps.put( FlinkCreateTableOptions.CATALOG_TABLE.key(), createTableOptions.catalogTable()); - mergedProps.putAll(createTableOptions.catalogProps()); tableProps.forEach( (k, v) -> { diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java index 062ff68d5d85..e45abe25f919 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java @@ -89,6 +89,7 @@ public void before() { config.put(CatalogProperties.URI, getURI(hiveConf)); } config.put(CatalogProperties.WAREHOUSE_LOCATION, String.format("file://%s", warehouseRoot())); + config.put("extra-catalog-prop", "extra-value"); this.flinkDatabase = catalogName + "." + DATABASE; this.icebergNamespace = diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index 4dd1d6e40be3..30daf6d55ef2 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -35,7 +35,6 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.catalog.CatalogMaterializedTable; import org.apache.flink.table.catalog.CatalogTable; -import org.apache.flink.table.catalog.CommonCatalogOptions; import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.TableNotExistException; @@ -231,19 +230,15 @@ public void testCreateTableLikeInFlinkCatalog() throws TableNotExistException { .column("id", DataTypes.BIGINT()) .build()); - // `type` option is filtered out by Flink - // https://github.com/apache/flink/blob/edc3d68736de73665440f4313ddcfd9142d8d42b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/FactoryUtil.java#L378 - Map filteredOptions = Maps.newHashMap(config); - filteredOptions.remove(CommonCatalogOptions.CATALOG_TYPE.key()); - - String srcCatalogProps = - FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl", filteredOptions); + String srcCatalogProps = FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl"); Map options = catalogTable.getOptions(); assertThat(options) .containsEntry( FlinkCreateTableOptions.CONNECTOR_PROPS_KEY, FlinkDynamicTableFactory.FACTORY_IDENTIFIER) .containsEntry(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY, srcCatalogProps); + assertThat(options.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY)) + .doesNotContain("extra-catalog-prop", "extra-value"); } @TestTemplate diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java index 0cdaf8371cbd..6cfc18868af0 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java @@ -179,7 +179,10 @@ public void testReadFlinkDynamicTable() throws Exception { List expected = generateExpectedRecords(false); SqlHelpers.sql( getTableEnv(), - "create table `default_catalog`.`default_database`.flink_table LIKE iceberg_catalog.`default`.%s", + "create table `default_catalog`.`default_database`.flink_table " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); // Read from table in flink catalog @@ -199,8 +202,11 @@ public void testWatermarkInvalidConfig() { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); assertThatThrownBy(() -> SqlHelpers.sql(getStreamingTableEnv(), "SELECT * FROM %s", flinkTable)) @@ -218,8 +224,11 @@ public void testWatermarkValidConfig() throws Exception { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) WITH ('watermark-column'='t1') LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s', 'watermark-column'='t1') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); TestHelpers.assertRecordsWithOrder( From 7269fc1750b13d3f871b5e051fe670bb19fbe870 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Fri, 26 Jun 2026 07:58:46 +0200 Subject: [PATCH 48/73] Flink: Backport: Add equality delete conversion API and integration tests (#16969) Backports #16948 --- .../api/ConvertEqualityDeletes.java | 286 ++++ .../operator/EqualityConvertCommitter.java | 3 +- .../operator/EqualityConvertPlanner.java | 20 + .../api/TestConvertEqualityDeletes.java | 1333 +++++++++++++++++ .../api/TestConvertEqualityDeletesE2E.java | 170 +++ .../maintenance/api/TestMaintenanceE2E.java | 33 + .../operator/TestEqualityConvertPlanner.java | 40 + .../api/ConvertEqualityDeletes.java | 286 ++++ .../operator/EqualityConvertCommitter.java | 3 +- .../operator/EqualityConvertPlanner.java | 20 + .../api/TestConvertEqualityDeletes.java | 1333 +++++++++++++++++ .../api/TestConvertEqualityDeletesE2E.java | 170 +++ .../maintenance/api/TestMaintenanceE2E.java | 33 + .../operator/TestEqualityConvertPlanner.java | 40 + 14 files changed, 3768 insertions(+), 2 deletions(-) create mode 100644 flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java create mode 100644 flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java create mode 100644 flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java create mode 100644 flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java create mode 100644 flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java create mode 100644 flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java new file mode 100644 index 000000000000..f51df3338d02 --- /dev/null +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.flink.annotation.Experimental; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.BroadcastStream; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.flink.maintenance.operator.DVPosition; +import org.apache.iceberg.flink.maintenance.operator.DVWriteResult; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertDVWriter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPKIndex; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlan; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlanner; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertReader; +import org.apache.iceberg.flink.maintenance.operator.IndexCommand; +import org.apache.iceberg.flink.maintenance.operator.ReadCommand; +import org.apache.iceberg.flink.maintenance.operator.SerializedEqualityValues; +import org.apache.iceberg.flink.maintenance.operator.TaskResultAggregator; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types.NestedField; + +/** + * Creates the equality delete to DV conversion data stream. Runs a single iteration of the + * conversion for every {@link Trigger} event. + * + *

The pipeline reads equality delete files from a staging branch, converts them to deletion + * vectors (DVs) using a primary key index stored in Flink state, and commits the data files and DVs + * to the target branch. + * + *

The conversion is split into parallel stages: + * + *

    + *
  1. Planner (p=1): scans staging branch, emits file-level ReadCommands with phase timestamps + *
  2. Reader (p=N): reads files, emits row-level IndexCommands + *
  3. PKIndex (p=N): maintains PK index shards, resolves equality deletes to DV positions + *
  4. DVWriter (p=N, keyed by data file path): buffers positions per file, writes Puffin DVs + * inline + *
  5. Committer (p=1): commits data files and DVs to the target branch + *
+ * + *

Mutual exclusion with concurrent maintenance tasks (e.g. compaction) is enforced by the Flink + * maintenance framework lock. + */ +@Experimental +public class ConvertEqualityDeletes { + static final String PLANNER_TASK_NAME = "EqConvert Planner"; + static final String READER_TASK_NAME = "EqConvert Reader"; + static final String PK_INDEX_TASK_NAME = "EqConvert PKIndex"; + static final String DV_WRITER_TASK_NAME = "EqConvert DVWriter"; + static final String UPSTREAM_ABORT_TASK_NAME = "EqConvert UpstreamAbort"; + static final String COMMIT_TASK_NAME = "EqConvert Commit"; + static final String AGGREGATOR_TASK_NAME = "EqConvert Aggregator"; + + private ConvertEqualityDeletes() {} + + public static Builder builder() { + return new Builder(); + } + + public static class Builder extends MaintenanceTaskBuilder { + private String stagingBranch; + private String targetBranch = SnapshotRef.MAIN_BRANCH; + private List equalityFieldColumns = Collections.emptyList(); + + @Override + String maintenanceTaskName() { + return "ConvertEqualityDeletes"; + } + + /** Sets the staging branch name that holds the equality delete files and data files. */ + public Builder stagingBranch(String newStagingBranch) { + this.stagingBranch = newStagingBranch; + return this; + } + + /** + * Sets the target branch where converted data files and DVs are committed. Defaults to the main + * branch. + */ + public Builder targetBranch(String newTargetBranch) { + this.targetBranch = newTargetBranch; + return this; + } + + /** + * Sets the equality field columns used by the worker index. Required. Must match the equality + * field columns the writer uses for staging eq-delete files. Mirrors {@link + * org.apache.iceberg.flink.sink.IcebergSink.Builder#equalityFieldColumns}. + * + *

The partition source columns of an equality delete's spec must be a subset of these + * columns. Writes via Flink's IcebergSink already ensure this. + */ + public Builder equalityFieldColumns(List columns) { + Preconditions.checkNotNull(columns, "equalityFieldColumns must not be null"); + Preconditions.checkArgument(!columns.isEmpty(), "equalityFieldColumns must not be empty"); + this.equalityFieldColumns = ImmutableList.copyOf(columns); + return this; + } + + @Override + DataStream append(DataStream trigger) { + Preconditions.checkNotNull(stagingBranch, "stagingBranch must be set"); + Preconditions.checkArgument( + !equalityFieldColumns.isEmpty(), "equalityFieldColumns must be set on the builder"); + Set eqFieldIds = resolveEqualityFieldIds(); + + // Planner (p=1): emits ReadCommands with phase timestamps and watermarks + SingleOutputStreamOperator planned = + setSlotSharingGroup( + trigger + .transform( + operatorName(PLANNER_TASK_NAME), + TypeInformation.of(ReadCommand.class), + new EqualityConvertPlanner( + tableName(), + taskName(), + tableLoader(), + stagingBranch, + targetBranch, + eqFieldIds)) + .uid(PLANNER_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Reader (p=N): reads files, emits IndexCommands + SingleOutputStreamOperator index = + setSlotSharingGroup( + planned + .rebalance() + .process( + new EqualityConvertReader( + tableLoader(), eqFieldIds, stagingBranch.equals(targetBranch))) + .name(operatorName(READER_TASK_NAME)) + .uid(READER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Broadcast from the planner to the PKIndex to clear the entire index + BroadcastStream clearIndexBroadcast = + planned + .getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM) + .broadcast(EqualityConvertPKIndex.CLEAR_BROADCAST_DESCRIPTOR); + + // PKIndex (p=N): keyed by full PK, phase-aware buffering. + SingleOutputStreamOperator dvPositions = + setSlotSharingGroup( + index + .keyBy(IndexCommand::key, TypeInformation.of(SerializedEqualityValues.class)) + .connect(clearIndexBroadcast) + .process(new EqualityConvertPKIndex(stagingBranch.equals(targetBranch))) + .name(operatorName(PK_INDEX_TASK_NAME)) + .uid(PK_INDEX_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Reader-side abort signals bypass the PKIndex and feed the DVWriter directly, so a reader + // failure can short-circuit the cycle without waiting on a keyed shuffle. This is not a full + // short-circuit: the abort is keyed by data file path (empty for ABORT), so only one resolver + // subtask observes it; the others still write their buffered DVs, which the committer then + // drops. + DataStream readerAborts = + index.getSideOutput(EqualityConvertReader.READER_ABORT_STREAM); + DataStream dvPositionsWithAborts = dvPositions.union(readerAborts); + + // Metadata side output from planner + DataStream metadata = + planned.getSideOutput(EqualityConvertPlanner.METADATA_STREAM); + + // DVWriter (p=N, keyed by data file path): groups positions per file, writes Puffin DV + // files inline, emits a DVWriteResult per cycle. Plan metadata broadcast so every subtask + // sees it. + SingleOutputStreamOperator resolved = + setSlotSharingGroup( + dvPositionsWithAborts + .keyBy(DVPosition::dataFilePath) + .connect(metadata.broadcast()) + .transform( + operatorName(DV_WRITER_TASK_NAME), + TypeInformation.of(DVWriteResult.class), + new EqualityConvertDVWriter( + tableName(), taskName(), tableLoader(), targetBranch)) + .uid(DV_WRITER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Upstream errors become abort signals so a partial read never commits. The same error side + // outputs also feed the aggregator below to surface the exception in TaskResult; the two + // consumers serve different purposes and must both exist. + DataStream upstreamAborts = + setSlotSharingGroup( + index + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .map(e -> DVWriteResult.ABORT) + .returns(TypeInformation.of(DVWriteResult.class)) + .name(operatorName(UPSTREAM_ABORT_TASK_NAME)) + .uid(UPSTREAM_ABORT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Committer (p=1): commits data files + DVs to main. + SingleOutputStreamOperator committed = + setSlotSharingGroup( + resolved + .union(upstreamAborts) + .connect(metadata) + .transform( + operatorName(COMMIT_TASK_NAME), + TypeInformation.of(Trigger.class), + new EqualityConvertCommitter( + tableName(), taskName(), tableLoader(), stagingBranch, targetBranch)) + .uid(COMMIT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Aggregator (p=1): collects errors and emits TaskResult. + return setSlotSharingGroup( + committed + .connect( + planned + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(index.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(resolved.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(committed.getSideOutput(TaskResultAggregator.ERROR_STREAM))) + .transform( + operatorName(AGGREGATOR_TASK_NAME), + TypeInformation.of(TaskResult.class), + new TaskResultAggregator(tableName(), taskName(), index())) + .uid(AGGREGATOR_TASK_NAME + uidSuffix()) + .forceNonParallel()); + } + + private Set resolveEqualityFieldIds() { + if (!tableLoader().isOpen()) { + tableLoader().open(); + } + + Table table = tableLoader().loadTable(); + int formatVersion = TableUtil.formatVersion(table); + Preconditions.checkArgument( + formatVersion >= 3, + "ConvertEqualityDeletes requires table format version >= 3 (DVs), " + + "but table '%s' is version %s", + tableName(), + formatVersion); + + Schema schema = table.schema(); + List fieldIds = Lists.newArrayListWithCapacity(equalityFieldColumns.size()); + for (String column : equalityFieldColumns) { + NestedField field = schema.findField(column); + Preconditions.checkArgument( + field != null, + "Equality field column '%s' not found in table schema %s", + column, + schema); + fieldIds.add(field.fieldId()); + } + + return ImmutableSet.copyOf(fieldIds); + } + } +} diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java index c83b06a55abf..b3e8fb4d8938 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -71,7 +71,8 @@ public class EqualityConvertCommitter extends AbstractStreamOperator private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); - static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = "equality-convert-staging-snapshot"; + public static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = + "equality-convert-staging-snapshot"; private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java index c3c1785290cf..89e5510dd848 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -43,6 +43,7 @@ import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileContent; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; @@ -515,6 +516,7 @@ private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { deleteFile.location(), deleteFieldIds, eqFieldIds); + validateDeleteSpecPartitionColumns(stagingSnapshot, deleteFile); eqDeleteFiles.add(deleteFile); } else if (ContentFileUtil.isDV(deleteFile)) { stagingDVFiles.add(deleteFile); @@ -531,6 +533,24 @@ private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); } + private void validateDeleteSpecPartitionColumns(Snapshot stagingSnapshot, DeleteFile deleteFile) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + for (PartitionField field : spec.fields()) { + Preconditions.checkState( + eqFieldIds.contains(field.sourceId()), + "Staging snapshot %s on branch '%s' contains an equality delete file %s under spec %s, " + + "which partitions by field '%s' (source id %s) that is not an equality field %s. " + + "Partition columns must be a subset of the equality fields.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + spec.specId(), + field.name(), + field.sourceId(), + eqFieldIds); + } + } + /** Files added by one staging snapshot, classified for cycle emission. */ private record StagingInputs( List newDataFiles, diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java new file mode 100644 index 000000000000..1c2f230315be --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java @@ -0,0 +1,1333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.COMMIT_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.PLANNER_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ERROR_COUNTER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.stream.StreamSupport; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.graph.StreamGraphGenerator; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.MetricsReporterFactoryForTests; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructLikeSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestConvertEqualityDeletes extends MaintenanceTaskTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void testRejectsFormatVersion2() { + createTableWithDelete(2); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("format version >= 3"); + } + + @Test + void testRejectsUnknownEqualityFieldColumns() { + createTableWithDelete(3); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("nonexistent")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Equality field column 'nonexistent' not found in table schema"); + } + + @Test + void testConvertEqualityDeletesToDVs() throws Exception { + Table table = createTableWithDelete(3); + + // Insert initial data to main + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + assertThat(dataFileCount(table)).isEqualTo(3); + + // Create staging branch from current main state + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Write a new data file (simulating insert of id=4) + DataFile newDataFile = writeDataFile(table, createRecord(4, "d")); + + // Write an equality delete for id=2 (simulating delete of row "b") + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + + // Commit both to the staging branch + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Verify staging branch has the eq delete + long stagingEqDeleteCount = + table.snapshot(STAGING_BRANCH).deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(stagingEqDeleteCount).isEqualTo(1); + + // Wire the ConvertEqualityDeletes maintenance task + appendConvertTask(); + + // Run the maintenance task + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Verify main branch now has 4 data files (3 original + 1 new) + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(4); + + // Verify main branch has exactly one DV (id=2 deleted from its single-row data file) + long mainDvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .count(); + assertThat(mainDvCount).isEqualTo(1); + + // Verify no equality deletes on main + long mainEqDeleteCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(mainEqDeleteCount).isEqualTo(0); + + // Verify data correctness: id=2 should be deleted, id=4 should be added + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(3, "c"), createRecord(4, "d"))); + } + + @Test + void testMetrics() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot: a new data file (id=3) plus an eq-delete (id=1). The conversion commits + // exactly one data file and one DV to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Only metrics named on TableMaintenanceMetrics flow through the test reporter. Among the + // converter operators only the planner and committer own an ERROR_COUNTER; the parallel reader, + // PK index, and DV writer report failures through ERROR_STREAM instead. The committer also + // counts the data files it adds. Operator-specific counters (reindexCount, addedDvNum, ...) are + // asserted by the operator unit tests. A -1 expected value means "present, value not checked". + MetricsReporterFactoryForTests.assertCounters( + new ImmutableMap.Builder, Long>() + .put(errorKey(PLANNER_TASK_NAME), 0L) + .put(errorKey(COMMIT_TASK_NAME), 0L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_NUM_METRIC), 1L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_SIZE_METRIC), -1L) + .build()); + } + + private static List errorKey(String taskName) { + return metricKey(taskName, ERROR_COUNTER); + } + + private static List metricKey(String taskName, String metric) { + return ImmutableList.of(taskName + "[0]", DUMMY_TABLE_NAME, DUMMY_TASK_NAME, "0", metric); + } + + @Test + void testNoOpWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + // Create staging branch with no new files + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + // Should complete successfully with no changes + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(1); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testMultipleEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + insert(table, 4, "d"); + insert(table, 5, "e"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Delete id=1 and id=4 via equality deletes on staging + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + DeleteFile eqDelete2 = writeEqualityDelete(table, 4, "d"); + + table + .newRowDelta() + .addDeletes(eqDelete1) + .addDeletes(eqDelete2) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"), createRecord(5, "e"))); + } + + @Test + void testDuplicateKeyAcrossDataFiles() throws Exception { + Table table = createTableWithDelete(3); + + // Two data files with the same key (id=1, data="a") + insert(table, 1, "a"); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Eq delete for id=1 should produce DVs for both data files containing id=1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only id=2 should remain + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } + + @Test + void testMultiSnapshotStagingWithPerSnapshotScoping() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: delete id=1 from main + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: re-insert id=1 (new data file) + DataFile newDataFile = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (delete id=1) + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: processes S2 (re-insert id=1) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // id=1 should still exist: the delete from S1 removed the original, + // but S2 re-inserted it. Per-snapshot scoping ensures S1's delete + // doesn't affect S2's data. + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertThenDeleteAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: insert id=1 (data-only, no eq deletes) + DataFile insertS1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(insertS1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: eq delete the row written in S1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (insert id=1), commits data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: processes S2 (eq delete id=1), must find id=1 on main + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertUpdateDeleteInsertUpdateChain() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: insert K=1, V=A + DataFile s1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(s1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: update K=1 to V=B (eq delete + insert in same commit) + DataFile s2 = writeDataFile(table, createRecord(1, "b")); + DeleteFile e2 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(s2).addDeletes(e2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S3: delete K=1 + DeleteFile e3 = writeEqualityDelete(table, 1, "b"); + table.newRowDelta().addDeletes(e3).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S4: insert K=1, V=C + DataFile s4 = writeDataFile(table, createRecord(1, "c")); + table.newAppend().appendFile(s4).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S5: update K=1 to V=D (eq delete + insert in same commit) + DataFile s5 = writeDataFile(table, createRecord(1, "d")); + DeleteFile e5 = writeEqualityDelete(table, 1, "c"); + table.newRowDelta().addRows(s5).addDeletes(e5).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time = System.currentTimeMillis(); + + // Cycle 1: S1 inserts K=1, V=A + long time1 = time; + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: S2 updates K=1 from V=A to V=B (eq delete + insert) + long time2 = time + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "b"))); + + // Cycle 3: S3 deletes K=1 + long time3 = time + 2; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.exceptions()).isEmpty(); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 4: S4 inserts K=1, V=C + long time4 = time + 3; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.exceptions()).isEmpty(); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "c"))); + + // Cycle 5: S5 updates K=1 from V=C to V=D (eq delete + insert) + long time5 = time + 4; + infra.source().sendRecord(Trigger.create(time5, 0), time5); + TaskResult result5 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result5.exceptions()).isEmpty(); + assertThat(result5.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "d"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testParallelInsertOfToBeDeletedKeySurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Main holds the original (1, "a"); the staging eq-delete below removes this copy. + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot updates id=1 in place: re-insert (1, "a") plus an eq-delete (1, "a") in + // the same commit. The re-insert shares the equality key with the delete and carries the + // delete's sequence, so it must survive: the delete only removes the lower-sequence main copy. + // At parallelism > 1 the staging-data ADD can reach the index before the eq-delete resolves. + // Event-time phase ordering is what keeps the re-insert from being accidentally deleted. + DataFile reinsert = writeDataFile(table, createRecord(1, "a")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(reinsert).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(STAGING_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testDVMergeAcrossConversionCycles() throws Exception { + Table table = createTableWithDelete(3); + + // Single data file with 3 rows so DV merge applies to the same file + insert( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"), createRecord(3, "c"))); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1 setup: eq delete for id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: convert eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2 setup: eq delete for id=2 (committed while job is running) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 2: convert eq delete for id=2, should merge DV with existing + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + + // Verify: main has DVs, no equality deletes + assertNoEqualityDeletesOnMain(table, 0); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConversionCorrectAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + + // Three separate data files + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Compact file2 and file3 on main into one file (leave file1 + its DV untouched) + Set allDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + allDataFiles.add(df.copy()); + } + } + } + + // Find file1 (contains id=1) by checking which file has a DV against it + Set dvReferencedFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + dvReferencedFiles.add(df.referencedDataFile()); + } + } + } + } + + Set filesToCompact = Sets.newHashSet(); + for (DataFile df : allDataFiles) { + if (!dvReferencedFiles.contains(df.location())) { + filesToCompact.add(df); + } + } + + assertThat(filesToCompact).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : filesToCompact) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: delete id=2 (should target the compacted file) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConvertEqualityDeletesPartitionedTable() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Insert data into two partitions + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 2, "b"); + insertPartitioned(table, 3, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Partition-scoped equality delete for id=1 in partition data="a" + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // id=1 deleted from partition "a", id=2 in partition "b" and id=3 in partition "a" remain + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "a"))); + } + + @Test + void testEqualityDeleteIsScopedToItsPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Same PK (id=1) exists in two partitions. An eq delete in one partition must not delete + // rows in the other. + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 1, "b"); + insertPartitioned(table, 2, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only (1, "a") is deleted; (1, "b") remains because the eq delete was scoped to partition + // "a", and (2, "a") remains because its equality field values don't match. + assertRecords(table, ImmutableList.of(createRecord(1, "b"), createRecord(2, "a"))); + } + + @Test + void testStagingPositionDeleteMergedIntoConversionDV() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: write a data file with two rows (id=1 at pos 0, id=2 at pos 1). + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: eq delete matches row 0 (will produce a conversion DV at pos 0) + a position + // delete DV referencing the same data file at pos 1. Both DVs target the same data + // file and must be merged into a single DV (V3 invariant). + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table + .newRowDelta() + .addDeletes(eqDelete) + .addDeletes(stagingDV) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: commits S1's data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: converts S2's eq delete to DV, merges with staging DV + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Both rows from S1 must be masked: pos 0 by the conversion DV, pos 1 by the staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant): the resolver must have folded the + // staging DV's positions into the conversion DV. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingDataFilesOnlyNoEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging has only a new data file, no equality deletes + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(2); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"))); + } + + @Test + void testReindexAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"))); + + // External commit: insert id=3 directly to main (not via staging) + insert(table, 3, "c"); + + // Cycle 2: delete id=2 (should reindex because of external commit) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testReindexEvictsGhostKeyAfterExternalDataFileRemoval() throws Exception { + // CoW removal case: an external commit removes a data file, leaving a stale + // PK in the worker's index. A later staging eq-delete for that PK must NOT produce a DV + // referencing the removed file. The external commit advances main, so the next cycle reindexes + // and the worker clears the ghost key before resolving the delete. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.refresh(); + DataFile file1 = table.currentSnapshot().addedDataFiles(table.io()).iterator().next().copy(); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=2. Bootstraps the worker index from main (id=1 -> file1, id=2 -> file2). + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // External CoW-style removal: drop file1 (id=1) from main without re-adding the row. The + // worker's index still holds the ghost id=1 -> file1 until the next reindex clears it. + table.newDelete().deleteFile(file1).commit(); + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 2: stage an eq-delete for the removed key id=1. Without ghost eviction the worker + // would emit a DV position against the now-absent file1. + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + // No deletion vector may reference the removed file1. + table.refresh(); + assertThat(dvReferencedDataFiles(table)).doesNotContain(file1.location()); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + private static Set dvReferencedDataFiles(Table table) { + Set referenced = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + referenced.add(df.referencedDataFile()); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return referenced; + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + Schema eqDeleteSchema = table.schema(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + eqDeleteSchema); + } + + private DeleteFile writePartitionedEqualityDelete(Table table, Integer id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + partition, + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private static long dataFileCount(Table table) { + table.refresh(); + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile ignored : reader) { + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return count; + } + + @Test + void testStagingEqualsTargetBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Write eq delete directly to main (no separate staging branch) + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DataFile newData = writeDataFile(table, createRecord(3, "c")); + table.newRowDelta().addRows(newData).addDeletes(eqDelete).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: process the eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + long dataFilesAfterCycle1 = dataFileCount(table); + // Expect 3: two from insert() + one from the writer's rowDelta.addRows(newData). + // When stagingBranch == targetBranch, the committer must NOT re-add newData via + // rowDelta.addRows(...) — that would duplicate (count=4). + assertThat(dataFilesAfterCycle1).isEqualTo(3); + + // Cycle 2: no-op (converter's own commit must be skipped) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle1); + + // New eq delete for id=2 committed directly to main between cycles + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + DataFile newData2 = writeDataFile(table, createRecord(4, "d")); + table.newRowDelta().addRows(newData2).addDeletes(eqDelete2).commit(); + table.refresh(); + + // Cycle 3: process the new eq delete + long time3 = time2 + 1; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + long dataFilesAfterCycle3 = dataFileCount(table); + + // Cycle 4: no-op again + long time4 = time3 + 1; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle3); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchColdStartCatchUp() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + // Writer commits three eq-deletes to main BEFORE the converter starts. + // Cold start must pick up the unconverted history, not just the head snapshot. + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 2, "b")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 3, "c")).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One unconverted snapshot per cycle, oldest first. After three cycles every eq-delete + // commit has its own committer commit carrying the marker. + for (int cycle = 1; cycle <= 3; cycle++) { + long ts = System.currentTimeMillis() + cycle; + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.success()).isTrue(); + } + + table.refresh(); + long convertedCount = + StreamSupport.stream(table.snapshots().spliterator(), false) + .filter(s -> s.summary().containsKey(COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .count(); + assertThat(convertedCount).isEqualTo(3); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchReinsertAfterDeleteSurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Shared branch: insert id=1, eq-delete id=1, then re-insert id=1. The re-insert has a higher + // sequence than the delete and must survive the conversion (sequence-aware resolution). + insert(table, 1, "a"); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newAppend().appendFile(writeDataFile(table, createRecord(1, "a"))).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One cycle converts the eq-delete: the original row is deleted, the newer re-insert is + // not (its sequence is at or above the delete's). + long ts = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.exceptions()).isEmpty(); + assertThat(result.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchMergesStagingDvIntoSingleDv() throws Exception { + Table table = createTableWithDelete(3); + + // One data file with two rows: id=1 at pos 0, id=2 at pos 1, committed to main. + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).commit(); + table.refresh(); + + // Same-branch commit: an eq-delete for id=1 (resolves to a conversion DV at pos 0) plus a + // writer DV at pos 1 on the same data file. The resolver folds the staging DV into the + // conversion DV; on a shared branch the committer must remove the superseded staging DV. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table.newRowDelta().addDeletes(eqDelete).addDeletes(stagingDV).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Both rows masked: pos 0 by the conversion DV, pos 1 by the merged-in staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant). Without removing the rewritten staging DV on a + // shared branch, the data file would carry two DVs. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } + + @Test + void testReaderErrorSkipsCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + long mainSnapshotBeforeStaging = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging data file + eq delete file, both referenced by the staging commit. + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + File eqDeleteLocalFile = new File(eqDelete.location().replace("file:", "")); + + // Delete the eq delete file; the committer must abort rather than committing data without its + // DV. + assertThat(eqDeleteLocalFile.delete()).isTrue(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result1.success()).isFalse(); + assertThat(result1.exceptions()).isNotEmpty(); + + table.refresh(); + // Main must not have advanced (no commit happened). + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotBeforeStaging); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Restore the eq delete file content by rewriting an identical delete, and retry: + // the planner must re-process the same staging snapshot (cursor didn't advance on failure). + DeleteFile recreated = + FileHelpers.writeDeleteFile( + table, + Files.localOutput(eqDeleteLocalFile), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(1, "a")), + table.schema()); + assertThat(recreated.location()).isEqualTo(eqDelete.location()); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Staging data file committed with DV for id=1: should see id=2 only. + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } finally { + closeJobClient(jobClient); + } + } + + private void appendConvertTask() { + appendConvertTask(STAGING_BRANCH); + } + + private void appendConvertTask(String stagingBranch) { + ConvertEqualityDeletes.builder() + .stagingBranch(stagingBranch) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1) + .sinkTo(infra.sink()); + } + + private static void assertRecords(Table table, List expected) throws IOException { + table.refresh(); + Types.StructType type = SimpleDataUtil.SCHEMA.asStruct(); + + StructLikeSet expectedSet = StructLikeSet.create(type); + expectedSet.addAll(expected); + + try (CloseableIterable iterable = + IcebergGenerics.read(table) + .useSnapshot(table.currentSnapshot().snapshotId()) + .project(SimpleDataUtil.SCHEMA) + .build()) { + StructLikeSet actualSet = StructLikeSet.create(type); + for (Record record : iterable) { + actualSet.add(record); + } + + assertThat(actualSet).isEqualTo(expectedSet); + } + } + + private static void assertNoEqualityDeletesOnMain(Table table, long expectedEqDeleteCount) { + long mainEqDeleteCount = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile f : reader) { + if (f.content() == FileContent.EQUALITY_DELETES) { + mainEqDeleteCount++; + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + assertThat(mainEqDeleteCount).isEqualTo(expectedEqDeleteCount); + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java new file mode 100644 index 000000000000..f6d1fd9464e2 --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.util.ContentFileUtil; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * End-to-end test for {@link ConvertEqualityDeletes} wired through the {@link TableMaintenance} + * framework. Verifies that the converter actually runs and commits a DV when the framework triggers + * it, exercising the full operator graph including the framework's monitor source, trigger manager, + * and lock remover. + */ +class TestConvertEqualityDeletesE2E extends OperatorTestBase { + private static final String STAGING_BRANCH = "staging"; + + @TempDir private Path tempDir; + private StreamExecutionEnvironment env; + + @BeforeEach + public void beforeEach() { + this.env = StreamExecutionEnvironment.getExecutionEnvironment(); + } + + @ParameterizedTest + @ValueSource(strings = {STAGING_BRANCH, SnapshotRef.MAIN_BRANCH}) + void testConvertEqualityDeletesE2E(String stagingBranch) throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // When staging is a separate branch, fork it from main first. + if (!stagingBranch.equals(SnapshotRef.MAIN_BRANCH)) { + table.manageSnapshots().createBranch(stagingBranch).commit(); + table.refresh(); + } + + // Commit a new data file + eq delete to staging. This pre-job snapshot exercises both the + // "new data file" and "eq delete" paths in one cycle. + DataFile newData = writeDataFile(table, 3, "c"); + DeleteFile firstDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newData).addDeletes(firstDelete).toBranch(stagingBranch).commit(); + table.refresh(); + assertThat(dvCountOnMain(table)).isZero(); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("ConvertEqualityDeletesE2EUID-" + stagingBranch) + .rateLimit(Duration.ofMillis(50)) + .lockCheckDelay(Duration.ofMillis(50)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnInterval(Duration.ofMillis(100)) + .stagingBranch(stagingBranch) + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Cycle 1: row 1 deleted by the converted DV; row 3 added on staging and committed to main. + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(1)); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: commit a second staging snapshot while the job is still running. The framework's + // next interval-trigger should pick it up and produce a second DV against the data file + // holding id=2. + table.refresh(); + DeleteFile secondDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(secondDelete).toBranch(stagingBranch).commit(); + + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(2)); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + private DataFile writeDataFile(Table table, Integer id, String data) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(SimpleDataUtil.createRecord(id, data))); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(SimpleDataUtil.createRecord(id, data)), + table.schema()); + } + + private static long dvCountOnMain(Table table) throws IOException { + table.refresh(); + if (table.currentSnapshot() == null) { + return 0; + } + + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (ContentFileUtil.isDV(file)) { + count++; + } + } + } + } + + return count; + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java index fe8457167a1f..f786f1cdb29d 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java @@ -24,8 +24,11 @@ import java.time.Duration; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -116,4 +119,34 @@ void testE2eUseCoordinator() throws Exception { closeJobClient(jobClient); } } + + @Test + void testE2eConvertEqualityDeletes() throws Exception { + // Converter requires V3 (DV support); replace the V2 table created in @BeforeEach. + dropTable(); + createTable(3, FileFormat.PARQUET); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("E2eConvertEqualityDeletesUID") + .rateLimit(Duration.ofMinutes(10)) + .lockCheckDelay(Duration.ofSeconds(10)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnEqDeleteFileCount(1) + .stagingBranch("staging") + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Just make sure that we are able to instantiate the flow + assertThat(jobClient).isNotNull(); + } finally { + closeJobClient(jobClient); + } + } } diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java index 82860a0de881..1ee5c5d0d5fd 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -110,6 +110,33 @@ void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { } } + @Test + void failsWhenStagingEqDeleteSpecPartitionsByNonEqualityColumn() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeIdOnlyPartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + harness.open(); + sendTrigger(harness); + + List> errOutput = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + + assertThat(errOutput).hasSize(1); + assertThat(errOutput.get(0).getValue()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Partition columns must be a subset of the equality fields."); + } + } + @Test void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { // When stagingBranch == targetBranch, the writer commits new data files directly to main. @@ -1063,6 +1090,19 @@ private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOExcep idOnly); } + private DeleteFile writeIdOnlyPartitionedEqualityDelete(Table table, int id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, Files.localOutput(file), partition, Lists.newArrayList(record), idOnly); + } + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) throws IOException { File file = File.createTempFile("junit", null, tempDir.toFile()); diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java new file mode 100644 index 000000000000..f51df3338d02 --- /dev/null +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.flink.annotation.Experimental; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.BroadcastStream; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.flink.maintenance.operator.DVPosition; +import org.apache.iceberg.flink.maintenance.operator.DVWriteResult; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertDVWriter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPKIndex; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlan; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlanner; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertReader; +import org.apache.iceberg.flink.maintenance.operator.IndexCommand; +import org.apache.iceberg.flink.maintenance.operator.ReadCommand; +import org.apache.iceberg.flink.maintenance.operator.SerializedEqualityValues; +import org.apache.iceberg.flink.maintenance.operator.TaskResultAggregator; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types.NestedField; + +/** + * Creates the equality delete to DV conversion data stream. Runs a single iteration of the + * conversion for every {@link Trigger} event. + * + *

The pipeline reads equality delete files from a staging branch, converts them to deletion + * vectors (DVs) using a primary key index stored in Flink state, and commits the data files and DVs + * to the target branch. + * + *

The conversion is split into parallel stages: + * + *

    + *
  1. Planner (p=1): scans staging branch, emits file-level ReadCommands with phase timestamps + *
  2. Reader (p=N): reads files, emits row-level IndexCommands + *
  3. PKIndex (p=N): maintains PK index shards, resolves equality deletes to DV positions + *
  4. DVWriter (p=N, keyed by data file path): buffers positions per file, writes Puffin DVs + * inline + *
  5. Committer (p=1): commits data files and DVs to the target branch + *
+ * + *

Mutual exclusion with concurrent maintenance tasks (e.g. compaction) is enforced by the Flink + * maintenance framework lock. + */ +@Experimental +public class ConvertEqualityDeletes { + static final String PLANNER_TASK_NAME = "EqConvert Planner"; + static final String READER_TASK_NAME = "EqConvert Reader"; + static final String PK_INDEX_TASK_NAME = "EqConvert PKIndex"; + static final String DV_WRITER_TASK_NAME = "EqConvert DVWriter"; + static final String UPSTREAM_ABORT_TASK_NAME = "EqConvert UpstreamAbort"; + static final String COMMIT_TASK_NAME = "EqConvert Commit"; + static final String AGGREGATOR_TASK_NAME = "EqConvert Aggregator"; + + private ConvertEqualityDeletes() {} + + public static Builder builder() { + return new Builder(); + } + + public static class Builder extends MaintenanceTaskBuilder { + private String stagingBranch; + private String targetBranch = SnapshotRef.MAIN_BRANCH; + private List equalityFieldColumns = Collections.emptyList(); + + @Override + String maintenanceTaskName() { + return "ConvertEqualityDeletes"; + } + + /** Sets the staging branch name that holds the equality delete files and data files. */ + public Builder stagingBranch(String newStagingBranch) { + this.stagingBranch = newStagingBranch; + return this; + } + + /** + * Sets the target branch where converted data files and DVs are committed. Defaults to the main + * branch. + */ + public Builder targetBranch(String newTargetBranch) { + this.targetBranch = newTargetBranch; + return this; + } + + /** + * Sets the equality field columns used by the worker index. Required. Must match the equality + * field columns the writer uses for staging eq-delete files. Mirrors {@link + * org.apache.iceberg.flink.sink.IcebergSink.Builder#equalityFieldColumns}. + * + *

The partition source columns of an equality delete's spec must be a subset of these + * columns. Writes via Flink's IcebergSink already ensure this. + */ + public Builder equalityFieldColumns(List columns) { + Preconditions.checkNotNull(columns, "equalityFieldColumns must not be null"); + Preconditions.checkArgument(!columns.isEmpty(), "equalityFieldColumns must not be empty"); + this.equalityFieldColumns = ImmutableList.copyOf(columns); + return this; + } + + @Override + DataStream append(DataStream trigger) { + Preconditions.checkNotNull(stagingBranch, "stagingBranch must be set"); + Preconditions.checkArgument( + !equalityFieldColumns.isEmpty(), "equalityFieldColumns must be set on the builder"); + Set eqFieldIds = resolveEqualityFieldIds(); + + // Planner (p=1): emits ReadCommands with phase timestamps and watermarks + SingleOutputStreamOperator planned = + setSlotSharingGroup( + trigger + .transform( + operatorName(PLANNER_TASK_NAME), + TypeInformation.of(ReadCommand.class), + new EqualityConvertPlanner( + tableName(), + taskName(), + tableLoader(), + stagingBranch, + targetBranch, + eqFieldIds)) + .uid(PLANNER_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Reader (p=N): reads files, emits IndexCommands + SingleOutputStreamOperator index = + setSlotSharingGroup( + planned + .rebalance() + .process( + new EqualityConvertReader( + tableLoader(), eqFieldIds, stagingBranch.equals(targetBranch))) + .name(operatorName(READER_TASK_NAME)) + .uid(READER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Broadcast from the planner to the PKIndex to clear the entire index + BroadcastStream clearIndexBroadcast = + planned + .getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM) + .broadcast(EqualityConvertPKIndex.CLEAR_BROADCAST_DESCRIPTOR); + + // PKIndex (p=N): keyed by full PK, phase-aware buffering. + SingleOutputStreamOperator dvPositions = + setSlotSharingGroup( + index + .keyBy(IndexCommand::key, TypeInformation.of(SerializedEqualityValues.class)) + .connect(clearIndexBroadcast) + .process(new EqualityConvertPKIndex(stagingBranch.equals(targetBranch))) + .name(operatorName(PK_INDEX_TASK_NAME)) + .uid(PK_INDEX_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Reader-side abort signals bypass the PKIndex and feed the DVWriter directly, so a reader + // failure can short-circuit the cycle without waiting on a keyed shuffle. This is not a full + // short-circuit: the abort is keyed by data file path (empty for ABORT), so only one resolver + // subtask observes it; the others still write their buffered DVs, which the committer then + // drops. + DataStream readerAborts = + index.getSideOutput(EqualityConvertReader.READER_ABORT_STREAM); + DataStream dvPositionsWithAborts = dvPositions.union(readerAborts); + + // Metadata side output from planner + DataStream metadata = + planned.getSideOutput(EqualityConvertPlanner.METADATA_STREAM); + + // DVWriter (p=N, keyed by data file path): groups positions per file, writes Puffin DV + // files inline, emits a DVWriteResult per cycle. Plan metadata broadcast so every subtask + // sees it. + SingleOutputStreamOperator resolved = + setSlotSharingGroup( + dvPositionsWithAborts + .keyBy(DVPosition::dataFilePath) + .connect(metadata.broadcast()) + .transform( + operatorName(DV_WRITER_TASK_NAME), + TypeInformation.of(DVWriteResult.class), + new EqualityConvertDVWriter( + tableName(), taskName(), tableLoader(), targetBranch)) + .uid(DV_WRITER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Upstream errors become abort signals so a partial read never commits. The same error side + // outputs also feed the aggregator below to surface the exception in TaskResult; the two + // consumers serve different purposes and must both exist. + DataStream upstreamAborts = + setSlotSharingGroup( + index + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .map(e -> DVWriteResult.ABORT) + .returns(TypeInformation.of(DVWriteResult.class)) + .name(operatorName(UPSTREAM_ABORT_TASK_NAME)) + .uid(UPSTREAM_ABORT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Committer (p=1): commits data files + DVs to main. + SingleOutputStreamOperator committed = + setSlotSharingGroup( + resolved + .union(upstreamAborts) + .connect(metadata) + .transform( + operatorName(COMMIT_TASK_NAME), + TypeInformation.of(Trigger.class), + new EqualityConvertCommitter( + tableName(), taskName(), tableLoader(), stagingBranch, targetBranch)) + .uid(COMMIT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Aggregator (p=1): collects errors and emits TaskResult. + return setSlotSharingGroup( + committed + .connect( + planned + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(index.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(resolved.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(committed.getSideOutput(TaskResultAggregator.ERROR_STREAM))) + .transform( + operatorName(AGGREGATOR_TASK_NAME), + TypeInformation.of(TaskResult.class), + new TaskResultAggregator(tableName(), taskName(), index())) + .uid(AGGREGATOR_TASK_NAME + uidSuffix()) + .forceNonParallel()); + } + + private Set resolveEqualityFieldIds() { + if (!tableLoader().isOpen()) { + tableLoader().open(); + } + + Table table = tableLoader().loadTable(); + int formatVersion = TableUtil.formatVersion(table); + Preconditions.checkArgument( + formatVersion >= 3, + "ConvertEqualityDeletes requires table format version >= 3 (DVs), " + + "but table '%s' is version %s", + tableName(), + formatVersion); + + Schema schema = table.schema(); + List fieldIds = Lists.newArrayListWithCapacity(equalityFieldColumns.size()); + for (String column : equalityFieldColumns) { + NestedField field = schema.findField(column); + Preconditions.checkArgument( + field != null, + "Equality field column '%s' not found in table schema %s", + column, + schema); + fieldIds.add(field.fieldId()); + } + + return ImmutableSet.copyOf(fieldIds); + } + } +} diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java index c83b06a55abf..b3e8fb4d8938 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -71,7 +71,8 @@ public class EqualityConvertCommitter extends AbstractStreamOperator private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); - static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = "equality-convert-staging-snapshot"; + public static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = + "equality-convert-staging-snapshot"; private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java index c3c1785290cf..89e5510dd848 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -43,6 +43,7 @@ import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileContent; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; @@ -515,6 +516,7 @@ private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { deleteFile.location(), deleteFieldIds, eqFieldIds); + validateDeleteSpecPartitionColumns(stagingSnapshot, deleteFile); eqDeleteFiles.add(deleteFile); } else if (ContentFileUtil.isDV(deleteFile)) { stagingDVFiles.add(deleteFile); @@ -531,6 +533,24 @@ private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); } + private void validateDeleteSpecPartitionColumns(Snapshot stagingSnapshot, DeleteFile deleteFile) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + for (PartitionField field : spec.fields()) { + Preconditions.checkState( + eqFieldIds.contains(field.sourceId()), + "Staging snapshot %s on branch '%s' contains an equality delete file %s under spec %s, " + + "which partitions by field '%s' (source id %s) that is not an equality field %s. " + + "Partition columns must be a subset of the equality fields.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + spec.specId(), + field.name(), + field.sourceId(), + eqFieldIds); + } + } + /** Files added by one staging snapshot, classified for cycle emission. */ private record StagingInputs( List newDataFiles, diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java new file mode 100644 index 000000000000..1c2f230315be --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java @@ -0,0 +1,1333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.COMMIT_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.PLANNER_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ERROR_COUNTER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.stream.StreamSupport; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.graph.StreamGraphGenerator; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.MetricsReporterFactoryForTests; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructLikeSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestConvertEqualityDeletes extends MaintenanceTaskTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void testRejectsFormatVersion2() { + createTableWithDelete(2); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("format version >= 3"); + } + + @Test + void testRejectsUnknownEqualityFieldColumns() { + createTableWithDelete(3); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("nonexistent")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Equality field column 'nonexistent' not found in table schema"); + } + + @Test + void testConvertEqualityDeletesToDVs() throws Exception { + Table table = createTableWithDelete(3); + + // Insert initial data to main + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + assertThat(dataFileCount(table)).isEqualTo(3); + + // Create staging branch from current main state + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Write a new data file (simulating insert of id=4) + DataFile newDataFile = writeDataFile(table, createRecord(4, "d")); + + // Write an equality delete for id=2 (simulating delete of row "b") + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + + // Commit both to the staging branch + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Verify staging branch has the eq delete + long stagingEqDeleteCount = + table.snapshot(STAGING_BRANCH).deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(stagingEqDeleteCount).isEqualTo(1); + + // Wire the ConvertEqualityDeletes maintenance task + appendConvertTask(); + + // Run the maintenance task + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Verify main branch now has 4 data files (3 original + 1 new) + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(4); + + // Verify main branch has exactly one DV (id=2 deleted from its single-row data file) + long mainDvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .count(); + assertThat(mainDvCount).isEqualTo(1); + + // Verify no equality deletes on main + long mainEqDeleteCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(mainEqDeleteCount).isEqualTo(0); + + // Verify data correctness: id=2 should be deleted, id=4 should be added + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(3, "c"), createRecord(4, "d"))); + } + + @Test + void testMetrics() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot: a new data file (id=3) plus an eq-delete (id=1). The conversion commits + // exactly one data file and one DV to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Only metrics named on TableMaintenanceMetrics flow through the test reporter. Among the + // converter operators only the planner and committer own an ERROR_COUNTER; the parallel reader, + // PK index, and DV writer report failures through ERROR_STREAM instead. The committer also + // counts the data files it adds. Operator-specific counters (reindexCount, addedDvNum, ...) are + // asserted by the operator unit tests. A -1 expected value means "present, value not checked". + MetricsReporterFactoryForTests.assertCounters( + new ImmutableMap.Builder, Long>() + .put(errorKey(PLANNER_TASK_NAME), 0L) + .put(errorKey(COMMIT_TASK_NAME), 0L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_NUM_METRIC), 1L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_SIZE_METRIC), -1L) + .build()); + } + + private static List errorKey(String taskName) { + return metricKey(taskName, ERROR_COUNTER); + } + + private static List metricKey(String taskName, String metric) { + return ImmutableList.of(taskName + "[0]", DUMMY_TABLE_NAME, DUMMY_TASK_NAME, "0", metric); + } + + @Test + void testNoOpWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + // Create staging branch with no new files + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + // Should complete successfully with no changes + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(1); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testMultipleEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + insert(table, 4, "d"); + insert(table, 5, "e"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Delete id=1 and id=4 via equality deletes on staging + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + DeleteFile eqDelete2 = writeEqualityDelete(table, 4, "d"); + + table + .newRowDelta() + .addDeletes(eqDelete1) + .addDeletes(eqDelete2) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"), createRecord(5, "e"))); + } + + @Test + void testDuplicateKeyAcrossDataFiles() throws Exception { + Table table = createTableWithDelete(3); + + // Two data files with the same key (id=1, data="a") + insert(table, 1, "a"); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Eq delete for id=1 should produce DVs for both data files containing id=1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only id=2 should remain + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } + + @Test + void testMultiSnapshotStagingWithPerSnapshotScoping() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: delete id=1 from main + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: re-insert id=1 (new data file) + DataFile newDataFile = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (delete id=1) + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: processes S2 (re-insert id=1) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // id=1 should still exist: the delete from S1 removed the original, + // but S2 re-inserted it. Per-snapshot scoping ensures S1's delete + // doesn't affect S2's data. + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertThenDeleteAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: insert id=1 (data-only, no eq deletes) + DataFile insertS1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(insertS1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: eq delete the row written in S1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (insert id=1), commits data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: processes S2 (eq delete id=1), must find id=1 on main + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertUpdateDeleteInsertUpdateChain() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: insert K=1, V=A + DataFile s1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(s1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: update K=1 to V=B (eq delete + insert in same commit) + DataFile s2 = writeDataFile(table, createRecord(1, "b")); + DeleteFile e2 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(s2).addDeletes(e2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S3: delete K=1 + DeleteFile e3 = writeEqualityDelete(table, 1, "b"); + table.newRowDelta().addDeletes(e3).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S4: insert K=1, V=C + DataFile s4 = writeDataFile(table, createRecord(1, "c")); + table.newAppend().appendFile(s4).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S5: update K=1 to V=D (eq delete + insert in same commit) + DataFile s5 = writeDataFile(table, createRecord(1, "d")); + DeleteFile e5 = writeEqualityDelete(table, 1, "c"); + table.newRowDelta().addRows(s5).addDeletes(e5).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time = System.currentTimeMillis(); + + // Cycle 1: S1 inserts K=1, V=A + long time1 = time; + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: S2 updates K=1 from V=A to V=B (eq delete + insert) + long time2 = time + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "b"))); + + // Cycle 3: S3 deletes K=1 + long time3 = time + 2; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.exceptions()).isEmpty(); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 4: S4 inserts K=1, V=C + long time4 = time + 3; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.exceptions()).isEmpty(); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "c"))); + + // Cycle 5: S5 updates K=1 from V=C to V=D (eq delete + insert) + long time5 = time + 4; + infra.source().sendRecord(Trigger.create(time5, 0), time5); + TaskResult result5 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result5.exceptions()).isEmpty(); + assertThat(result5.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "d"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testParallelInsertOfToBeDeletedKeySurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Main holds the original (1, "a"); the staging eq-delete below removes this copy. + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot updates id=1 in place: re-insert (1, "a") plus an eq-delete (1, "a") in + // the same commit. The re-insert shares the equality key with the delete and carries the + // delete's sequence, so it must survive: the delete only removes the lower-sequence main copy. + // At parallelism > 1 the staging-data ADD can reach the index before the eq-delete resolves. + // Event-time phase ordering is what keeps the re-insert from being accidentally deleted. + DataFile reinsert = writeDataFile(table, createRecord(1, "a")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(reinsert).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(STAGING_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testDVMergeAcrossConversionCycles() throws Exception { + Table table = createTableWithDelete(3); + + // Single data file with 3 rows so DV merge applies to the same file + insert( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"), createRecord(3, "c"))); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1 setup: eq delete for id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: convert eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2 setup: eq delete for id=2 (committed while job is running) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 2: convert eq delete for id=2, should merge DV with existing + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + + // Verify: main has DVs, no equality deletes + assertNoEqualityDeletesOnMain(table, 0); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConversionCorrectAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + + // Three separate data files + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Compact file2 and file3 on main into one file (leave file1 + its DV untouched) + Set allDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + allDataFiles.add(df.copy()); + } + } + } + + // Find file1 (contains id=1) by checking which file has a DV against it + Set dvReferencedFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + dvReferencedFiles.add(df.referencedDataFile()); + } + } + } + } + + Set filesToCompact = Sets.newHashSet(); + for (DataFile df : allDataFiles) { + if (!dvReferencedFiles.contains(df.location())) { + filesToCompact.add(df); + } + } + + assertThat(filesToCompact).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : filesToCompact) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: delete id=2 (should target the compacted file) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConvertEqualityDeletesPartitionedTable() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Insert data into two partitions + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 2, "b"); + insertPartitioned(table, 3, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Partition-scoped equality delete for id=1 in partition data="a" + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // id=1 deleted from partition "a", id=2 in partition "b" and id=3 in partition "a" remain + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "a"))); + } + + @Test + void testEqualityDeleteIsScopedToItsPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Same PK (id=1) exists in two partitions. An eq delete in one partition must not delete + // rows in the other. + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 1, "b"); + insertPartitioned(table, 2, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only (1, "a") is deleted; (1, "b") remains because the eq delete was scoped to partition + // "a", and (2, "a") remains because its equality field values don't match. + assertRecords(table, ImmutableList.of(createRecord(1, "b"), createRecord(2, "a"))); + } + + @Test + void testStagingPositionDeleteMergedIntoConversionDV() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: write a data file with two rows (id=1 at pos 0, id=2 at pos 1). + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: eq delete matches row 0 (will produce a conversion DV at pos 0) + a position + // delete DV referencing the same data file at pos 1. Both DVs target the same data + // file and must be merged into a single DV (V3 invariant). + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table + .newRowDelta() + .addDeletes(eqDelete) + .addDeletes(stagingDV) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: commits S1's data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: converts S2's eq delete to DV, merges with staging DV + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Both rows from S1 must be masked: pos 0 by the conversion DV, pos 1 by the staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant): the resolver must have folded the + // staging DV's positions into the conversion DV. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingDataFilesOnlyNoEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging has only a new data file, no equality deletes + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(2); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"))); + } + + @Test + void testReindexAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"))); + + // External commit: insert id=3 directly to main (not via staging) + insert(table, 3, "c"); + + // Cycle 2: delete id=2 (should reindex because of external commit) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testReindexEvictsGhostKeyAfterExternalDataFileRemoval() throws Exception { + // CoW removal case: an external commit removes a data file, leaving a stale + // PK in the worker's index. A later staging eq-delete for that PK must NOT produce a DV + // referencing the removed file. The external commit advances main, so the next cycle reindexes + // and the worker clears the ghost key before resolving the delete. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.refresh(); + DataFile file1 = table.currentSnapshot().addedDataFiles(table.io()).iterator().next().copy(); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=2. Bootstraps the worker index from main (id=1 -> file1, id=2 -> file2). + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // External CoW-style removal: drop file1 (id=1) from main without re-adding the row. The + // worker's index still holds the ghost id=1 -> file1 until the next reindex clears it. + table.newDelete().deleteFile(file1).commit(); + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 2: stage an eq-delete for the removed key id=1. Without ghost eviction the worker + // would emit a DV position against the now-absent file1. + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + // No deletion vector may reference the removed file1. + table.refresh(); + assertThat(dvReferencedDataFiles(table)).doesNotContain(file1.location()); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + private static Set dvReferencedDataFiles(Table table) { + Set referenced = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + referenced.add(df.referencedDataFile()); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return referenced; + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + Schema eqDeleteSchema = table.schema(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + eqDeleteSchema); + } + + private DeleteFile writePartitionedEqualityDelete(Table table, Integer id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + partition, + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private static long dataFileCount(Table table) { + table.refresh(); + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile ignored : reader) { + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return count; + } + + @Test + void testStagingEqualsTargetBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Write eq delete directly to main (no separate staging branch) + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DataFile newData = writeDataFile(table, createRecord(3, "c")); + table.newRowDelta().addRows(newData).addDeletes(eqDelete).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: process the eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + long dataFilesAfterCycle1 = dataFileCount(table); + // Expect 3: two from insert() + one from the writer's rowDelta.addRows(newData). + // When stagingBranch == targetBranch, the committer must NOT re-add newData via + // rowDelta.addRows(...) — that would duplicate (count=4). + assertThat(dataFilesAfterCycle1).isEqualTo(3); + + // Cycle 2: no-op (converter's own commit must be skipped) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle1); + + // New eq delete for id=2 committed directly to main between cycles + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + DataFile newData2 = writeDataFile(table, createRecord(4, "d")); + table.newRowDelta().addRows(newData2).addDeletes(eqDelete2).commit(); + table.refresh(); + + // Cycle 3: process the new eq delete + long time3 = time2 + 1; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + long dataFilesAfterCycle3 = dataFileCount(table); + + // Cycle 4: no-op again + long time4 = time3 + 1; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle3); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchColdStartCatchUp() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + // Writer commits three eq-deletes to main BEFORE the converter starts. + // Cold start must pick up the unconverted history, not just the head snapshot. + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 2, "b")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 3, "c")).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One unconverted snapshot per cycle, oldest first. After three cycles every eq-delete + // commit has its own committer commit carrying the marker. + for (int cycle = 1; cycle <= 3; cycle++) { + long ts = System.currentTimeMillis() + cycle; + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.success()).isTrue(); + } + + table.refresh(); + long convertedCount = + StreamSupport.stream(table.snapshots().spliterator(), false) + .filter(s -> s.summary().containsKey(COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .count(); + assertThat(convertedCount).isEqualTo(3); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchReinsertAfterDeleteSurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Shared branch: insert id=1, eq-delete id=1, then re-insert id=1. The re-insert has a higher + // sequence than the delete and must survive the conversion (sequence-aware resolution). + insert(table, 1, "a"); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newAppend().appendFile(writeDataFile(table, createRecord(1, "a"))).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One cycle converts the eq-delete: the original row is deleted, the newer re-insert is + // not (its sequence is at or above the delete's). + long ts = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.exceptions()).isEmpty(); + assertThat(result.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchMergesStagingDvIntoSingleDv() throws Exception { + Table table = createTableWithDelete(3); + + // One data file with two rows: id=1 at pos 0, id=2 at pos 1, committed to main. + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).commit(); + table.refresh(); + + // Same-branch commit: an eq-delete for id=1 (resolves to a conversion DV at pos 0) plus a + // writer DV at pos 1 on the same data file. The resolver folds the staging DV into the + // conversion DV; on a shared branch the committer must remove the superseded staging DV. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table.newRowDelta().addDeletes(eqDelete).addDeletes(stagingDV).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Both rows masked: pos 0 by the conversion DV, pos 1 by the merged-in staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant). Without removing the rewritten staging DV on a + // shared branch, the data file would carry two DVs. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } + + @Test + void testReaderErrorSkipsCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + long mainSnapshotBeforeStaging = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging data file + eq delete file, both referenced by the staging commit. + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + File eqDeleteLocalFile = new File(eqDelete.location().replace("file:", "")); + + // Delete the eq delete file; the committer must abort rather than committing data without its + // DV. + assertThat(eqDeleteLocalFile.delete()).isTrue(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result1.success()).isFalse(); + assertThat(result1.exceptions()).isNotEmpty(); + + table.refresh(); + // Main must not have advanced (no commit happened). + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotBeforeStaging); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Restore the eq delete file content by rewriting an identical delete, and retry: + // the planner must re-process the same staging snapshot (cursor didn't advance on failure). + DeleteFile recreated = + FileHelpers.writeDeleteFile( + table, + Files.localOutput(eqDeleteLocalFile), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(1, "a")), + table.schema()); + assertThat(recreated.location()).isEqualTo(eqDelete.location()); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Staging data file committed with DV for id=1: should see id=2 only. + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } finally { + closeJobClient(jobClient); + } + } + + private void appendConvertTask() { + appendConvertTask(STAGING_BRANCH); + } + + private void appendConvertTask(String stagingBranch) { + ConvertEqualityDeletes.builder() + .stagingBranch(stagingBranch) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1) + .sinkTo(infra.sink()); + } + + private static void assertRecords(Table table, List expected) throws IOException { + table.refresh(); + Types.StructType type = SimpleDataUtil.SCHEMA.asStruct(); + + StructLikeSet expectedSet = StructLikeSet.create(type); + expectedSet.addAll(expected); + + try (CloseableIterable iterable = + IcebergGenerics.read(table) + .useSnapshot(table.currentSnapshot().snapshotId()) + .project(SimpleDataUtil.SCHEMA) + .build()) { + StructLikeSet actualSet = StructLikeSet.create(type); + for (Record record : iterable) { + actualSet.add(record); + } + + assertThat(actualSet).isEqualTo(expectedSet); + } + } + + private static void assertNoEqualityDeletesOnMain(Table table, long expectedEqDeleteCount) { + long mainEqDeleteCount = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile f : reader) { + if (f.content() == FileContent.EQUALITY_DELETES) { + mainEqDeleteCount++; + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + assertThat(mainEqDeleteCount).isEqualTo(expectedEqDeleteCount); + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java new file mode 100644 index 000000000000..f6d1fd9464e2 --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.util.ContentFileUtil; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * End-to-end test for {@link ConvertEqualityDeletes} wired through the {@link TableMaintenance} + * framework. Verifies that the converter actually runs and commits a DV when the framework triggers + * it, exercising the full operator graph including the framework's monitor source, trigger manager, + * and lock remover. + */ +class TestConvertEqualityDeletesE2E extends OperatorTestBase { + private static final String STAGING_BRANCH = "staging"; + + @TempDir private Path tempDir; + private StreamExecutionEnvironment env; + + @BeforeEach + public void beforeEach() { + this.env = StreamExecutionEnvironment.getExecutionEnvironment(); + } + + @ParameterizedTest + @ValueSource(strings = {STAGING_BRANCH, SnapshotRef.MAIN_BRANCH}) + void testConvertEqualityDeletesE2E(String stagingBranch) throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // When staging is a separate branch, fork it from main first. + if (!stagingBranch.equals(SnapshotRef.MAIN_BRANCH)) { + table.manageSnapshots().createBranch(stagingBranch).commit(); + table.refresh(); + } + + // Commit a new data file + eq delete to staging. This pre-job snapshot exercises both the + // "new data file" and "eq delete" paths in one cycle. + DataFile newData = writeDataFile(table, 3, "c"); + DeleteFile firstDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newData).addDeletes(firstDelete).toBranch(stagingBranch).commit(); + table.refresh(); + assertThat(dvCountOnMain(table)).isZero(); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("ConvertEqualityDeletesE2EUID-" + stagingBranch) + .rateLimit(Duration.ofMillis(50)) + .lockCheckDelay(Duration.ofMillis(50)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnInterval(Duration.ofMillis(100)) + .stagingBranch(stagingBranch) + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Cycle 1: row 1 deleted by the converted DV; row 3 added on staging and committed to main. + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(1)); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: commit a second staging snapshot while the job is still running. The framework's + // next interval-trigger should pick it up and produce a second DV against the data file + // holding id=2. + table.refresh(); + DeleteFile secondDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(secondDelete).toBranch(stagingBranch).commit(); + + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(2)); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + private DataFile writeDataFile(Table table, Integer id, String data) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(SimpleDataUtil.createRecord(id, data))); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(SimpleDataUtil.createRecord(id, data)), + table.schema()); + } + + private static long dvCountOnMain(Table table) throws IOException { + table.refresh(); + if (table.currentSnapshot() == null) { + return 0; + } + + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (ContentFileUtil.isDV(file)) { + count++; + } + } + } + } + + return count; + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java index fe8457167a1f..f786f1cdb29d 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java @@ -24,8 +24,11 @@ import java.time.Duration; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -116,4 +119,34 @@ void testE2eUseCoordinator() throws Exception { closeJobClient(jobClient); } } + + @Test + void testE2eConvertEqualityDeletes() throws Exception { + // Converter requires V3 (DV support); replace the V2 table created in @BeforeEach. + dropTable(); + createTable(3, FileFormat.PARQUET); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("E2eConvertEqualityDeletesUID") + .rateLimit(Duration.ofMinutes(10)) + .lockCheckDelay(Duration.ofSeconds(10)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnEqDeleteFileCount(1) + .stagingBranch("staging") + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Just make sure that we are able to instantiate the flow + assertThat(jobClient).isNotNull(); + } finally { + closeJobClient(jobClient); + } + } } diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java index 82860a0de881..1ee5c5d0d5fd 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -110,6 +110,33 @@ void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { } } + @Test + void failsWhenStagingEqDeleteSpecPartitionsByNonEqualityColumn() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeIdOnlyPartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + harness.open(); + sendTrigger(harness); + + List> errOutput = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + + assertThat(errOutput).hasSize(1); + assertThat(errOutput.get(0).getValue()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Partition columns must be a subset of the equality fields."); + } + } + @Test void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { // When stagingBranch == targetBranch, the writer commits new data files directly to main. @@ -1063,6 +1090,19 @@ private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOExcep idOnly); } + private DeleteFile writeIdOnlyPartitionedEqualityDelete(Table table, int id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, Files.localOutput(file), partition, Lists.newArrayList(record), idOnly); + } + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) throws IOException { File file = File.createTempFile("junit", null, tempDir.toFile()); From 4af3ebee5d12a964a9c19721cc5ffebc219a44fc Mon Sep 17 00:00:00 2001 From: Sejal Gupta Date: Sat, 27 Jun 2026 01:35:18 +0530 Subject: [PATCH 49/73] Build: Support toggling log levels in iceberg-rest-fixture Docker image (#16725) * Docker: Add configuration to toggle info logs flooding in iceberg-rest-fixture * Docker: Support REST fixture logging config Generated-by: GPT-5 --------- Co-authored-by: Kevin Liu --- docker/iceberg-rest-fixture/Dockerfile | 5 ++-- docker/iceberg-rest-fixture/README.md | 26 +++++++++++++++-- docker/iceberg-rest-fixture/entrypoint.sh | 35 +++++++++++++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 docker/iceberg-rest-fixture/entrypoint.sh diff --git a/docker/iceberg-rest-fixture/Dockerfile b/docker/iceberg-rest-fixture/Dockerfile index 5629c959bd5a..1f3fc881a1b2 100644 --- a/docker/iceberg-rest-fixture/Dockerfile +++ b/docker/iceberg-rest-fixture/Dockerfile @@ -30,8 +30,9 @@ RUN set -xeu && \ # Working directory for the application WORKDIR /usr/lib/iceberg-rest -# Copy the JAR file directly to the target location +# Copy runtime files directly to the target location COPY --chown=iceberg:iceberg open-api/build/libs/iceberg-open-api-test-fixtures-runtime-*.jar /usr/lib/iceberg-rest/iceberg-rest-adapter.jar +COPY --chown=iceberg:iceberg docker/iceberg-rest-fixture/entrypoint.sh /usr/lib/iceberg-rest/entrypoint.sh ENV CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog ENV CATALOG_URI=jdbc:sqlite:/tmp/iceberg_catalog.db @@ -47,4 +48,4 @@ HEALTHCHECK --retries=10 --interval=1s \ EXPOSE $REST_PORT USER iceberg:iceberg ENV LANG=en_US.UTF-8 -CMD ["java", "-jar", "iceberg-rest-adapter.jar"] +CMD ["sh", "/usr/lib/iceberg-rest/entrypoint.sh"] diff --git a/docker/iceberg-rest-fixture/README.md b/docker/iceberg-rest-fixture/README.md index 5e02a2b4712a..928c37fe38dd 100644 --- a/docker/iceberg-rest-fixture/README.md +++ b/docker/iceberg-rest-fixture/README.md @@ -56,6 +56,30 @@ docker run -e CATALOG_CATALOG_NAME=mycatalog -p 8181:8181 apache/iceberg-rest-fi ``` +### Logging + +By default, the fixture logs at `INFO`. To reduce log output, set `LOG_LEVEL` +to another slf4j-simple level such as `WARN`, `ERROR`, or `OFF`: + +```bash +docker run -e LOG_LEVEL=WARN -p 8181:8181 apache/iceberg-rest-fixture +``` + +To use a full slf4j-simple properties file, mount a directory containing +`simplelogger.properties` and set `LOG_CONFIG_DIR` to that directory: + +```bash +docker run \ + -v "$PWD/simplelogger.properties:/etc/iceberg-rest/simplelogger.properties:ro" \ + -e LOG_CONFIG_DIR=/etc/iceberg-rest \ + -p 8181:8181 \ + apache/iceberg-rest-fixture +``` + +If both `LOG_LEVEL` and `LOG_CONFIG_DIR` are set, `LOG_LEVEL` overrides the +default log level from `simplelogger.properties`; the properties file can still +configure other slf4j-simple settings. + ## Build the Docker Image When making changes to the local files and test them out, you can build the image locally: @@ -115,5 +139,3 @@ Snapshots Snapshots Properties write.object-storage.enabled true write.object-storage.path s3://iceberg-test-data/tpc/tpc-ds/3.2.0/1000/iceberg/customer/data ``` - - diff --git a/docker/iceberg-rest-fixture/entrypoint.sh b/docker/iceberg-rest-fixture/entrypoint.sh new file mode 100644 index 000000000000..052bd7b7229b --- /dev/null +++ b/docker/iceberg-rest-fixture/entrypoint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -eu + +CLASSPATH="iceberg-rest-adapter.jar" + +if [ -n "${LOG_CONFIG_DIR:-}" ]; then + CLASSPATH="${LOG_CONFIG_DIR}:${CLASSPATH}" +fi + +set -- java + +if [ -n "${LOG_LEVEL:-}" ]; then + set -- "$@" "-Dorg.slf4j.simpleLogger.defaultLogLevel=${LOG_LEVEL}" +fi + +exec "$@" -cp "$CLASSPATH" org.apache.iceberg.rest.RESTCatalogServer From 8e46f49123f81fc5d8c87dbbdaa6eb5ea103cb86 Mon Sep 17 00:00:00 2001 From: Xin Huang <42597328+huan233usc@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:43:56 -0700 Subject: [PATCH 50/73] API, Parquet: Map geometry and geography to Parquet logical types (#16765) --- .../java/org/apache/iceberg/types/Types.java | 64 ++++++++------ .../iceberg/TestPartitionSpecValidation.java | 8 +- .../org/apache/iceberg/types/TestTypes.java | 45 +++++++++- .../data/parquet/BaseParquetWriter.java | 16 ++++ .../iceberg/parquet/MessageTypeToType.java | 19 ++++ .../iceberg/parquet/TypeToMessageType.java | 26 ++++++ .../parquet/TestParquetDataWriter.java | 34 +++++++ .../parquet/TestParquetSchemaUtil.java | 88 +++++++++++++++++++ 8 files changed, 269 insertions(+), 31 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index ec6076b04fa0..f082915920ea 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -59,8 +59,8 @@ private Types() {} .put(BinaryType.get().toString(), BinaryType.get()) .put(UnknownType.get().toString(), UnknownType.get()) .put(VariantType.get().toString(), VariantType.get()) - .put(GeometryType.crs84().toString(), GeometryType.crs84()) - .put(GeographyType.crs84().toString(), GeographyType.crs84()) + .put(GeometryType.NAME, GeometryType.crs84()) + .put(GeographyType.NAME, GeographyType.crs84()) .buildOrThrow(); private static final Pattern FIXED = Pattern.compile("fixed\\[\\s*(\\d+)\\s*\\]"); @@ -570,8 +570,10 @@ public int hashCode() { } } + /** A geometry type, optionally parameterized by a CRS. The default CRS is {@code OGC:CRS84}. */ public static class GeometryType extends PrimitiveType { public static final String DEFAULT_CRS = "OGC:CRS84"; + private static final String NAME = "geometry"; public static GeometryType crs84() { return new GeometryType(); @@ -584,12 +586,14 @@ public static GeometryType of(String crs) { private final String crs; private GeometryType() { - crs = null; + this(null); } private GeometryType(String crs) { Preconditions.checkArgument(crs == null || !crs.isEmpty(), "Invalid CRS: (empty string)"); - this.crs = DEFAULT_CRS.equalsIgnoreCase(crs) ? null : crs; + // an omitted CRS canonicalizes to the default; a provided value is kept as-is (case + // preserved) + this.crs = crs == null ? DEFAULT_CRS : crs; } @Override @@ -598,9 +602,13 @@ public TypeID typeId() { } public String crs() { - return crs != null ? crs : DEFAULT_CRS; + return crs; } + /** + * Two geometry types are equal when their CRS match case-insensitively, so {@code OGC:CRS84} + * and {@code ogc:crs84} are equal. + */ @Override public boolean equals(Object o) { if (this == o) { @@ -610,27 +618,29 @@ public boolean equals(Object o) { } GeometryType that = (GeometryType) o; - return Objects.equals(crs, that.crs); + return crs.equalsIgnoreCase(that.crs); } @Override public int hashCode() { - return Objects.hash(GeometryType.class, crs); + // hash the upper-cased CRS so it stays consistent with the case-insensitive equals + return Objects.hash(GeometryType.class, crs.toUpperCase(Locale.ROOT)); } @Override public String toString() { - if (crs == null) { - return "geometry"; - } - - return String.format("geometry(%s)", crs); + return String.format("%s(%s)", NAME, crs()); } } + /** + * A geography type, optionally parameterized by a CRS and an edge-interpolation algorithm. The + * default CRS is {@code OGC:CRS84} and the default algorithm is {@code spherical}. + */ public static class GeographyType extends PrimitiveType { public static final String DEFAULT_CRS = "OGC:CRS84"; public static final EdgeAlgorithm DEFAULT_ALGORITHM = EdgeAlgorithm.SPHERICAL; + private static final String NAME = "geography"; public static GeographyType crs84() { return new GeographyType(); @@ -648,14 +658,15 @@ public static GeographyType of(String crs, EdgeAlgorithm algorithm) { private final EdgeAlgorithm algorithm; private GeographyType() { - this.crs = null; - this.algorithm = null; + this(null, null); } private GeographyType(String crs, EdgeAlgorithm algorithm) { Preconditions.checkArgument(crs == null || !crs.isEmpty(), "Invalid CRS: (empty string)"); - this.crs = DEFAULT_CRS.equalsIgnoreCase(crs) ? null : crs; - this.algorithm = algorithm; + // an omitted CRS/algorithm canonicalizes to the default; a provided CRS is kept as-is (case + // preserved) + this.crs = crs == null ? DEFAULT_CRS : crs; + this.algorithm = algorithm == null ? DEFAULT_ALGORITHM : algorithm; } @Override @@ -664,13 +675,17 @@ public TypeID typeId() { } public String crs() { - return crs != null ? crs : DEFAULT_CRS; + return crs; } public EdgeAlgorithm algorithm() { - return algorithm != null ? algorithm : DEFAULT_ALGORITHM; + return algorithm; } + /** + * Two geography types are equal when their edge algorithms are equal and their CRS match + * case-insensitively, so {@code OGC:CRS84} and {@code ogc:crs84} are equal. + */ @Override public boolean equals(Object o) { if (this == o) { @@ -680,23 +695,18 @@ public boolean equals(Object o) { } GeographyType that = (GeographyType) o; - return Objects.equals(crs, that.crs) && Objects.equals(algorithm, that.algorithm); + return crs.equalsIgnoreCase(that.crs) && Objects.equals(algorithm, that.algorithm); } @Override public int hashCode() { - return Objects.hash(GeographyType.class, crs, algorithm); + // hash the upper-cased CRS so it stays consistent with the case-insensitive equals + return Objects.hash(GeographyType.class, crs.toUpperCase(Locale.ROOT), algorithm); } @Override public String toString() { - if (algorithm != null) { - return String.format("geography(%s, %s)", crs != null ? crs : DEFAULT_CRS, algorithm); - } else if (crs != null) { - return String.format("geography(%s)", crs); - } else { - return "geography"; - } + return String.format("%s(%s, %s)", NAME, crs(), algorithm()); } } diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java index a1709d2a2e06..5c37ad527291 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java @@ -351,8 +351,12 @@ public void testUnsupported(int fieldId, String partitionName, String expectedEr private static Object[][] unsupportedFieldsProvider() { return new Object[][] { {7, "variant_partition1", "Cannot partition by non-primitive source field: variant"}, - {8, "geom_partition1", "Invalid source type geometry for transform: bucket[5]"}, - {9, "geog_partition1", "Invalid source type geography for transform: bucket[5]"}, + {8, "geom_partition1", "Invalid source type geometry(OGC:CRS84) for transform: bucket[5]"}, + { + 9, + "geog_partition1", + "Invalid source type geography(OGC:CRS84, spherical) for transform: bucket[5]" + }, {10, "unknown_partition1", "Invalid source type unknown for transform: bucket[5]"} }; } diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypes.java b/api/src/test/java/org/apache/iceberg/types/TestTypes.java index fa5ed4304d3c..2fb224aefb15 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypes.java @@ -165,13 +165,54 @@ public void fromPrimitiveString() { @Test public void testGeospatialTypeToString() { - assertThat(Types.GeometryType.crs84().toString()).isEqualTo("geometry"); + assertThat(Types.GeometryType.crs84().toString()).isEqualTo("geometry(OGC:CRS84)"); + assertThat(Types.GeometryType.of(Types.GeometryType.DEFAULT_CRS).toString()) + .isEqualTo("geometry(OGC:CRS84)"); assertThat(Types.GeometryType.of("srid:4326").toString()).isEqualTo("geometry(srid:4326)"); - assertThat(Types.GeographyType.crs84().toString()).isEqualTo("geography"); + assertThat(Types.GeographyType.crs84().toString()).isEqualTo("geography(OGC:CRS84, spherical)"); assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.KARNEY).toString()) .isEqualTo("geography(srid:4326, karney)"); assertThat(Types.GeographyType.of(null, EdgeAlgorithm.KARNEY).toString()) .isEqualTo("geography(OGC:CRS84, karney)"); + assertThat( + Types.GeographyType.of(Types.GeographyType.DEFAULT_CRS, EdgeAlgorithm.SPHERICAL) + .toString()) + .isEqualTo("geography(OGC:CRS84, spherical)"); + assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.SPHERICAL).toString()) + .isEqualTo("geography(srid:4326, spherical)"); + + // the CRS keeps its original casing in toString even though comparison is case-insensitive + assertThat(Types.GeometryType.of("ogc:crs84").toString()).isEqualTo("geometry(ogc:crs84)"); + assertThat(Types.GeographyType.of("ogc:crs84").toString()) + .isEqualTo("geography(ogc:crs84, spherical)"); + } + + @Test + public void testGeospatialTypeDefaultNormalization() { + // an omitted default and an explicit default (exact CRS / spherical algorithm) are equal + assertThat(Types.GeometryType.of(Types.GeometryType.DEFAULT_CRS)) + .isEqualTo(Types.GeometryType.crs84()); + assertThat(Types.GeographyType.of(Types.GeographyType.DEFAULT_CRS)) + .isEqualTo(Types.GeographyType.crs84()); + assertThat(Types.GeographyType.of(Types.GeographyType.DEFAULT_CRS, EdgeAlgorithm.SPHERICAL)) + .isEqualTo(Types.GeographyType.crs84()) + .hasSameHashCodeAs(Types.GeographyType.crs84()); + assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.SPHERICAL)) + .isEqualTo(Types.GeographyType.of("srid:4326")) + .hasSameHashCodeAs(Types.GeographyType.of("srid:4326")); + assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.SPHERICAL).algorithm()) + .isEqualTo(EdgeAlgorithm.SPHERICAL); + assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.KARNEY)) + .isNotEqualTo(Types.GeographyType.of("srid:4326")); + assertThat(Types.GeographyType.of("srid:4326")).isNotEqualTo(Types.GeographyType.crs84()); + + // CRS comparison is case-insensitive: any casing of a CRS is equal and hashes the same + assertThat(Types.GeometryType.of("ogc:crs84")) + .isEqualTo(Types.GeometryType.crs84()) + .hasSameHashCodeAs(Types.GeometryType.crs84()); + assertThat(Types.GeographyType.of("ogc:crs84")) + .isEqualTo(Types.GeographyType.crs84()) + .hasSameHashCodeAs(Types.GeographyType.crs84()); } @Test diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java index 2a986fc62d00..1f3c7ab31f1c 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java @@ -265,6 +265,22 @@ public Optional> visit( return Optional.of(ParquetValueWriters.byteBuffers(desc)); } + @Override + public Optional> visit( + LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryType) { + // reject geometry so it does not silently fall through to the generic binary writer; the + // geospatial value path is a separate follow-up + throw new UnsupportedOperationException("Cannot write geometry value to Parquet"); + } + + @Override + public Optional> visit( + LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyType) { + // reject geography so it does not silently fall through to the generic binary writer; the + // geospatial value path is a separate follow-up + throw new UnsupportedOperationException("Cannot write geography value to Parquet"); + } + @Override public Optional> visit( LogicalTypeAnnotation.UUIDLogicalTypeAnnotation uuidLogicalType) { diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java b/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java index 98023bafcb8f..2b01bf882e75 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java @@ -29,9 +29,11 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.TimestampType; +import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; @@ -254,6 +256,23 @@ public Optional visit(LogicalTypeAnnotation.JsonLogicalTypeAnnotation json public Optional visit(LogicalTypeAnnotation.BsonLogicalTypeAnnotation bsonType) { return Optional.of(Types.BinaryType.get()); } + + @Override + public Optional visit(LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryType) { + // a null crs resolves to the Iceberg default in GeometryType.of + return Optional.of(Types.GeometryType.of(geometryType.getCrs())); + } + + @Override + public Optional visit( + LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyType) { + // a null crs / algorithm resolves to the Iceberg default in GeographyType.of + EdgeInterpolationAlgorithm algorithm = geographyType.getAlgorithm(); + return Optional.of( + Types.GeographyType.of( + geographyType.getCrs(), + algorithm != null ? EdgeAlgorithm.fromName(algorithm.name()) : null)); + } } private void addAlias(String name, int fieldId) { diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/TypeToMessageType.java b/parquet/src/main/java/org/apache/iceberg/parquet/TypeToMessageType.java index d648cbf0694b..f05001f5f43d 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/TypeToMessageType.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/TypeToMessageType.java @@ -30,12 +30,15 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.avro.AvroSchemaUtil; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type.NestedType; import org.apache.iceberg.types.Type.PrimitiveType; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types.DecimalType; import org.apache.iceberg.types.Types.FixedType; +import org.apache.iceberg.types.Types.GeographyType; +import org.apache.iceberg.types.Types.GeometryType; import org.apache.iceberg.types.Types.ListType; import org.apache.iceberg.types.Types.MapType; import org.apache.iceberg.types.Types.NestedField; @@ -43,6 +46,7 @@ import org.apache.iceberg.types.Types.TimestampNanoType; import org.apache.iceberg.types.Types.TimestampType; import org.apache.iceberg.variants.Variant; +import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit; @@ -280,11 +284,33 @@ public Type primitive( .id(id) .named(name); + case GEOMETRY: + GeometryType geometry = (GeometryType) primitive; + return Types.primitive(BINARY, repetition) + .as(LogicalTypeAnnotation.geometryType(geometry.crs())) + .id(id) + .named(name); + + case GEOGRAPHY: + GeographyType geography = (GeographyType) primitive; + return Types.primitive(BINARY, repetition) + .as( + LogicalTypeAnnotation.geographyType( + geography.crs(), toParquet(geography.algorithm()))) + .id(id) + .named(name); + default: throw new UnsupportedOperationException("Unsupported type for Parquet: " + primitive); } } + private static EdgeInterpolationAlgorithm toParquet(EdgeAlgorithm algorithm) { + // Iceberg and Parquet use the same algorithm names (SPHERICAL, VINCENTY, THOMAS, ANDOYER, + // KARNEY) so the algorithm is mapped by name + return EdgeInterpolationAlgorithm.valueOf(algorithm.name()); + } + private static LogicalTypeAnnotation decimalAnnotation(int precision, int scale) { return LogicalTypeAnnotation.decimalType(scale, precision); } diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java index 19f0b9129857..0c17a871677b 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java @@ -20,6 +20,7 @@ import static org.apache.iceberg.parquet.ParquetWritingTestUtils.createTempFile; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.IOException; import java.nio.ByteBuffer; @@ -102,6 +103,39 @@ public void testDataWriter() throws IOException { testDataWriter(SCHEMA, (id, name) -> null); } + @Test + public void testGeospatialWriteIsRejected() { + Schema geometrySchema = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "geom", Types.GeometryType.crs84())); + assertThatThrownBy( + () -> + Parquet.writeData(Files.localOutput(createTempFile(temp))) + .schema(geometrySchema) + .createWriterFunc(GenericParquetWriter::create) + .overwrite() + .withSpec(PartitionSpec.unpartitioned()) + .build()) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot write geometry value to Parquet"); + + Schema geographySchema = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "geog", Types.GeographyType.crs84())); + assertThatThrownBy( + () -> + Parquet.writeData(Files.localOutput(createTempFile(temp))) + .schema(geographySchema) + .createWriterFunc(GenericParquetWriter::create) + .overwrite() + .withSpec(PartitionSpec.unpartitioned()) + .build()) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot write geography value to Parquet"); + } + private void testDataWriter(Schema schema, VariantShreddingFunction variantShreddingFunc) throws IOException { OutputFile file = Files.localOutput(createTempFile(temp)); diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetSchemaUtil.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetSchemaUtil.java index 51bfc1e811f4..6acf5eda0e86 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetSchemaUtil.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetSchemaUtil.java @@ -22,13 +22,17 @@ import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.iceberg.Schema; import org.apache.iceberg.mapping.MappingUtil; import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.variants.Variant; +import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; @@ -338,6 +342,90 @@ public void testVariantTypesWithoutAssigningIds() { .isEqualTo(expectedSchema.asStruct()); } + @Test + public void testGeospatialTypeRoundTrip() { + List fields = + Lists.newArrayList( + required(1, "geom_default", Types.GeometryType.crs84()), + optional(2, "geom_3857", Types.GeometryType.of("EPSG:3857")), + required(3, "geog_default", Types.GeographyType.crs84())); + // cover the by-name algorithm mapping for every edge algorithm in both directions + int nextId = fields.size() + 1; + for (EdgeAlgorithm algorithm : EdgeAlgorithm.values()) { + fields.add( + optional(nextId++, "geog_" + algorithm, Types.GeographyType.of("EPSG:4326", algorithm))); + } + + Schema schema = new Schema(fields); + MessageType messageType = ParquetSchemaUtil.convert(schema, "geo_table"); + Schema actualSchema = ParquetSchemaUtil.convert(messageType); + assertThat(actualSchema.asStruct()) + .as("Schema must round-trip through Parquet geometry/geography logical types") + .isEqualTo(schema.asStruct()); + } + + @Test + public void testGeospatialAnnotationsWithOmittedParameters() { + // unset CRS and algorithm parameters must map to Iceberg's defaults, and explicit values + // (including explicit default values) must be preserved + MessageType messageType = + org.apache.parquet.schema.Types.buildMessage() + .required(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType(null)) + .id(1) + .named("geom_bare") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("OGC:CRS84")) + .id(2) + .named("geom_explicit_default") + .required(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geographyType(null, null)) + .id(3) + .named("geog_bare") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geographyType("EPSG:4326", null)) + .id(4) + .named("geog_crs_only") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geographyType(null, EdgeInterpolationAlgorithm.ANDOYER)) + .id(5) + .named("geog_algorithm_only") + .optional(PrimitiveTypeName.BINARY) + .as( + LogicalTypeAnnotation.geographyType( + "EPSG:4326", EdgeInterpolationAlgorithm.SPHERICAL)) + .id(6) + .named("geog_explicit_spherical") + .named("geo_table"); + + Schema expectedSchema = + new Schema( + required(1, "geom_bare", Types.GeometryType.crs84()), + optional(2, "geom_explicit_default", Types.GeometryType.crs84()), + required(3, "geog_bare", Types.GeographyType.crs84()), + optional(4, "geog_crs_only", Types.GeographyType.of("EPSG:4326")), + optional( + 5, + "geog_algorithm_only", + Types.GeographyType.of(Types.GeographyType.DEFAULT_CRS, EdgeAlgorithm.ANDOYER)), + optional( + 6, + "geog_explicit_spherical", + Types.GeographyType.of("EPSG:4326", EdgeAlgorithm.SPHERICAL))); + + Schema actualSchema = ParquetSchemaUtil.convert(messageType); + assertThat(actualSchema.asStruct()) + .as("Geometry and geography annotations must convert to the expected Iceberg types") + .isEqualTo(expectedSchema.asStruct()); + assertThat(actualSchema.findType("geom_bare").toString()).isEqualTo("geometry(OGC:CRS84)"); + assertThat(actualSchema.findType("geom_explicit_default").toString()) + .isEqualTo("geometry(OGC:CRS84)"); + assertThat(actualSchema.findType("geog_bare").toString()) + .isEqualTo("geography(OGC:CRS84, spherical)"); + assertThat(actualSchema.findType("geog_explicit_spherical").toString()) + .isEqualTo("geography(EPSG:4326, spherical)"); + } + @Test public void testSchemaConversionForHiveStyleLists() { String parquetSchemaString = From 874e4096e5d16fddbbf43b91f20e248616232cc3 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Sat, 27 Jun 2026 16:50:32 -0400 Subject: [PATCH 51/73] Core, Spark: Ensure correct delete file sizes in rewrite table action (#15470) --- .../apache/iceberg/RewriteTablePathUtil.java | 120 ++++++- .../iceberg/TestRewriteTablePathUtil.java | 97 +++++- .../actions/RewriteTablePathSparkAction.java | 195 ++++++++--- .../actions/TestRewriteTablePathsAction.java | 304 +++++++++++++++++- .../actions/RewriteTablePathSparkAction.java | 195 ++++++++--- .../actions/TestRewriteTablePathsAction.java | 304 +++++++++++++++++- .../actions/RewriteTablePathSparkAction.java | 195 ++++++++--- .../actions/TestRewriteTablePathsAction.java | 304 +++++++++++++++++- 8 files changed, 1523 insertions(+), 191 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java index 69f82931833e..e72dbef6dbcc 100644 --- a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java +++ b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java @@ -47,6 +47,7 @@ import org.apache.iceberg.puffin.PuffinReader; import org.apache.iceberg.puffin.PuffinWriter; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; @@ -377,7 +378,12 @@ public static RewriteResult rewriteDataManifest( * @param stagingLocation staging location for rewritten files (referred delete file will be * rewritten here) * @return a copy plan of content files in the manifest that was rewritten + * @deprecated since 1.12.0, will be removed in 1.13.0; use the overload that accepts the map of + * rewritten position delete file sizes. This overload records the original {@code + * file_size_in_bytes}, which can be inconsistent with the rewritten file size on disk once + * embedded data file paths change length. */ + @Deprecated public static RewriteResult rewriteDeleteManifest( ManifestFile manifestFile, Set snapshotIds, @@ -389,6 +395,52 @@ public static RewriteResult rewriteDeleteManifest( String targetPrefix, String stagingLocation) throws IOException { + return rewriteDeleteManifest( + manifestFile, + snapshotIds, + outputFile, + io, + format, + specsById, + sourcePrefix, + targetPrefix, + stagingLocation, + ImmutableMap.of()); + } + + /** + * Rewrite a delete manifest, replacing path references. + * + *

This is a metadata-only operation: position delete file content is rewritten separately (see + * {@link #rewritePositionDelete}). The actual sizes of those rewritten files are supplied via + * {@code rewrittenDeleteFileSizes} and recorded in the manifest so that {@code + * file_size_in_bytes} stays consistent with the rewritten file on disk. + * + * @param manifestFile source delete manifest to rewrite + * @param snapshotIds snapshot ids for filtering returned delete manifest entries + * @param outputFile output file to rewrite manifest file to + * @param io file io + * @param format format of the manifest file + * @param specsById map of partition specs by id + * @param sourcePrefix source prefix that will be replaced + * @param targetPrefix target prefix that will replace it + * @param stagingLocation staging location for rewritten position delete files + * @param rewrittenDeleteFileSizes map from source position delete file path to the actual size of + * the rewritten file; entries absent from the map keep their original size + * @return a copy plan of content files in the manifest that was rewritten + */ + public static RewriteResult rewriteDeleteManifest( + ManifestFile manifestFile, + Set snapshotIds, + OutputFile outputFile, + FileIO io, + int format, + Map specsById, + String sourcePrefix, + String targetPrefix, + String stagingLocation, + Map rewrittenDeleteFileSizes) + throws IOException { PartitionSpec spec = specsById.get(manifestFile.partitionSpecId()); try (ManifestWriter writer = ManifestFiles.writeDeleteManifest(format, spec, outputFile, manifestFile.snapshotId()); @@ -405,7 +457,8 @@ public static RewriteResult rewriteDeleteManifest( sourcePrefix, targetPrefix, stagingLocation, - writer)) + writer, + rewrittenDeleteFileSizes)) .reduce(new RewriteResult<>(), RewriteResult::append); } } @@ -445,14 +498,20 @@ private static RewriteResult writeDeleteFileEntry( String sourcePrefix, String targetPrefix, String stagingLocation, - ManifestWriter writer) { + ManifestWriter writer, + Map rewrittenDeleteFileSizes) { DeleteFile file = entry.file(); RewriteResult result = new RewriteResult<>(); switch (file.content()) { case POSITION_DELETES: - DeleteFile posDeleteFile = newPositionDeleteEntry(file, spec, sourcePrefix, targetPrefix); + // Path rewrites change the file size; use the measured size, falling back to the original + // for entries that were not rewritten (e.g. deleted entries not copied to the target). + long fileSizeInBytes = + rewrittenDeleteFileSizes.getOrDefault(file.location(), file.fileSizeInBytes()); + DeleteFile posDeleteFile = + newPositionDeleteEntry(file, spec, sourcePrefix, targetPrefix, fileSizeInBytes); appendEntryWithFile(entry, writer, posDeleteFile); // keep the following entries in metadata but exclude them from copyPlan // 1) deleted position delete files @@ -523,7 +582,11 @@ private static DeleteFile newEqualityDeleteEntry( } private static DeleteFile newPositionDeleteEntry( - DeleteFile file, PartitionSpec spec, String sourcePrefix, String targetPrefix) { + DeleteFile file, + PartitionSpec spec, + String sourcePrefix, + String targetPrefix, + long fileSizeInBytes) { String path = file.location(); Preconditions.checkArgument( path.startsWith(sourcePrefix), @@ -535,6 +598,7 @@ private static DeleteFile newPositionDeleteEntry( FileMetadata.deleteFileBuilder(spec) .copy(file) .withPath(newPath(path, sourcePrefix, targetPrefix)) + .withFileSizeInBytes(fileSizeInBytes) .withMetrics(ContentFileUtil.replacePathBounds(file, sourcePrefix, targetPrefix)); // Update referencedDataFile for DV files @@ -607,7 +671,11 @@ PositionDeleteWriter writer( * @param sourcePrefix source prefix that will be replaced * @param targetPrefix target prefix to replace it * @param posDeleteReaderWriter class to read and write position delete files + * @deprecated since 1.12.0, will be removed in 1.13.0; use {@link #rewritePositionDelete} which + * returns the size of the rewritten file so callers can record an accurate {@code + * file_size_in_bytes}. */ + @Deprecated public static void rewritePositionDeleteFile( DeleteFile deleteFile, OutputFile outputFile, @@ -617,6 +685,37 @@ public static void rewritePositionDeleteFile( String targetPrefix, PositionDeleteReaderWriter posDeleteReaderWriter) throws IOException { + rewritePositionDelete( + deleteFile, outputFile, io, spec, sourcePrefix, targetPrefix, posDeleteReaderWriter); + } + + /** + * Rewrite a position delete file, replacing path references, and return the size of the rewritten + * file. + * + *

The size is measured from the writer after it is closed (rather than via a separate {@code + * getLength()}/HEAD call), so it is accurate even on file systems where the length of an + * in-progress write underreports. Callers record this size as {@code file_size_in_bytes} in the + * rewritten manifest. + * + * @param deleteFile source position delete file to rewrite + * @param outputFile output file to write the rewritten delete file to + * @param io file io + * @param spec spec of delete file + * @param sourcePrefix source prefix that will be replaced + * @param targetPrefix target prefix to replace it + * @param posDeleteReaderWriter class to read and write position delete files + * @return the size in bytes of the rewritten file + */ + public static long rewritePositionDelete( + DeleteFile deleteFile, + OutputFile outputFile, + FileIO io, + PartitionSpec spec, + String sourcePrefix, + String targetPrefix, + PositionDeleteReaderWriter posDeleteReaderWriter) + throws IOException { String path = deleteFile.location(); if (!path.startsWith(sourcePrefix)) { throw new UnsupportedOperationException( @@ -625,8 +724,7 @@ public static void rewritePositionDeleteFile( // DV files (Puffin format for v3+) need special handling to rewrite internal blob metadata if (ContentFileUtil.isDV(deleteFile)) { - rewriteDVFile(deleteFile, outputFile, io, sourcePrefix, targetPrefix); - return; + return rewriteDVFile(deleteFile, outputFile, io, sourcePrefix, targetPrefix); } // For non-DV position delete files (v2), rewrite using the reader/writer @@ -655,9 +753,14 @@ record = recordIt.next(); writer.write(newPositionDeleteRecord(record, sourcePrefix, targetPrefix)); } } + + writer.close(); + return writer.length(); } } } + + return 0; } /** @@ -668,8 +771,9 @@ record = recordIt.next(); * @param io file io * @param sourcePrefix source prefix that will be replaced * @param targetPrefix target prefix to replace it + * @return the size in bytes of the rewritten DV file */ - private static void rewriteDVFile( + private static long rewriteDVFile( DeleteFile deleteFile, OutputFile outputFile, FileIO io, @@ -708,6 +812,8 @@ private static void rewriteDVFile( try (PuffinWriter writer = Puffin.write(outputFile).createdBy(IcebergBuild.fullVersion()).build()) { rewrittenBlobs.forEach(writer::write); + writer.close(); + return writer.length(); } } diff --git a/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java b/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java index bedd8dd66d71..1b0f5f6b1c70 100644 --- a/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java @@ -24,6 +24,9 @@ import java.io.IOException; import java.util.Set; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; @@ -277,8 +280,100 @@ public void testRewritingMultiplePositionDeleteEntriesWithinManifestFile() throw table.specs(), sourcePrefix, targetPrefix, - stagingDir); + stagingDir, + ImmutableMap.of()); assertThat(deleteFileRewriteResult.toRewrite()).hasSize(2); } + + // A position delete entry that is not rewritten (e.g. a DELETED entry not copied to the target) + // is absent from the measured-size map and must keep its original file_size_in_bytes. + @TestTemplate + public void testRewriteDeleteManifestFallsBackToOriginalSizeForDeletedEntries() + throws IOException { + assumeThat(formatVersion) + .as("Delete files only work for format version 2+") + .isGreaterThanOrEqualTo(2); + + String sourcePrefix = "/path/to/"; + String targetPrefix = "/path/new/"; + String stagingDir = "/staging/"; + + // FILE_A_DELETES is live, so its rewritten size is measured and supplied; FILE_B_DELETES is a + // DELETED entry, absent from the size map. + ManifestFile manifest = deleteManifestWithLiveAndDeletedEntry(FILE_A_DELETES, FILE_B_DELETES); + + long measuredSizeForA = 9999L; + OutputFile output = + Files.localOutput( + FileFormat.AVRO.addExtension( + temp.resolve("junit" + System.nanoTime()).toFile().toString())); + RewriteTablePathUtil.rewriteDeleteManifest( + manifest, + Set.of(1000L), + output, + table.io(), + formatVersion, + table.specs(), + sourcePrefix, + targetPrefix, + stagingDir, + ImmutableMap.of(FILE_A_DELETES.location(), measuredSizeForA)); + + InputFile rewrittenInput = output.toInputFile(); + ManifestFile rewritten = + new GenericManifestFile( + rewrittenInput.location(), + rewrittenInput.getLength(), + SPEC.specId(), + ManifestContent.DELETES, + 0L, + 0L, + 1000L, + null, + null, + null, + null, + null, + null, + null, + null, + null); + int seen = 0; + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(rewritten, table.io(), table.specs())) { + for (ManifestEntry entry : reader.entries()) { + seen++; + if (entry.status() == ManifestEntry.Status.DELETED) { + assertThat(entry.file().fileSizeInBytes()) + .as("DELETED entry should fall back to its original size") + .isEqualTo(FILE_B_DELETES.fileSizeInBytes()); + } else { + assertThat(entry.file().fileSizeInBytes()) + .as("Live entry should use the measured rewritten size") + .isEqualTo(measuredSizeForA); + } + } + } + + assertThat(seen).as("Both the live and deleted entries should be present").isEqualTo(2); + } + + private ManifestFile deleteManifestWithLiveAndDeletedEntry(DeleteFile live, DeleteFile deleted) + throws IOException { + OutputFile manifestFile = + Files.localOutput( + FileFormat.AVRO.addExtension( + temp.resolve("junit" + System.nanoTime()).toFile().toString())); + ManifestWriter writer = + ManifestFiles.writeDeleteManifest(formatVersion, SPEC, manifestFile, 1000L); + try { + writer.add(live); + writer.delete(deleted, 1, null); + } finally { + writer.close(); + } + + return writer.toManifestFile(); + } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..11935e815e76 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,9 +32,13 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.RewriteTablePathUtil; @@ -69,13 +74,14 @@ import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.JobGroupInfo; import org.apache.iceberg.spark.source.SerializableTableWithSize; import org.apache.iceberg.util.DeleteFileSet; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; -import org.apache.spark.api.java.function.ForeachFunction; +import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.broadcast.Broadcast; @@ -87,6 +93,7 @@ import org.apache.spark.sql.functions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Tuple2; public class RewriteTablePathSparkAction extends BaseSparkAction implements RewriteTablePath { @@ -272,7 +279,8 @@ private String jobDesc() { *

    *
  • Rebuild version files to staging *
  • Rebuild manifest list files to staging - *
  • Rebuild manifest to staging + *
  • Rewrite referenced position delete files to staging + *
  • Rebuild manifests to staging *
  • Get all files needed to move *
*/ @@ -308,24 +316,29 @@ private Result rebuildMetadata() { RewriteResult rewriteManifestListResult = new RewriteResult<>(); manifestListResults.forEach(rewriteManifestListResult::append); - // rebuild manifest files - Set metaFiles = rewriteManifestListResult.toRewrite(); - RewriteContentFileResult rewriteManifestResult = - rewriteManifests(deltaSnapshots, endMetadata, metaFiles); + Set manifestFiles = rewriteManifestListResult.toRewrite(); // rebuild position delete files - Set deleteFiles = - rewriteManifestResult.toRewrite().stream() - .filter(e -> e instanceof DeleteFile) - .map(e -> (DeleteFile) e) - .collect(Collectors.toCollection(DeleteFileSet::create)); - rewritePositionDeletes(deleteFiles); + Set deleteManifests = + manifestFiles.stream() + .filter(manifest -> manifest.content() == ManifestContent.DELETES) + .collect(Collectors.toSet()); + Set deleteFilesToRewrite = positionDeletesToRewrite(deleteManifests); + Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); + + // rebuild manifest files + RewriteContentFileResult rewriteManifestResult = + rewriteManifests( + deltaSnapshots, + endMetadata, + manifestFiles, + sparkContext().broadcast(rewrittenDeleteFileSizes)); ImmutableRewriteTablePath.Result.Builder builder = ImmutableRewriteTablePath.Result.builder() .stagingLocation(stagingDir) - .rewrittenDeleteFilePathsCount(deleteFiles.size()) - .rewrittenManifestFilePathsCount(metaFiles.size()) + .rewrittenDeleteFilePathsCount(deleteFilesToRewrite.size()) + .rewrittenManifestFilePathsCount(manifestFiles.size()) .latestVersion(RewriteTablePathUtil.fileName(endVersionName)); if (!createFileList) { @@ -561,7 +574,10 @@ public RewriteContentFileResult appendDeleteFile(RewriteResult r1) { /** Rewrite manifest files in a distributed manner and return rewritten data files path pairs. */ private RewriteContentFileResult rewriteManifests( - Set deltaSnapshots, TableMetadata tableMetadata, Set toRewrite) { + Set deltaSnapshots, + TableMetadata tableMetadata, + Set toRewrite, + Broadcast> rewrittenDeleteFileSizes) { if (toRewrite.isEmpty()) { return new RewriteContentFileResult(); } @@ -581,7 +597,8 @@ private RewriteContentFileResult rewriteManifests( stagingDir, tableMetadata.formatVersion(), sourcePrefix, - targetPrefix), + targetPrefix, + rewrittenDeleteFileSizes), Encoders.bean(RewriteContentFileResult.class)) // duplicates are expected here as the same data file can have different statuses // (e.g. added and deleted) @@ -594,7 +611,8 @@ private static MapFunction toManifests( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { return manifestFile -> { RewriteContentFileResult result = new RewriteContentFileResult(); @@ -619,7 +637,8 @@ private static MapFunction toManifests( stagingLocation, format, sourcePrefix, - targetPrefix)); + targetPrefix, + rewrittenDeleteFileSizes)); break; default: throw new UnsupportedOperationException( @@ -665,7 +684,8 @@ private static RewriteResult writeDeleteManifest( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { try { String stagingPath = RewriteTablePathUtil.stagingPath(manifestFile.path(), sourcePrefix, stagingLocation); @@ -682,27 +702,120 @@ private static RewriteResult writeDeleteManifest( specsById, sourcePrefix, targetPrefix, - stagingLocation); + stagingLocation, + rewrittenDeleteFileSizes.value()); } catch (IOException e) { throw new RuntimeIOException(e); } } - private void rewritePositionDeletes(Set toRewrite) { + /** + * Enumerate the distinct position delete files referenced by the given delete manifests. Deduped + * by identity (location, offset, size) so a file shared across manifests is counted once; the + * physical rewrite is further deduped by location in {@link #rewritePositionDeletes}. + */ + private Set positionDeletesToRewrite(Set deleteManifests) { + if (deleteManifests.isEmpty()) { + return Collections.emptySet(); + } + + Encoder manifestFileEncoder = Encoders.javaSerialization(ManifestFile.class); + Dataset manifestDS = + spark().createDataset(Lists.newArrayList(deleteManifests), manifestFileEncoder); + Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); + + List referencedDeleteFiles = + manifestDS + .repartition(deleteManifests.size()) + .flatMap(positionDeletesInManifest(tableBroadcast()), deleteFileEncoder) + .collectAsList(); + + return DeleteFileSet.of(referencedDeleteFiles); + } + + private static FlatMapFunction positionDeletesInManifest( + Broadcast

tableArg) { + return manifestFile -> { + Table table = tableArg.getValue(); + List deleteFiles = Lists.newArrayList(); + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifestFile, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (deleteFile.content() == FileContent.POSITION_DELETES) { + deleteFiles.add(deleteFile.copy()); + } + } + } + return deleteFiles.iterator(); + }; + } + + /** + * Rewrite the given position delete files in parallel, returning a map from each source delete + * file path to the size of its rewritten file. Physical files are deduped by location, so a + * Puffin file holding multiple DVs is rewritten once and its size keyed once. + */ + private Map rewritePositionDeletes(Set toRewrite) { if (toRewrite.isEmpty()) { - return; + return Collections.emptyMap(); + } + + // Multiple DVs can share one Puffin file at different blob offsets; rewrite each physical file + // once. The measured size is keyed by location and applied to every referencing manifest entry. + Map byLocation = Maps.newHashMapWithExpectedSize(toRewrite.size()); + for (DeleteFile deleteFile : toRewrite) { + byLocation.putIfAbsent(deleteFile.location(), deleteFile); } + List physicalFiles = Lists.newArrayList(byLocation.values()); Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); - Dataset deleteFileDs = - spark().createDataset(Lists.newArrayList(toRewrite), deleteFileEncoder); + Dataset deleteFileDS = spark().createDataset(physicalFiles, deleteFileEncoder); PositionDeleteReaderWriter posDeleteReaderWriter = new SparkPositionDeleteReaderWriter(); - deleteFileDs - .repartition(toRewrite.size()) - .foreach( - rewritePositionDelete( - tableBroadcast(), sourcePrefix, targetPrefix, stagingDir, posDeleteReaderWriter)); + List> rewrittenSizes = + deleteFileDS + .repartition(physicalFiles.size()) + .map( + rewritePositionDelete( + tableBroadcast(), + sourcePrefix, + targetPrefix, + stagingDir, + posDeleteReaderWriter), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .collectAsList(); + + Map sizesBySourcePath = Maps.newHashMapWithExpectedSize(rewrittenSizes.size()); + for (Tuple2 entry : rewrittenSizes) { + sizesBySourcePath.put(entry._1(), entry._2()); + } + return sizesBySourcePath; + } + + private static MapFunction> rewritePositionDelete( + Broadcast
tableArg, + String sourcePrefixArg, + String targetPrefixArg, + String stagingLocationArg, + PositionDeleteReaderWriter posDeleteReaderWriter) { + return deleteFile -> { + FileIO io = tableArg.getValue().io(); + String newPath = + RewriteTablePathUtil.stagingPath( + deleteFile.location(), sourcePrefixArg, stagingLocationArg); + OutputFile outputFile = io.newOutputFile(newPath); + PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); + long rewrittenLength = + RewriteTablePathUtil.rewritePositionDelete( + deleteFile, + outputFile, + io, + spec, + sourcePrefixArg, + targetPrefixArg, + posDeleteReaderWriter); + return new Tuple2<>(deleteFile.location(), rewrittenLength); + }; } private static class SparkPositionDeleteReaderWriter implements PositionDeleteReaderWriter { @@ -724,30 +837,6 @@ public PositionDeleteWriter writer( } } - private ForeachFunction rewritePositionDelete( - Broadcast
tableArg, - String sourcePrefixArg, - String targetPrefixArg, - String stagingLocationArg, - PositionDeleteReaderWriter posDeleteReaderWriter) { - return deleteFile -> { - FileIO io = tableArg.getValue().io(); - String newPath = - RewriteTablePathUtil.stagingPath( - deleteFile.location(), sourcePrefixArg, stagingLocationArg); - OutputFile outputFile = io.newOutputFile(newPath); - PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); - RewriteTablePathUtil.rewritePositionDeleteFile( - deleteFile, - outputFile, - io, - spec, - sourcePrefixArg, - targetPrefixArg, - posDeleteReaderWriter); - }; - } - private static CloseableIterable positionDeletesReader( InputFile inputFile, FileFormat format, PartitionSpec spec) { return FormatModelRegistry.readBuilder(format, Record.class, inputFile) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index c5db04762f21..9660beae187e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Predicate; @@ -41,15 +42,21 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.Parameter; import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Parameters; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; @@ -62,14 +69,18 @@ import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.DVFileWriter; import org.apache.iceberg.deletes.PositionDelete; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.SparkCatalog; import org.apache.iceberg.spark.TestBase; import org.apache.iceberg.spark.source.ThreeColumnRecord; @@ -629,12 +640,7 @@ public void testPositionDeletesDeduplication() throws Exception { // in a new manifest, which will cause duplicate DeleteFile objects when processing tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); - // This should NOT throw AlreadyExistsException - the fix uses DeleteFileSet to deduplicate - // Without the fix (using Collectors.toSet()), this would fail because: - // 1. Both manifests contain entries for the same delete file - // 2. Processing returns two different DeleteFile objects for the same file - // 3. HashSet doesn't deduplicate them (DeleteFile doesn't override equals()) - // 4. rewritePositionDeletes tries to write the same file twice -> AlreadyExistsException + // This should NOT throw AlreadyExistsException RewriteTablePath.Result result = actions() .rewriteTablePath(tableWithPosDeletes) @@ -642,13 +648,295 @@ public void testPositionDeletesDeduplication() throws Exception { .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) .execute(); - // Verify the rewrite completed successfully - should have rewritten exactly 1 delete file - // (the duplicate should be deduplicated by DeleteFileSet) assertThat(result.rewrittenDeleteFilePathsCount()) .as("Should have rewritten exactly 1 delete file after deduplication") .isEqualTo(1); } + // Regression test: when the same position delete file is referenced from manifests in different + // snapshots, it must be rewritten once and the resulting size stamped consistently into every + // manifest that references it. The delete file is enumerated and deduped by path before the + // rewrite, so its measured size is shared across all referencing delete manifests. + @TestTemplate + public void testSharedDeleteFileSizeAcrossManifests() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedDelete"), + 1, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + DataFile dataFile = + tableWithPosDeletes + .currentSnapshot() + .addedDataFiles(tableWithPosDeletes.io()) + .iterator() + .next(); + + List> deletes = Lists.newArrayList(Pair.of(dataFile.location(), 0L)); + File deleteFile = + new File( + removePrefix(tableWithPosDeletes.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(deleteFile.toURI().toString()), + deletes, + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List deleteManifests = + targetTable.currentSnapshot().deleteManifests(targetTable.io()); + assertThat(deleteManifests) + .as("Expected the shared delete file to be referenced by multiple manifests") + .hasSizeGreaterThanOrEqualTo(2); + for (ManifestFile manifest : deleteManifests) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single delete manifest can reference multiple distinct position delete + // files, and each entry must be stamped with its own rewritten size. The two delete files carry + // a different number of records so they rewrite to different sizes, which catches a per-path size + // map that collapses entries to a single size or falls back to the stale original size. + @TestTemplate + public void testMultipleDistinctDeleteFileSizesAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithDistinctDeletes"), + 2, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + List dataFiles = Lists.newArrayList(); + tableWithPosDeletes + .snapshots() + .forEach( + snapshot -> snapshot.addedDataFiles(tableWithPosDeletes.io()).forEach(dataFiles::add)); + assertThat(dataFiles).as("Expected two data files to reference from deletes").hasSize(2); + + // One delete file holds a single record, the other holds two, so they rewrite to distinct + // on-disk sizes. + File smallDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-small.parquet")); + DeleteFile smallDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(smallDeleteFile.toURI().toString()), + Lists.newArrayList(Pair.of(dataFiles.get(0).location(), 0L)), + formatVersion) + .first(); + + File largeDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-large.parquet")); + DeleteFile largeDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(largeDeleteFile.toURI().toString()), + Lists.newArrayList( + Pair.of(dataFiles.get(0).location(), 0L), + Pair.of(dataFiles.get(1).location(), 0L)), + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(smallDelete).addDeletes(largeDelete).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List rewrittenSizes = Lists.newArrayList(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + rewrittenSizes.add(manifestSize); + } + } + } + + assertThat(rewrittenSizes) + .as( + "The two distinct delete files should rewrite to distinct, independently recorded sizes") + .hasSize(2) + .doesNotHaveDuplicates(); + } + + // Regression test: rewriting delete file paths changes the file size (since the + // embedded data file paths may differ in length), but file_size_in_bytes in the rewritten + // manifest was not updated. Readers that use file_size_in_bytes to elide a stat() call may + // fail. + @TestTemplate + public void testDeleteFileSizeInBytesAfterRewrite() throws Exception { + List> deletes = + Lists.newArrayList( + Pair.of( + SnapshotChanges.builderFor(table) + .build() + .addedDataFiles() + .iterator() + .next() + .location(), + 0L)); + + File file = new File(removePrefix(table.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + table, table.io().newOutputFile(file.toURI().toString()), deletes, formatVersion) + .first(); + table.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(table) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(table.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single Puffin file can hold multiple DVs (one blob per data file) + // referenced by distinct DeleteFile entries that share the same location. The rewrite must + // rewrite + // the physical Puffin file once (rather than colliding on the staging path) and stamp the + // rewritten size into every referencing manifest entry. + @TestTemplate + public void testSharedPuffinDeleteFileSizeAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("DVs are introduced in v3; v4 writes parquet manifests the test setup cannot read") + .isEqualTo(3); + + Table tableWithDVs = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedPuffin"), 2); + + List dataFilePaths = Lists.newArrayList(); + tableWithDVs + .snapshots() + .forEach( + snapshot -> + snapshot + .addedDataFiles(tableWithDVs.io()) + .forEach(dataFile -> dataFilePaths.add(dataFile.location()))); + assertThat(dataFilePaths).as("Expected two data files to back two DVs").hasSize(2); + + List dvs = writeDVsForDataFiles(tableWithDVs, dataFilePaths); + assertThat(dvs) + .as("Both DVs should live in a single Puffin file") + .hasSize(2) + .allSatisfy(dv -> assertThat(dv.location()).isEqualTo(dvs.get(0).location())); + + RowDelta rowDelta = tableWithDVs.newRowDelta(); + dvs.forEach(rowDelta::addDeletes); + rowDelta.commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithDVs) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithDVs.location(), targetTableLocation()) + .execute(); + assertThat(result.rewrittenDeleteFilePathsCount()) + .as("Two DVs are two delete files with rewritten paths") + .isEqualTo(2); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + Set rewrittenLocations = Sets.newHashSet(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + rewrittenLocations.add(df.location()); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(df.fileSizeInBytes()) + .as("file_size_in_bytes should match the rewritten Puffin size for %s", df.location()) + .isEqualTo(actualSize); + } + } + } + assertThat(rewrittenLocations) + .as("Both DVs should point at the single rewritten Puffin file") + .hasSize(1); + } + + // Writes one DV per data file path into a single Puffin file, returning the resulting DeleteFiles + // (which share a location but carry distinct blob offsets). + private List writeDVsForDataFiles(Table targetTable, List dataFilePaths) + throws IOException { + OutputFileFactory fileFactory = + OutputFileFactory.builderFor(targetTable, 1, 1).format(FileFormat.PUFFIN).build(); + DVFileWriter writer = new BaseDVFileWriter(fileFactory, p -> null); + try (DVFileWriter closeableWriter = writer) { + for (String path : dataFilePaths) { + closeableWriter.delete(path, 0L, targetTable.spec(), (StructLike) null); + } + } + + return writer.result().deleteFiles(); + } + @TestTemplate public void testEqualityDeletes() throws Exception { Table sourceTable = createTableWithSnapshots(newTableLocation(), 1); diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..11935e815e76 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,9 +32,13 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.RewriteTablePathUtil; @@ -69,13 +74,14 @@ import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.JobGroupInfo; import org.apache.iceberg.spark.source.SerializableTableWithSize; import org.apache.iceberg.util.DeleteFileSet; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; -import org.apache.spark.api.java.function.ForeachFunction; +import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.broadcast.Broadcast; @@ -87,6 +93,7 @@ import org.apache.spark.sql.functions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Tuple2; public class RewriteTablePathSparkAction extends BaseSparkAction implements RewriteTablePath { @@ -272,7 +279,8 @@ private String jobDesc() { *
    *
  • Rebuild version files to staging *
  • Rebuild manifest list files to staging - *
  • Rebuild manifest to staging + *
  • Rewrite referenced position delete files to staging + *
  • Rebuild manifests to staging *
  • Get all files needed to move *
*/ @@ -308,24 +316,29 @@ private Result rebuildMetadata() { RewriteResult rewriteManifestListResult = new RewriteResult<>(); manifestListResults.forEach(rewriteManifestListResult::append); - // rebuild manifest files - Set metaFiles = rewriteManifestListResult.toRewrite(); - RewriteContentFileResult rewriteManifestResult = - rewriteManifests(deltaSnapshots, endMetadata, metaFiles); + Set manifestFiles = rewriteManifestListResult.toRewrite(); // rebuild position delete files - Set deleteFiles = - rewriteManifestResult.toRewrite().stream() - .filter(e -> e instanceof DeleteFile) - .map(e -> (DeleteFile) e) - .collect(Collectors.toCollection(DeleteFileSet::create)); - rewritePositionDeletes(deleteFiles); + Set deleteManifests = + manifestFiles.stream() + .filter(manifest -> manifest.content() == ManifestContent.DELETES) + .collect(Collectors.toSet()); + Set deleteFilesToRewrite = positionDeletesToRewrite(deleteManifests); + Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); + + // rebuild manifest files + RewriteContentFileResult rewriteManifestResult = + rewriteManifests( + deltaSnapshots, + endMetadata, + manifestFiles, + sparkContext().broadcast(rewrittenDeleteFileSizes)); ImmutableRewriteTablePath.Result.Builder builder = ImmutableRewriteTablePath.Result.builder() .stagingLocation(stagingDir) - .rewrittenDeleteFilePathsCount(deleteFiles.size()) - .rewrittenManifestFilePathsCount(metaFiles.size()) + .rewrittenDeleteFilePathsCount(deleteFilesToRewrite.size()) + .rewrittenManifestFilePathsCount(manifestFiles.size()) .latestVersion(RewriteTablePathUtil.fileName(endVersionName)); if (!createFileList) { @@ -561,7 +574,10 @@ public RewriteContentFileResult appendDeleteFile(RewriteResult r1) { /** Rewrite manifest files in a distributed manner and return rewritten data files path pairs. */ private RewriteContentFileResult rewriteManifests( - Set deltaSnapshots, TableMetadata tableMetadata, Set toRewrite) { + Set deltaSnapshots, + TableMetadata tableMetadata, + Set toRewrite, + Broadcast> rewrittenDeleteFileSizes) { if (toRewrite.isEmpty()) { return new RewriteContentFileResult(); } @@ -581,7 +597,8 @@ private RewriteContentFileResult rewriteManifests( stagingDir, tableMetadata.formatVersion(), sourcePrefix, - targetPrefix), + targetPrefix, + rewrittenDeleteFileSizes), Encoders.bean(RewriteContentFileResult.class)) // duplicates are expected here as the same data file can have different statuses // (e.g. added and deleted) @@ -594,7 +611,8 @@ private static MapFunction toManifests( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { return manifestFile -> { RewriteContentFileResult result = new RewriteContentFileResult(); @@ -619,7 +637,8 @@ private static MapFunction toManifests( stagingLocation, format, sourcePrefix, - targetPrefix)); + targetPrefix, + rewrittenDeleteFileSizes)); break; default: throw new UnsupportedOperationException( @@ -665,7 +684,8 @@ private static RewriteResult writeDeleteManifest( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { try { String stagingPath = RewriteTablePathUtil.stagingPath(manifestFile.path(), sourcePrefix, stagingLocation); @@ -682,27 +702,120 @@ private static RewriteResult writeDeleteManifest( specsById, sourcePrefix, targetPrefix, - stagingLocation); + stagingLocation, + rewrittenDeleteFileSizes.value()); } catch (IOException e) { throw new RuntimeIOException(e); } } - private void rewritePositionDeletes(Set toRewrite) { + /** + * Enumerate the distinct position delete files referenced by the given delete manifests. Deduped + * by identity (location, offset, size) so a file shared across manifests is counted once; the + * physical rewrite is further deduped by location in {@link #rewritePositionDeletes}. + */ + private Set positionDeletesToRewrite(Set deleteManifests) { + if (deleteManifests.isEmpty()) { + return Collections.emptySet(); + } + + Encoder manifestFileEncoder = Encoders.javaSerialization(ManifestFile.class); + Dataset manifestDS = + spark().createDataset(Lists.newArrayList(deleteManifests), manifestFileEncoder); + Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); + + List referencedDeleteFiles = + manifestDS + .repartition(deleteManifests.size()) + .flatMap(positionDeletesInManifest(tableBroadcast()), deleteFileEncoder) + .collectAsList(); + + return DeleteFileSet.of(referencedDeleteFiles); + } + + private static FlatMapFunction positionDeletesInManifest( + Broadcast
tableArg) { + return manifestFile -> { + Table table = tableArg.getValue(); + List deleteFiles = Lists.newArrayList(); + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifestFile, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (deleteFile.content() == FileContent.POSITION_DELETES) { + deleteFiles.add(deleteFile.copy()); + } + } + } + return deleteFiles.iterator(); + }; + } + + /** + * Rewrite the given position delete files in parallel, returning a map from each source delete + * file path to the size of its rewritten file. Physical files are deduped by location, so a + * Puffin file holding multiple DVs is rewritten once and its size keyed once. + */ + private Map rewritePositionDeletes(Set toRewrite) { if (toRewrite.isEmpty()) { - return; + return Collections.emptyMap(); + } + + // Multiple DVs can share one Puffin file at different blob offsets; rewrite each physical file + // once. The measured size is keyed by location and applied to every referencing manifest entry. + Map byLocation = Maps.newHashMapWithExpectedSize(toRewrite.size()); + for (DeleteFile deleteFile : toRewrite) { + byLocation.putIfAbsent(deleteFile.location(), deleteFile); } + List physicalFiles = Lists.newArrayList(byLocation.values()); Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); - Dataset deleteFileDs = - spark().createDataset(Lists.newArrayList(toRewrite), deleteFileEncoder); + Dataset deleteFileDS = spark().createDataset(physicalFiles, deleteFileEncoder); PositionDeleteReaderWriter posDeleteReaderWriter = new SparkPositionDeleteReaderWriter(); - deleteFileDs - .repartition(toRewrite.size()) - .foreach( - rewritePositionDelete( - tableBroadcast(), sourcePrefix, targetPrefix, stagingDir, posDeleteReaderWriter)); + List> rewrittenSizes = + deleteFileDS + .repartition(physicalFiles.size()) + .map( + rewritePositionDelete( + tableBroadcast(), + sourcePrefix, + targetPrefix, + stagingDir, + posDeleteReaderWriter), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .collectAsList(); + + Map sizesBySourcePath = Maps.newHashMapWithExpectedSize(rewrittenSizes.size()); + for (Tuple2 entry : rewrittenSizes) { + sizesBySourcePath.put(entry._1(), entry._2()); + } + return sizesBySourcePath; + } + + private static MapFunction> rewritePositionDelete( + Broadcast
tableArg, + String sourcePrefixArg, + String targetPrefixArg, + String stagingLocationArg, + PositionDeleteReaderWriter posDeleteReaderWriter) { + return deleteFile -> { + FileIO io = tableArg.getValue().io(); + String newPath = + RewriteTablePathUtil.stagingPath( + deleteFile.location(), sourcePrefixArg, stagingLocationArg); + OutputFile outputFile = io.newOutputFile(newPath); + PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); + long rewrittenLength = + RewriteTablePathUtil.rewritePositionDelete( + deleteFile, + outputFile, + io, + spec, + sourcePrefixArg, + targetPrefixArg, + posDeleteReaderWriter); + return new Tuple2<>(deleteFile.location(), rewrittenLength); + }; } private static class SparkPositionDeleteReaderWriter implements PositionDeleteReaderWriter { @@ -724,30 +837,6 @@ public PositionDeleteWriter writer( } } - private ForeachFunction rewritePositionDelete( - Broadcast
tableArg, - String sourcePrefixArg, - String targetPrefixArg, - String stagingLocationArg, - PositionDeleteReaderWriter posDeleteReaderWriter) { - return deleteFile -> { - FileIO io = tableArg.getValue().io(); - String newPath = - RewriteTablePathUtil.stagingPath( - deleteFile.location(), sourcePrefixArg, stagingLocationArg); - OutputFile outputFile = io.newOutputFile(newPath); - PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); - RewriteTablePathUtil.rewritePositionDeleteFile( - deleteFile, - outputFile, - io, - spec, - sourcePrefixArg, - targetPrefixArg, - posDeleteReaderWriter); - }; - } - private static CloseableIterable positionDeletesReader( InputFile inputFile, FileFormat format, PartitionSpec spec) { return FormatModelRegistry.readBuilder(format, Record.class, inputFile) diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index c5db04762f21..9660beae187e 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Predicate; @@ -41,15 +42,21 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.Parameter; import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Parameters; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; @@ -62,14 +69,18 @@ import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.DVFileWriter; import org.apache.iceberg.deletes.PositionDelete; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.SparkCatalog; import org.apache.iceberg.spark.TestBase; import org.apache.iceberg.spark.source.ThreeColumnRecord; @@ -629,12 +640,7 @@ public void testPositionDeletesDeduplication() throws Exception { // in a new manifest, which will cause duplicate DeleteFile objects when processing tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); - // This should NOT throw AlreadyExistsException - the fix uses DeleteFileSet to deduplicate - // Without the fix (using Collectors.toSet()), this would fail because: - // 1. Both manifests contain entries for the same delete file - // 2. Processing returns two different DeleteFile objects for the same file - // 3. HashSet doesn't deduplicate them (DeleteFile doesn't override equals()) - // 4. rewritePositionDeletes tries to write the same file twice -> AlreadyExistsException + // This should NOT throw AlreadyExistsException RewriteTablePath.Result result = actions() .rewriteTablePath(tableWithPosDeletes) @@ -642,13 +648,295 @@ public void testPositionDeletesDeduplication() throws Exception { .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) .execute(); - // Verify the rewrite completed successfully - should have rewritten exactly 1 delete file - // (the duplicate should be deduplicated by DeleteFileSet) assertThat(result.rewrittenDeleteFilePathsCount()) .as("Should have rewritten exactly 1 delete file after deduplication") .isEqualTo(1); } + // Regression test: when the same position delete file is referenced from manifests in different + // snapshots, it must be rewritten once and the resulting size stamped consistently into every + // manifest that references it. The delete file is enumerated and deduped by path before the + // rewrite, so its measured size is shared across all referencing delete manifests. + @TestTemplate + public void testSharedDeleteFileSizeAcrossManifests() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedDelete"), + 1, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + DataFile dataFile = + tableWithPosDeletes + .currentSnapshot() + .addedDataFiles(tableWithPosDeletes.io()) + .iterator() + .next(); + + List> deletes = Lists.newArrayList(Pair.of(dataFile.location(), 0L)); + File deleteFile = + new File( + removePrefix(tableWithPosDeletes.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(deleteFile.toURI().toString()), + deletes, + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List deleteManifests = + targetTable.currentSnapshot().deleteManifests(targetTable.io()); + assertThat(deleteManifests) + .as("Expected the shared delete file to be referenced by multiple manifests") + .hasSizeGreaterThanOrEqualTo(2); + for (ManifestFile manifest : deleteManifests) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single delete manifest can reference multiple distinct position delete + // files, and each entry must be stamped with its own rewritten size. The two delete files carry + // a different number of records so they rewrite to different sizes, which catches a per-path size + // map that collapses entries to a single size or falls back to the stale original size. + @TestTemplate + public void testMultipleDistinctDeleteFileSizesAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithDistinctDeletes"), + 2, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + List dataFiles = Lists.newArrayList(); + tableWithPosDeletes + .snapshots() + .forEach( + snapshot -> snapshot.addedDataFiles(tableWithPosDeletes.io()).forEach(dataFiles::add)); + assertThat(dataFiles).as("Expected two data files to reference from deletes").hasSize(2); + + // One delete file holds a single record, the other holds two, so they rewrite to distinct + // on-disk sizes. + File smallDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-small.parquet")); + DeleteFile smallDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(smallDeleteFile.toURI().toString()), + Lists.newArrayList(Pair.of(dataFiles.get(0).location(), 0L)), + formatVersion) + .first(); + + File largeDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-large.parquet")); + DeleteFile largeDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(largeDeleteFile.toURI().toString()), + Lists.newArrayList( + Pair.of(dataFiles.get(0).location(), 0L), + Pair.of(dataFiles.get(1).location(), 0L)), + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(smallDelete).addDeletes(largeDelete).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List rewrittenSizes = Lists.newArrayList(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + rewrittenSizes.add(manifestSize); + } + } + } + + assertThat(rewrittenSizes) + .as( + "The two distinct delete files should rewrite to distinct, independently recorded sizes") + .hasSize(2) + .doesNotHaveDuplicates(); + } + + // Regression test: rewriting delete file paths changes the file size (since the + // embedded data file paths may differ in length), but file_size_in_bytes in the rewritten + // manifest was not updated. Readers that use file_size_in_bytes to elide a stat() call may + // fail. + @TestTemplate + public void testDeleteFileSizeInBytesAfterRewrite() throws Exception { + List> deletes = + Lists.newArrayList( + Pair.of( + SnapshotChanges.builderFor(table) + .build() + .addedDataFiles() + .iterator() + .next() + .location(), + 0L)); + + File file = new File(removePrefix(table.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + table, table.io().newOutputFile(file.toURI().toString()), deletes, formatVersion) + .first(); + table.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(table) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(table.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single Puffin file can hold multiple DVs (one blob per data file) + // referenced by distinct DeleteFile entries that share the same location. The rewrite must + // rewrite + // the physical Puffin file once (rather than colliding on the staging path) and stamp the + // rewritten size into every referencing manifest entry. + @TestTemplate + public void testSharedPuffinDeleteFileSizeAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("DVs are introduced in v3; v4 writes parquet manifests the test setup cannot read") + .isEqualTo(3); + + Table tableWithDVs = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedPuffin"), 2); + + List dataFilePaths = Lists.newArrayList(); + tableWithDVs + .snapshots() + .forEach( + snapshot -> + snapshot + .addedDataFiles(tableWithDVs.io()) + .forEach(dataFile -> dataFilePaths.add(dataFile.location()))); + assertThat(dataFilePaths).as("Expected two data files to back two DVs").hasSize(2); + + List dvs = writeDVsForDataFiles(tableWithDVs, dataFilePaths); + assertThat(dvs) + .as("Both DVs should live in a single Puffin file") + .hasSize(2) + .allSatisfy(dv -> assertThat(dv.location()).isEqualTo(dvs.get(0).location())); + + RowDelta rowDelta = tableWithDVs.newRowDelta(); + dvs.forEach(rowDelta::addDeletes); + rowDelta.commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithDVs) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithDVs.location(), targetTableLocation()) + .execute(); + assertThat(result.rewrittenDeleteFilePathsCount()) + .as("Two DVs are two delete files with rewritten paths") + .isEqualTo(2); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + Set rewrittenLocations = Sets.newHashSet(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + rewrittenLocations.add(df.location()); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(df.fileSizeInBytes()) + .as("file_size_in_bytes should match the rewritten Puffin size for %s", df.location()) + .isEqualTo(actualSize); + } + } + } + assertThat(rewrittenLocations) + .as("Both DVs should point at the single rewritten Puffin file") + .hasSize(1); + } + + // Writes one DV per data file path into a single Puffin file, returning the resulting DeleteFiles + // (which share a location but carry distinct blob offsets). + private List writeDVsForDataFiles(Table targetTable, List dataFilePaths) + throws IOException { + OutputFileFactory fileFactory = + OutputFileFactory.builderFor(targetTable, 1, 1).format(FileFormat.PUFFIN).build(); + DVFileWriter writer = new BaseDVFileWriter(fileFactory, p -> null); + try (DVFileWriter closeableWriter = writer) { + for (String path : dataFilePaths) { + closeableWriter.delete(path, 0L, targetTable.spec(), (StructLike) null); + } + } + + return writer.result().deleteFiles(); + } + @TestTemplate public void testEqualityDeletes() throws Exception { Table sourceTable = createTableWithSnapshots(newTableLocation(), 1); diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..11935e815e76 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,9 +32,13 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.RewriteTablePathUtil; @@ -69,13 +74,14 @@ import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.JobGroupInfo; import org.apache.iceberg.spark.source.SerializableTableWithSize; import org.apache.iceberg.util.DeleteFileSet; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; -import org.apache.spark.api.java.function.ForeachFunction; +import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.broadcast.Broadcast; @@ -87,6 +93,7 @@ import org.apache.spark.sql.functions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Tuple2; public class RewriteTablePathSparkAction extends BaseSparkAction implements RewriteTablePath { @@ -272,7 +279,8 @@ private String jobDesc() { *
    *
  • Rebuild version files to staging *
  • Rebuild manifest list files to staging - *
  • Rebuild manifest to staging + *
  • Rewrite referenced position delete files to staging + *
  • Rebuild manifests to staging *
  • Get all files needed to move *
*/ @@ -308,24 +316,29 @@ private Result rebuildMetadata() { RewriteResult rewriteManifestListResult = new RewriteResult<>(); manifestListResults.forEach(rewriteManifestListResult::append); - // rebuild manifest files - Set metaFiles = rewriteManifestListResult.toRewrite(); - RewriteContentFileResult rewriteManifestResult = - rewriteManifests(deltaSnapshots, endMetadata, metaFiles); + Set manifestFiles = rewriteManifestListResult.toRewrite(); // rebuild position delete files - Set deleteFiles = - rewriteManifestResult.toRewrite().stream() - .filter(e -> e instanceof DeleteFile) - .map(e -> (DeleteFile) e) - .collect(Collectors.toCollection(DeleteFileSet::create)); - rewritePositionDeletes(deleteFiles); + Set deleteManifests = + manifestFiles.stream() + .filter(manifest -> manifest.content() == ManifestContent.DELETES) + .collect(Collectors.toSet()); + Set deleteFilesToRewrite = positionDeletesToRewrite(deleteManifests); + Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); + + // rebuild manifest files + RewriteContentFileResult rewriteManifestResult = + rewriteManifests( + deltaSnapshots, + endMetadata, + manifestFiles, + sparkContext().broadcast(rewrittenDeleteFileSizes)); ImmutableRewriteTablePath.Result.Builder builder = ImmutableRewriteTablePath.Result.builder() .stagingLocation(stagingDir) - .rewrittenDeleteFilePathsCount(deleteFiles.size()) - .rewrittenManifestFilePathsCount(metaFiles.size()) + .rewrittenDeleteFilePathsCount(deleteFilesToRewrite.size()) + .rewrittenManifestFilePathsCount(manifestFiles.size()) .latestVersion(RewriteTablePathUtil.fileName(endVersionName)); if (!createFileList) { @@ -561,7 +574,10 @@ public RewriteContentFileResult appendDeleteFile(RewriteResult r1) { /** Rewrite manifest files in a distributed manner and return rewritten data files path pairs. */ private RewriteContentFileResult rewriteManifests( - Set deltaSnapshots, TableMetadata tableMetadata, Set toRewrite) { + Set deltaSnapshots, + TableMetadata tableMetadata, + Set toRewrite, + Broadcast> rewrittenDeleteFileSizes) { if (toRewrite.isEmpty()) { return new RewriteContentFileResult(); } @@ -581,7 +597,8 @@ private RewriteContentFileResult rewriteManifests( stagingDir, tableMetadata.formatVersion(), sourcePrefix, - targetPrefix), + targetPrefix, + rewrittenDeleteFileSizes), Encoders.bean(RewriteContentFileResult.class)) // duplicates are expected here as the same data file can have different statuses // (e.g. added and deleted) @@ -594,7 +611,8 @@ private static MapFunction toManifests( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { return manifestFile -> { RewriteContentFileResult result = new RewriteContentFileResult(); @@ -619,7 +637,8 @@ private static MapFunction toManifests( stagingLocation, format, sourcePrefix, - targetPrefix)); + targetPrefix, + rewrittenDeleteFileSizes)); break; default: throw new UnsupportedOperationException( @@ -665,7 +684,8 @@ private static RewriteResult writeDeleteManifest( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { try { String stagingPath = RewriteTablePathUtil.stagingPath(manifestFile.path(), sourcePrefix, stagingLocation); @@ -682,27 +702,120 @@ private static RewriteResult writeDeleteManifest( specsById, sourcePrefix, targetPrefix, - stagingLocation); + stagingLocation, + rewrittenDeleteFileSizes.value()); } catch (IOException e) { throw new RuntimeIOException(e); } } - private void rewritePositionDeletes(Set toRewrite) { + /** + * Enumerate the distinct position delete files referenced by the given delete manifests. Deduped + * by identity (location, offset, size) so a file shared across manifests is counted once; the + * physical rewrite is further deduped by location in {@link #rewritePositionDeletes}. + */ + private Set positionDeletesToRewrite(Set deleteManifests) { + if (deleteManifests.isEmpty()) { + return Collections.emptySet(); + } + + Encoder manifestFileEncoder = Encoders.javaSerialization(ManifestFile.class); + Dataset manifestDS = + spark().createDataset(Lists.newArrayList(deleteManifests), manifestFileEncoder); + Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); + + List referencedDeleteFiles = + manifestDS + .repartition(deleteManifests.size()) + .flatMap(positionDeletesInManifest(tableBroadcast()), deleteFileEncoder) + .collectAsList(); + + return DeleteFileSet.of(referencedDeleteFiles); + } + + private static FlatMapFunction positionDeletesInManifest( + Broadcast
tableArg) { + return manifestFile -> { + Table table = tableArg.getValue(); + List deleteFiles = Lists.newArrayList(); + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifestFile, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (deleteFile.content() == FileContent.POSITION_DELETES) { + deleteFiles.add(deleteFile.copy()); + } + } + } + return deleteFiles.iterator(); + }; + } + + /** + * Rewrite the given position delete files in parallel, returning a map from each source delete + * file path to the size of its rewritten file. Physical files are deduped by location, so a + * Puffin file holding multiple DVs is rewritten once and its size keyed once. + */ + private Map rewritePositionDeletes(Set toRewrite) { if (toRewrite.isEmpty()) { - return; + return Collections.emptyMap(); + } + + // Multiple DVs can share one Puffin file at different blob offsets; rewrite each physical file + // once. The measured size is keyed by location and applied to every referencing manifest entry. + Map byLocation = Maps.newHashMapWithExpectedSize(toRewrite.size()); + for (DeleteFile deleteFile : toRewrite) { + byLocation.putIfAbsent(deleteFile.location(), deleteFile); } + List physicalFiles = Lists.newArrayList(byLocation.values()); Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); - Dataset deleteFileDs = - spark().createDataset(Lists.newArrayList(toRewrite), deleteFileEncoder); + Dataset deleteFileDS = spark().createDataset(physicalFiles, deleteFileEncoder); PositionDeleteReaderWriter posDeleteReaderWriter = new SparkPositionDeleteReaderWriter(); - deleteFileDs - .repartition(toRewrite.size()) - .foreach( - rewritePositionDelete( - tableBroadcast(), sourcePrefix, targetPrefix, stagingDir, posDeleteReaderWriter)); + List> rewrittenSizes = + deleteFileDS + .repartition(physicalFiles.size()) + .map( + rewritePositionDelete( + tableBroadcast(), + sourcePrefix, + targetPrefix, + stagingDir, + posDeleteReaderWriter), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .collectAsList(); + + Map sizesBySourcePath = Maps.newHashMapWithExpectedSize(rewrittenSizes.size()); + for (Tuple2 entry : rewrittenSizes) { + sizesBySourcePath.put(entry._1(), entry._2()); + } + return sizesBySourcePath; + } + + private static MapFunction> rewritePositionDelete( + Broadcast
tableArg, + String sourcePrefixArg, + String targetPrefixArg, + String stagingLocationArg, + PositionDeleteReaderWriter posDeleteReaderWriter) { + return deleteFile -> { + FileIO io = tableArg.getValue().io(); + String newPath = + RewriteTablePathUtil.stagingPath( + deleteFile.location(), sourcePrefixArg, stagingLocationArg); + OutputFile outputFile = io.newOutputFile(newPath); + PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); + long rewrittenLength = + RewriteTablePathUtil.rewritePositionDelete( + deleteFile, + outputFile, + io, + spec, + sourcePrefixArg, + targetPrefixArg, + posDeleteReaderWriter); + return new Tuple2<>(deleteFile.location(), rewrittenLength); + }; } private static class SparkPositionDeleteReaderWriter implements PositionDeleteReaderWriter { @@ -724,30 +837,6 @@ public PositionDeleteWriter writer( } } - private ForeachFunction rewritePositionDelete( - Broadcast
tableArg, - String sourcePrefixArg, - String targetPrefixArg, - String stagingLocationArg, - PositionDeleteReaderWriter posDeleteReaderWriter) { - return deleteFile -> { - FileIO io = tableArg.getValue().io(); - String newPath = - RewriteTablePathUtil.stagingPath( - deleteFile.location(), sourcePrefixArg, stagingLocationArg); - OutputFile outputFile = io.newOutputFile(newPath); - PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); - RewriteTablePathUtil.rewritePositionDeleteFile( - deleteFile, - outputFile, - io, - spec, - sourcePrefixArg, - targetPrefixArg, - posDeleteReaderWriter); - }; - } - private static CloseableIterable positionDeletesReader( InputFile inputFile, FileFormat format, PartitionSpec spec) { return FormatModelRegistry.readBuilder(format, Record.class, inputFile) diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index c5db04762f21..9660beae187e 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Predicate; @@ -41,15 +42,21 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.Parameter; import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Parameters; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; @@ -62,14 +69,18 @@ import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.DVFileWriter; import org.apache.iceberg.deletes.PositionDelete; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.SparkCatalog; import org.apache.iceberg.spark.TestBase; import org.apache.iceberg.spark.source.ThreeColumnRecord; @@ -629,12 +640,7 @@ public void testPositionDeletesDeduplication() throws Exception { // in a new manifest, which will cause duplicate DeleteFile objects when processing tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); - // This should NOT throw AlreadyExistsException - the fix uses DeleteFileSet to deduplicate - // Without the fix (using Collectors.toSet()), this would fail because: - // 1. Both manifests contain entries for the same delete file - // 2. Processing returns two different DeleteFile objects for the same file - // 3. HashSet doesn't deduplicate them (DeleteFile doesn't override equals()) - // 4. rewritePositionDeletes tries to write the same file twice -> AlreadyExistsException + // This should NOT throw AlreadyExistsException RewriteTablePath.Result result = actions() .rewriteTablePath(tableWithPosDeletes) @@ -642,13 +648,295 @@ public void testPositionDeletesDeduplication() throws Exception { .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) .execute(); - // Verify the rewrite completed successfully - should have rewritten exactly 1 delete file - // (the duplicate should be deduplicated by DeleteFileSet) assertThat(result.rewrittenDeleteFilePathsCount()) .as("Should have rewritten exactly 1 delete file after deduplication") .isEqualTo(1); } + // Regression test: when the same position delete file is referenced from manifests in different + // snapshots, it must be rewritten once and the resulting size stamped consistently into every + // manifest that references it. The delete file is enumerated and deduped by path before the + // rewrite, so its measured size is shared across all referencing delete manifests. + @TestTemplate + public void testSharedDeleteFileSizeAcrossManifests() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedDelete"), + 1, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + DataFile dataFile = + tableWithPosDeletes + .currentSnapshot() + .addedDataFiles(tableWithPosDeletes.io()) + .iterator() + .next(); + + List> deletes = Lists.newArrayList(Pair.of(dataFile.location(), 0L)); + File deleteFile = + new File( + removePrefix(tableWithPosDeletes.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(deleteFile.toURI().toString()), + deletes, + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List deleteManifests = + targetTable.currentSnapshot().deleteManifests(targetTable.io()); + assertThat(deleteManifests) + .as("Expected the shared delete file to be referenced by multiple manifests") + .hasSizeGreaterThanOrEqualTo(2); + for (ManifestFile manifest : deleteManifests) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single delete manifest can reference multiple distinct position delete + // files, and each entry must be stamped with its own rewritten size. The two delete files carry + // a different number of records so they rewrite to different sizes, which catches a per-path size + // map that collapses entries to a single size or falls back to the stale original size. + @TestTemplate + public void testMultipleDistinctDeleteFileSizesAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithDistinctDeletes"), + 2, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + List dataFiles = Lists.newArrayList(); + tableWithPosDeletes + .snapshots() + .forEach( + snapshot -> snapshot.addedDataFiles(tableWithPosDeletes.io()).forEach(dataFiles::add)); + assertThat(dataFiles).as("Expected two data files to reference from deletes").hasSize(2); + + // One delete file holds a single record, the other holds two, so they rewrite to distinct + // on-disk sizes. + File smallDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-small.parquet")); + DeleteFile smallDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(smallDeleteFile.toURI().toString()), + Lists.newArrayList(Pair.of(dataFiles.get(0).location(), 0L)), + formatVersion) + .first(); + + File largeDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-large.parquet")); + DeleteFile largeDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(largeDeleteFile.toURI().toString()), + Lists.newArrayList( + Pair.of(dataFiles.get(0).location(), 0L), + Pair.of(dataFiles.get(1).location(), 0L)), + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(smallDelete).addDeletes(largeDelete).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List rewrittenSizes = Lists.newArrayList(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + rewrittenSizes.add(manifestSize); + } + } + } + + assertThat(rewrittenSizes) + .as( + "The two distinct delete files should rewrite to distinct, independently recorded sizes") + .hasSize(2) + .doesNotHaveDuplicates(); + } + + // Regression test: rewriting delete file paths changes the file size (since the + // embedded data file paths may differ in length), but file_size_in_bytes in the rewritten + // manifest was not updated. Readers that use file_size_in_bytes to elide a stat() call may + // fail. + @TestTemplate + public void testDeleteFileSizeInBytesAfterRewrite() throws Exception { + List> deletes = + Lists.newArrayList( + Pair.of( + SnapshotChanges.builderFor(table) + .build() + .addedDataFiles() + .iterator() + .next() + .location(), + 0L)); + + File file = new File(removePrefix(table.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + table, table.io().newOutputFile(file.toURI().toString()), deletes, formatVersion) + .first(); + table.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(table) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(table.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single Puffin file can hold multiple DVs (one blob per data file) + // referenced by distinct DeleteFile entries that share the same location. The rewrite must + // rewrite + // the physical Puffin file once (rather than colliding on the staging path) and stamp the + // rewritten size into every referencing manifest entry. + @TestTemplate + public void testSharedPuffinDeleteFileSizeAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("DVs are introduced in v3; v4 writes parquet manifests the test setup cannot read") + .isEqualTo(3); + + Table tableWithDVs = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedPuffin"), 2); + + List dataFilePaths = Lists.newArrayList(); + tableWithDVs + .snapshots() + .forEach( + snapshot -> + snapshot + .addedDataFiles(tableWithDVs.io()) + .forEach(dataFile -> dataFilePaths.add(dataFile.location()))); + assertThat(dataFilePaths).as("Expected two data files to back two DVs").hasSize(2); + + List dvs = writeDVsForDataFiles(tableWithDVs, dataFilePaths); + assertThat(dvs) + .as("Both DVs should live in a single Puffin file") + .hasSize(2) + .allSatisfy(dv -> assertThat(dv.location()).isEqualTo(dvs.get(0).location())); + + RowDelta rowDelta = tableWithDVs.newRowDelta(); + dvs.forEach(rowDelta::addDeletes); + rowDelta.commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithDVs) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithDVs.location(), targetTableLocation()) + .execute(); + assertThat(result.rewrittenDeleteFilePathsCount()) + .as("Two DVs are two delete files with rewritten paths") + .isEqualTo(2); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + Set rewrittenLocations = Sets.newHashSet(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + rewrittenLocations.add(df.location()); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(df.fileSizeInBytes()) + .as("file_size_in_bytes should match the rewritten Puffin size for %s", df.location()) + .isEqualTo(actualSize); + } + } + } + assertThat(rewrittenLocations) + .as("Both DVs should point at the single rewritten Puffin file") + .hasSize(1); + } + + // Writes one DV per data file path into a single Puffin file, returning the resulting DeleteFiles + // (which share a location but carry distinct blob offsets). + private List writeDVsForDataFiles(Table targetTable, List dataFilePaths) + throws IOException { + OutputFileFactory fileFactory = + OutputFileFactory.builderFor(targetTable, 1, 1).format(FileFormat.PUFFIN).build(); + DVFileWriter writer = new BaseDVFileWriter(fileFactory, p -> null); + try (DVFileWriter closeableWriter = writer) { + for (String path : dataFilePaths) { + closeableWriter.delete(path, 0L, targetTable.spec(), (StructLike) null); + } + } + + return writer.result().deleteFiles(); + } + @TestTemplate public void testEqualityDeletes() throws Exception { Table sourceTable = createTableWithSnapshots(newTableLocation(), 1); From 9ca2a4b024e9cb060eab31aba62c57933bc558f4 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Sun, 28 Jun 2026 06:27:48 +0200 Subject: [PATCH 52/73] Flink: Fix monitor source rate limit for sub-second intervals (#16979) The monitor source rate was computed as perSecond(1.0 / rateLimit.getSeconds()). Duration.getSeconds() rounds any sub-second rate limit down to 0, so the rate became perSecond(Infinity), which turns off rate limiting. This change computes the rate from millis, so sub-second intervals are limited correctly. The 60s default is unchanged. This made the maintenance E2E tests flaky, since they use sub-second rate limits. With no rate limit the source calls table.refresh() in a loop and uses a full CPU core per job. CI runs the tests with -DtestParallelism=auto, which starts one MiniCluster per fork, so these busy sources use up all the cores. The converter runs on a timer, and under that load the timer fires too slowly, letting the test fail after the timeout hits. --- .../flink/maintenance/api/TableMaintenance.java | 15 ++++++++++++++- .../maintenance/api/TestTableMaintenance.java | 8 ++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java index 025a6d17c023..46b38858b8aa 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java @@ -26,6 +26,7 @@ import java.util.UUID; import javax.annotation.Nullable; import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.eventtime.TimestampAssigner; import org.apache.flink.api.common.eventtime.TimestampAssignerSupplier; @@ -378,7 +379,9 @@ private DataStream changeStream(String tableName, TableLoader loade // Create a monitor source to provide the TableChange stream MonitorSource source = new MonitorSource( - loader, RateLimiterStrategy.perSecond(1.0 / rateLimit.getSeconds()), maxReadBack); + loader, + RateLimiterStrategy.perSecond(monitorRatePerSecond(rateLimit.toMillis())), + maxReadBack); return setSlotSharingGroup( env.fromSource( source, @@ -395,6 +398,16 @@ private static String nameFor(MaintenanceTaskBuilder streamBuilder, int taskI return String.format(Locale.ROOT, "%s [%d]", streamBuilder.maintenanceTaskName(), taskIndex); } + /** + * Monitor poll rate per rate-limit interval, in checks/second. We compute from millis instead + * of seconds, otherwise sub-second intervals could be truncated to 0, yielding an infinite rate + * which busy-loops the source. + */ + @VisibleForTesting + static double monitorRatePerSecond(long rateLimitMillis) { + return 1000.0 / rateLimitMillis; + } + private SingleOutputStreamOperator setSlotSharingGroup( SingleOutputStreamOperator operator) { return slotSharingGroup == null ? operator : operator.slotSharingGroup(slotSharingGroup); diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java index 49219d5b4698..96b2fe04d66e 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java @@ -350,6 +350,14 @@ void testUidAndSlotSharingGroupForMonitorSource() throws IOException { checkSlotSharingGroupsAreSet(env, SLOT_SHARING_GROUP); } + @Test + void testMonitorRatePerSecond() { + // Sub-second rate limits must not yield infinite (unthrottled) monitor rates. + assertThat(TableMaintenance.Builder.monitorRatePerSecond(50)).isEqualTo(20.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(100)).isEqualTo(10.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(60_000)).isEqualTo(1.0 / 60); + } + /** * Sends the events though the {@link ManualSource} provided, and waits until the given number of * records are processed. From ff1108756294f886863b149125508de79d93c251 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 22:07:27 -0700 Subject: [PATCH 53/73] Build: Bump org.codehaus.jettison:jettison from 1.5.5 to 1.5.6 (#16988) Bumps [org.codehaus.jettison:jettison](https://github.com/jettison-json/jettison) from 1.5.5 to 1.5.6. - [Release notes](https://github.com/jettison-json/jettison/releases) - [Commits](https://github.com/jettison-json/jettison/compare/jettison-1.5.5...jettison-1.5.6) --- updated-dependencies: - dependency-name: org.codehaus.jettison:jettison dependency-version: 1.5.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- kafka-connect/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-connect/build.gradle b/kafka-connect/build.gradle index 8dcea6f7ce37..61b5ed53fcfb 100644 --- a/kafka-connect/build.gradle +++ b/kafka-connect/build.gradle @@ -75,7 +75,7 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') { exclude group: 'org.jspecify', module: 'jspecify' // force upgrades for dependencies with known vulnerabilities... resolutionStrategy { - force 'org.codehaus.jettison:jettison:1.5.5' + force 'org.codehaus.jettison:jettison:1.5.6' force 'org.xerial.snappy:snappy-java:1.1.10.8' force 'org.apache.commons:commons-compress:1.28.0' force 'org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.5.0' From 45975ec1555b0365e4f2f6bdffa948ff2441711a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 23:48:30 -0700 Subject: [PATCH 54/73] Build: Bump actions/checkout from 6.0.3 to 7.0.0 (#16986) Bumps [actions/checkout](https://github.com/actions/checkout) from 6.0.3 to 7.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/df4cb1c069e1874edd31b4311f1884172cec0e10...9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/api-binary-compatibility.yml | 2 +- .github/workflows/asf-allowlist-check.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cve-scan.yml | 2 +- .github/workflows/delta-conversion-ci.yml | 4 ++-- .github/workflows/docs-ci.yml | 2 +- .github/workflows/flink-ci.yml | 2 +- .github/workflows/hive-ci.yml | 2 +- .github/workflows/java-ci.yml | 8 ++++---- .github/workflows/jmh-benchmarks.yml | 4 ++-- .github/workflows/kafka-connect-ci.yml | 2 +- .github/workflows/license-check.yml | 2 +- .github/workflows/open-api.yml | 2 +- .github/workflows/publish-iceberg-rest-fixture-docker.yml | 2 +- .github/workflows/publish-snapshot.yml | 2 +- .github/workflows/recurring-jmh-benchmarks.yml | 2 +- .github/workflows/site-ci.yml | 2 +- .github/workflows/spark-ci.yml | 2 +- .github/workflows/zizmor.yml | 2 +- 19 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/api-binary-compatibility.yml b/.github/workflows/api-binary-compatibility.yml index 50758c1c5d4c..0a09062b79bb 100644 --- a/.github/workflows/api-binary-compatibility.yml +++ b/.github/workflows/api-binary-compatibility.yml @@ -46,7 +46,7 @@ jobs: revapi: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # fetch-depth of zero ensures that the tags are pulled in and we're not in a detached HEAD state # revapi depends on the tags, specifically the tag from git describe, to find the relevant override diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml index ea5920511029..c5690c113228 100644 --- a/.github/workflows/asf-allowlist-check.yml +++ b/.github/workflows/asf-allowlist-check.yml @@ -36,7 +36,7 @@ jobs: asf-allowlist-check: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: apache/infrastructure-actions/allowlist-check@4e9c961f587f72b170874b6f5cd4ac15f7f26eb8 # main diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c22c8b5438aa..9b7b204e9272 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/.github/workflows/cve-scan.yml b/.github/workflows/cve-scan.yml index 043d2378bd43..a13f48a03da8 100644 --- a/.github/workflows/cve-scan.yml +++ b/.github/workflows/cve-scan.yml @@ -171,7 +171,7 @@ jobs: unpack: false name: ${{ matrix.distribution }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index 7bdd48307fd2..8fe24b3b9538 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -82,7 +82,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 @@ -111,7 +111,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index df49e215e370..615b290a3119 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -36,7 +36,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index a06c30c81e84..200e8a7cf8ee 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -86,7 +86,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index 2876502b8fed..c6ffa84e4926 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -83,7 +83,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index bf0ede1107c1..f5c0cb1b2427 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -78,7 +78,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 @@ -105,7 +105,7 @@ jobs: matrix: jvm: [17, 21] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 @@ -125,7 +125,7 @@ jobs: matrix: jvm: [17, 21] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 @@ -141,7 +141,7 @@ jobs: check-runtime-deps: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/jmh-benchmarks.yml b/.github/workflows/jmh-benchmarks.yml index 5d2feae989f5..6954dfd467d3 100644 --- a/.github/workflows/jmh-benchmarks.yml +++ b/.github/workflows/jmh-benchmarks.yml @@ -49,7 +49,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} foundlabel: ${{ steps.set-matrix.outputs.foundlabel }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.ref }} @@ -94,7 +94,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.ref }} diff --git a/.github/workflows/kafka-connect-ci.yml b/.github/workflows/kafka-connect-ci.yml index 115f9d0c898e..545260ca4baa 100644 --- a/.github/workflows/kafka-connect-ci.yml +++ b/.github/workflows/kafka-connect-ci.yml @@ -83,7 +83,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 997f698b1d52..6fdeda86f4e6 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -27,7 +27,7 @@ jobs: rat: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - run: | diff --git a/.github/workflows/open-api.yml b/.github/workflows/open-api.yml index 8a6fb17c9863..92732cb118ff 100644 --- a/.github/workflows/open-api.yml +++ b/.github/workflows/open-api.yml @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-slim steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Install uv diff --git a/.github/workflows/publish-iceberg-rest-fixture-docker.yml b/.github/workflows/publish-iceberg-rest-fixture-docker.yml index 71e6c68b07e5..4529e9510984 100644 --- a/.github/workflows/publish-iceberg-rest-fixture-docker.yml +++ b/.github/workflows/publish-iceberg-rest-fixture-docker.yml @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest environment: docker-publish steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index dd9a0f107983..6496b97ad568 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-24.04 environment: maven-publish steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # we need to fetch all tags so that getProjectVersion() in build.gradle correctly determines the next SNAPSHOT version from the newest tag fetch-depth: 0 diff --git a/.github/workflows/recurring-jmh-benchmarks.yml b/.github/workflows/recurring-jmh-benchmarks.yml index ef05e2b671d1..721a80d55850 100644 --- a/.github/workflows/recurring-jmh-benchmarks.yml +++ b/.github/workflows/recurring-jmh-benchmarks.yml @@ -51,7 +51,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/site-ci.yml b/.github/workflows/site-ci.yml index 43e61ff1feb2..c762fd421f3c 100644 --- a/.github/workflows/site-ci.yml +++ b/.github/workflows/site-ci.yml @@ -37,7 +37,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index ce5cae2969f9..421a8d398972 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -97,7 +97,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 7bf94e1943aa..4f231f62155d 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -34,7 +34,7 @@ jobs: permissions: {} steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false From ce92c469bd38616f3ae1bb4cf4311d9c662c4ca4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 23:48:47 -0700 Subject: [PATCH 55/73] Build: Bump actions/setup-java from 5.2.0 to 5.3.0 (#16991) Bumps [actions/setup-java](https://github.com/actions/setup-java) from 5.2.0 to 5.3.0. - [Release notes](https://github.com/actions/setup-java/releases) - [Commits](https://github.com/actions/setup-java/compare/be666c2fcd27ec809703dec50e508c2fdc7f6654...ad2b38190b15e4d6bdf0c97fb4fca8412226d287) --- updated-dependencies: - dependency-name: actions/setup-java dependency-version: 5.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/api-binary-compatibility.yml | 2 +- .github/workflows/cve-scan.yml | 2 +- .github/workflows/delta-conversion-ci.yml | 4 ++-- .github/workflows/flink-ci.yml | 2 +- .github/workflows/hive-ci.yml | 2 +- .github/workflows/java-ci.yml | 8 ++++---- .github/workflows/jmh-benchmarks.yml | 2 +- .github/workflows/kafka-connect-ci.yml | 2 +- .github/workflows/publish-iceberg-rest-fixture-docker.yml | 2 +- .github/workflows/publish-snapshot.yml | 2 +- .github/workflows/recurring-jmh-benchmarks.yml | 2 +- .github/workflows/spark-ci.yml | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/api-binary-compatibility.yml b/.github/workflows/api-binary-compatibility.yml index 0a09062b79bb..e8bd23c9b475 100644 --- a/.github/workflows/api-binary-compatibility.yml +++ b/.github/workflows/api-binary-compatibility.yml @@ -55,7 +55,7 @@ jobs: # See https://github.com/actions/checkout/issues/124 fetch-depth: 0 persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 diff --git a/.github/workflows/cve-scan.yml b/.github/workflows/cve-scan.yml index a13f48a03da8..18b0f1e0a569 100644 --- a/.github/workflows/cve-scan.yml +++ b/.github/workflows/cve-scan.yml @@ -174,7 +174,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 21 diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index 8fe24b3b9538..a1a5e952f095 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -85,7 +85,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} @@ -114,7 +114,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index 200e8a7cf8ee..f7964315e174 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -89,7 +89,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index c6ffa84e4926..59b5c832c853 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -86,7 +86,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index f5c0cb1b2427..ec6ea01c5096 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -81,7 +81,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} @@ -108,7 +108,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} @@ -128,7 +128,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} @@ -144,7 +144,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 diff --git a/.github/workflows/jmh-benchmarks.yml b/.github/workflows/jmh-benchmarks.yml index 6954dfd467d3..77856f9bec20 100644 --- a/.github/workflows/jmh-benchmarks.yml +++ b/.github/workflows/jmh-benchmarks.yml @@ -99,7 +99,7 @@ jobs: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.ref }} persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 diff --git a/.github/workflows/kafka-connect-ci.yml b/.github/workflows/kafka-connect-ci.yml index 545260ca4baa..799bf9a39ae8 100644 --- a/.github/workflows/kafka-connect-ci.yml +++ b/.github/workflows/kafka-connect-ci.yml @@ -86,7 +86,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} diff --git a/.github/workflows/publish-iceberg-rest-fixture-docker.yml b/.github/workflows/publish-iceberg-rest-fixture-docker.yml index 4529e9510984..8927f187e998 100644 --- a/.github/workflows/publish-iceberg-rest-fixture-docker.yml +++ b/.github/workflows/publish-iceberg-rest-fixture-docker.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 21 diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index 6496b97ad568..5e34aab78998 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -39,7 +39,7 @@ jobs: # we need to fetch all tags so that getProjectVersion() in build.gradle correctly determines the next SNAPSHOT version from the newest tag fetch-depth: 0 persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 diff --git a/.github/workflows/recurring-jmh-benchmarks.yml b/.github/workflows/recurring-jmh-benchmarks.yml index 721a80d55850..e91ea55be8d3 100644 --- a/.github/workflows/recurring-jmh-benchmarks.yml +++ b/.github/workflows/recurring-jmh-benchmarks.yml @@ -54,7 +54,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index 421a8d398972..32fd7c3a1a58 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -100,7 +100,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} From 839b22647ce5aa08e27220bbe85cde1292129fea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jun 2026 23:49:07 -0700 Subject: [PATCH 56/73] Build: Bump com.diffplug.spotless:spotless-plugin-gradle (#16990) Bumps [com.diffplug.spotless:spotless-plugin-gradle](https://github.com/diffplug/spotless) from 8.6.0 to 8.7.0. - [Release notes](https://github.com/diffplug/spotless/releases) - [Changelog](https://github.com/diffplug/spotless/blob/main/CHANGES.md) - [Commits](https://github.com/diffplug/spotless/compare/gradle/8.6.0...gradle/8.7.0) --- updated-dependencies: - dependency-name: com.diffplug.spotless:spotless-plugin-gradle dependency-version: 8.7.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 01e282ecf9dd..7a8f9a91128f 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ buildscript { dependencies { classpath 'com.gradleup.shadow:shadow-gradle-plugin:8.3.11' classpath 'com.palantir.baseline:gradle-baseline-java:6.90.0' - classpath 'com.diffplug.spotless:spotless-plugin-gradle:8.6.0' + classpath 'com.diffplug.spotless:spotless-plugin-gradle:8.7.0' classpath 'gradle.plugin.org.inferred:gradle-processors:3.7.0' classpath 'me.champeau.jmh:jmh-gradle-plugin:0.7.3' classpath 'gradle.plugin.io.morethan.jmhreport:gradle-jmh-report:0.9.6' From 958864156d92e56d93b8cde45e21364c05d1ee30 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Sun, 28 Jun 2026 17:24:30 +0200 Subject: [PATCH 57/73] Flink: Backport: Fix monitor source rate limit for sub-second intervals (#16979) (#16992) --- .../flink/maintenance/api/TableMaintenance.java | 15 ++++++++++++++- .../maintenance/api/TestTableMaintenance.java | 8 ++++++++ .../flink/maintenance/api/TableMaintenance.java | 15 ++++++++++++++- .../maintenance/api/TestTableMaintenance.java | 8 ++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java index 025a6d17c023..46b38858b8aa 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java @@ -26,6 +26,7 @@ import java.util.UUID; import javax.annotation.Nullable; import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.eventtime.TimestampAssigner; import org.apache.flink.api.common.eventtime.TimestampAssignerSupplier; @@ -378,7 +379,9 @@ private DataStream changeStream(String tableName, TableLoader loade // Create a monitor source to provide the TableChange stream MonitorSource source = new MonitorSource( - loader, RateLimiterStrategy.perSecond(1.0 / rateLimit.getSeconds()), maxReadBack); + loader, + RateLimiterStrategy.perSecond(monitorRatePerSecond(rateLimit.toMillis())), + maxReadBack); return setSlotSharingGroup( env.fromSource( source, @@ -395,6 +398,16 @@ private static String nameFor(MaintenanceTaskBuilder streamBuilder, int taskI return String.format(Locale.ROOT, "%s [%d]", streamBuilder.maintenanceTaskName(), taskIndex); } + /** + * Monitor poll rate per rate-limit interval, in checks/second. We compute from millis instead + * of seconds, otherwise sub-second intervals could be truncated to 0, yielding an infinite rate + * which busy-loops the source. + */ + @VisibleForTesting + static double monitorRatePerSecond(long rateLimitMillis) { + return 1000.0 / rateLimitMillis; + } + private SingleOutputStreamOperator setSlotSharingGroup( SingleOutputStreamOperator operator) { return slotSharingGroup == null ? operator : operator.slotSharingGroup(slotSharingGroup); diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java index 49219d5b4698..96b2fe04d66e 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java @@ -350,6 +350,14 @@ void testUidAndSlotSharingGroupForMonitorSource() throws IOException { checkSlotSharingGroupsAreSet(env, SLOT_SHARING_GROUP); } + @Test + void testMonitorRatePerSecond() { + // Sub-second rate limits must not yield infinite (unthrottled) monitor rates. + assertThat(TableMaintenance.Builder.monitorRatePerSecond(50)).isEqualTo(20.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(100)).isEqualTo(10.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(60_000)).isEqualTo(1.0 / 60); + } + /** * Sends the events though the {@link ManualSource} provided, and waits until the given number of * records are processed. diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java index 025a6d17c023..46b38858b8aa 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java @@ -26,6 +26,7 @@ import java.util.UUID; import javax.annotation.Nullable; import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.eventtime.TimestampAssigner; import org.apache.flink.api.common.eventtime.TimestampAssignerSupplier; @@ -378,7 +379,9 @@ private DataStream changeStream(String tableName, TableLoader loade // Create a monitor source to provide the TableChange stream MonitorSource source = new MonitorSource( - loader, RateLimiterStrategy.perSecond(1.0 / rateLimit.getSeconds()), maxReadBack); + loader, + RateLimiterStrategy.perSecond(monitorRatePerSecond(rateLimit.toMillis())), + maxReadBack); return setSlotSharingGroup( env.fromSource( source, @@ -395,6 +398,16 @@ private static String nameFor(MaintenanceTaskBuilder streamBuilder, int taskI return String.format(Locale.ROOT, "%s [%d]", streamBuilder.maintenanceTaskName(), taskIndex); } + /** + * Monitor poll rate per rate-limit interval, in checks/second. We compute from millis instead + * of seconds, otherwise sub-second intervals could be truncated to 0, yielding an infinite rate + * which busy-loops the source. + */ + @VisibleForTesting + static double monitorRatePerSecond(long rateLimitMillis) { + return 1000.0 / rateLimitMillis; + } + private SingleOutputStreamOperator setSlotSharingGroup( SingleOutputStreamOperator operator) { return slotSharingGroup == null ? operator : operator.slotSharingGroup(slotSharingGroup); diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java index 49219d5b4698..96b2fe04d66e 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java @@ -350,6 +350,14 @@ void testUidAndSlotSharingGroupForMonitorSource() throws IOException { checkSlotSharingGroupsAreSet(env, SLOT_SHARING_GROUP); } + @Test + void testMonitorRatePerSecond() { + // Sub-second rate limits must not yield infinite (unthrottled) monitor rates. + assertThat(TableMaintenance.Builder.monitorRatePerSecond(50)).isEqualTo(20.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(100)).isEqualTo(10.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(60_000)).isEqualTo(1.0 / 60); + } + /** * Sends the events though the {@link ManualSource} provided, and waits until the given number of * records are processed. From 2dbe96bb41bd5a1087220e8a1136e7cfe708bf38 Mon Sep 17 00:00:00 2001 From: drexler-sky Date: Mon, 29 Jun 2026 09:51:24 -0700 Subject: [PATCH 58/73] Build: Bump datamodel-code-generator from 0.63.0 to 0.64.1 (#16993) * Build: Bump datamodel-code-generator from 0.63.0 to 0.64.1 Regenerate open-api/rest-catalog-open-api.py with datamodel-code-generator 0.64.1. As of 0.64.0, optional primitive const fields no longer emit the const value as an injected default, so the generated discriminator fields (action/type) are now `Literal[...] | None = None`. Committing the regenerated output keeps `make generate` + `git diff --exit-code` green. Supersedes #16985. Co-Authored-By: Claude Opus 4.8 * Mark REST spec discriminators as required so they are not generated as nullable --------- Co-authored-by: Claude Opus 4.8 --- open-api/requirements.txt | 2 +- open-api/rest-catalog-open-api.py | 64 ++++++++++++++--------------- open-api/rest-catalog-open-api.yaml | 31 ++++++++++++++ 3 files changed, 63 insertions(+), 34 deletions(-) diff --git a/open-api/requirements.txt b/open-api/requirements.txt index 35dc0d4b4d57..3e3b59c33bb4 100644 --- a/open-api/requirements.txt +++ b/open-api/requirements.txt @@ -16,5 +16,5 @@ # under the License. openapi-spec-validator==0.9.0 -datamodel-code-generator==0.63.0 +datamodel-code-generator==0.64.1 yamllint==1.38.0 diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 7b6c4f73fc9e..99fe855147c8 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -368,17 +368,17 @@ class AssignUUIDUpdate(BaseUpdate): Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned """ - action: Literal['assign-uuid'] = 'assign-uuid' + action: Literal['assign-uuid'] uuid: str class UpgradeFormatVersionUpdate(BaseUpdate): - action: Literal['upgrade-format-version'] = 'upgrade-format-version' + action: Literal['upgrade-format-version'] format_version: int = Field(..., alias='format-version') class SetCurrentSchemaUpdate(BaseUpdate): - action: Literal['set-current-schema'] = 'set-current-schema' + action: Literal['set-current-schema'] schema_id: int = Field( ..., alias='schema-id', @@ -387,12 +387,12 @@ class SetCurrentSchemaUpdate(BaseUpdate): class AddPartitionSpecUpdate(BaseUpdate): - action: Literal['add-spec'] = 'add-spec' + action: Literal['add-spec'] spec: PartitionSpec class SetDefaultSpecUpdate(BaseUpdate): - action: Literal['set-default-spec'] = 'set-default-spec' + action: Literal['set-default-spec'] spec_id: int = Field( ..., alias='spec-id', @@ -401,12 +401,12 @@ class SetDefaultSpecUpdate(BaseUpdate): class AddSortOrderUpdate(BaseUpdate): - action: Literal['add-sort-order'] = 'add-sort-order' + action: Literal['add-sort-order'] sort_order: SortOrder = Field(..., alias='sort-order') class SetDefaultSortOrderUpdate(BaseUpdate): - action: Literal['set-default-sort-order'] = 'set-default-sort-order' + action: Literal['set-default-sort-order'] sort_order_id: int = Field( ..., alias='sort-order-id', @@ -415,47 +415,47 @@ class SetDefaultSortOrderUpdate(BaseUpdate): class AddSnapshotUpdate(BaseUpdate): - action: Literal['add-snapshot'] = 'add-snapshot' + action: Literal['add-snapshot'] snapshot: Snapshot class SetSnapshotRefUpdate(BaseUpdate, SnapshotReference): - action: Literal['set-snapshot-ref'] = 'set-snapshot-ref' + action: Literal['set-snapshot-ref'] ref_name: str = Field(..., alias='ref-name') class RemoveSnapshotsUpdate(BaseUpdate): - action: Literal['remove-snapshots'] = 'remove-snapshots' + action: Literal['remove-snapshots'] snapshot_ids: list[int] = Field(..., alias='snapshot-ids') class RemoveSnapshotRefUpdate(BaseUpdate): - action: Literal['remove-snapshot-ref'] = 'remove-snapshot-ref' + action: Literal['remove-snapshot-ref'] ref_name: str = Field(..., alias='ref-name') class SetLocationUpdate(BaseUpdate): - action: Literal['set-location'] = 'set-location' + action: Literal['set-location'] location: str class SetPropertiesUpdate(BaseUpdate): - action: Literal['set-properties'] = 'set-properties' + action: Literal['set-properties'] updates: dict[str, str] class RemovePropertiesUpdate(BaseUpdate): - action: Literal['remove-properties'] = 'remove-properties' + action: Literal['remove-properties'] removals: list[str] class AddViewVersionUpdate(BaseUpdate): - action: Literal['add-view-version'] = 'add-view-version' + action: Literal['add-view-version'] view_version: ViewVersion = Field(..., alias='view-version') class SetCurrentViewVersionUpdate(BaseUpdate): - action: Literal['set-current-view-version'] = 'set-current-view-version' + action: Literal['set-current-view-version'] view_version_id: int = Field( ..., alias='view-version-id', @@ -464,32 +464,32 @@ class SetCurrentViewVersionUpdate(BaseUpdate): class RemoveStatisticsUpdate(BaseUpdate): - action: Literal['remove-statistics'] = 'remove-statistics' + action: Literal['remove-statistics'] snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionStatisticsUpdate(BaseUpdate): - action: Literal['remove-partition-statistics'] = 'remove-partition-statistics' + action: Literal['remove-partition-statistics'] snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionSpecsUpdate(BaseUpdate): - action: Literal['remove-partition-specs'] = 'remove-partition-specs' + action: Literal['remove-partition-specs'] spec_ids: list[int] = Field(..., alias='spec-ids') class RemoveSchemasUpdate(BaseUpdate): - action: Literal['remove-schemas'] = 'remove-schemas' + action: Literal['remove-schemas'] schema_ids: list[int] = Field(..., alias='schema-ids') class AddEncryptionKeyUpdate(BaseUpdate): - action: Literal['add-encryption-key'] = 'add-encryption-key' + action: Literal['add-encryption-key'] encryption_key: EncryptedKey = Field(..., alias='encryption-key') class RemoveEncryptionKeyUpdate(BaseUpdate): - action: Literal['remove-encryption-key'] = 'remove-encryption-key' + action: Literal['remove-encryption-key'] key_id: str = Field(..., alias='key-id') @@ -522,7 +522,7 @@ class AssertRefSnapshotId(TableRequirement): """ - type: Literal['assert-ref-snapshot-id'] = 'assert-ref-snapshot-id' + type: Literal['assert-ref-snapshot-id'] ref: str snapshot_id: int = Field(..., alias='snapshot-id') @@ -532,7 +532,7 @@ class AssertLastAssignedFieldId(TableRequirement): The table's last assigned column id must match the requirement's `last-assigned-field-id` """ - type: Literal['assert-last-assigned-field-id'] = 'assert-last-assigned-field-id' + type: Literal['assert-last-assigned-field-id'] last_assigned_field_id: int = Field(..., alias='last-assigned-field-id') @@ -541,7 +541,7 @@ class AssertCurrentSchemaId(TableRequirement): The table's current schema id must match the requirement's `current-schema-id` """ - type: Literal['assert-current-schema-id'] = 'assert-current-schema-id' + type: Literal['assert-current-schema-id'] current_schema_id: int = Field(..., alias='current-schema-id') @@ -550,9 +550,7 @@ class AssertLastAssignedPartitionId(TableRequirement): The table's last assigned partition id must match the requirement's `last-assigned-partition-id` """ - type: Literal['assert-last-assigned-partition-id'] = ( - 'assert-last-assigned-partition-id' - ) + type: Literal['assert-last-assigned-partition-id'] last_assigned_partition_id: int = Field(..., alias='last-assigned-partition-id') @@ -561,7 +559,7 @@ class AssertDefaultSpecId(TableRequirement): The table's default spec id must match the requirement's `default-spec-id` """ - type: Literal['assert-default-spec-id'] = 'assert-default-spec-id' + type: Literal['assert-default-spec-id'] default_spec_id: int = Field(..., alias='default-spec-id') @@ -570,7 +568,7 @@ class AssertDefaultSortOrderId(TableRequirement): The table's default sort order id must match the requirement's `default-sort-order-id` """ - type: Literal['assert-default-sort-order-id'] = 'assert-default-sort-order-id' + type: Literal['assert-default-sort-order-id'] default_sort_order_id: int = Field(..., alias='default-sort-order-id') @@ -1144,7 +1142,7 @@ class TransformTerm(BaseModel): class SetPartitionStatisticsUpdate(BaseUpdate): - action: Literal['set-partition-statistics'] = 'set-partition-statistics' + action: Literal['set-partition-statistics'] partition_statistics: PartitionStatisticsFile = Field( ..., alias='partition-statistics' ) @@ -1254,7 +1252,7 @@ class Term(RootModel[Reference | TransformTerm]): class SetStatisticsUpdate(BaseUpdate): - action: Literal['set-statistics'] = 'set-statistics' + action: Literal['set-statistics'] snapshot_id: int | None = Field( None, alias='snapshot-id', @@ -1518,7 +1516,7 @@ class ViewMetadata(BaseModel): class AddSchemaUpdate(BaseUpdate): - action: Literal['add-schema'] = 'add-schema' + action: Literal['add-schema'] schema_: Schema = Field(..., alias='schema') last_column_id: int | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 990f2285e53b..b5fe7f69e37d 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3093,6 +3093,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - uuid properties: action: @@ -3105,6 +3106,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - format-version properties: action: @@ -3117,6 +3119,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - schema properties: action: @@ -3138,6 +3141,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - schema-id properties: action: @@ -3151,6 +3155,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - spec properties: action: @@ -3163,6 +3168,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - spec-id properties: action: @@ -3176,6 +3182,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - sort-order properties: action: @@ -3188,6 +3195,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - sort-order-id properties: action: @@ -3201,6 +3209,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - snapshot properties: action: @@ -3214,6 +3223,7 @@ components: - $ref: '#/components/schemas/BaseUpdate' - $ref: '#/components/schemas/SnapshotReference' required: + - action - ref-name properties: action: @@ -3226,6 +3236,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - snapshot-ids properties: action: @@ -3241,6 +3252,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - ref-name properties: action: @@ -3253,6 +3265,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - location properties: action: @@ -3265,6 +3278,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - updates properties: action: @@ -3279,6 +3293,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - removals properties: action: @@ -3293,6 +3308,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - view-version properties: action: @@ -3305,6 +3321,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - view-version-id properties: action: @@ -3318,6 +3335,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - statistics properties: action: @@ -3337,6 +3355,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - snapshot-id properties: action: @@ -3350,6 +3369,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - partition-statistics properties: action: @@ -3362,6 +3382,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - snapshot-id properties: action: @@ -3375,6 +3396,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - spec-ids properties: action: @@ -3389,6 +3411,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - schema-ids properties: action: @@ -3403,6 +3426,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - encryption-key properties: action: @@ -3415,6 +3439,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - key-id properties: action: @@ -3513,6 +3538,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - ref - snapshot-id properties: @@ -3532,6 +3558,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - last-assigned-field-id properties: type: @@ -3546,6 +3573,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - current-schema-id properties: type: @@ -3560,6 +3588,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - last-assigned-partition-id properties: type: @@ -3574,6 +3603,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - default-spec-id properties: type: @@ -3588,6 +3618,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - default-sort-order-id properties: type: From 535d032e964d4c2297a91823fb2964b3aa97cc0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:51:47 -0700 Subject: [PATCH 59/73] Build: Bump software.amazon.awssdk:bom from 2.46.10 to 2.46.15 (#16989) Bumps software.amazon.awssdk:bom from 2.46.10 to 2.46.15. --- updated-dependencies: - dependency-name: software.amazon.awssdk:bom dependency-version: 2.46.15 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d10badc2e4af..a30280fcaff0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -33,7 +33,7 @@ arrow = "15.0.2" avro = "1.12.1" assertj-core = "3.27.7" awaitility = "4.3.0" -awssdk-bom = "2.46.10" +awssdk-bom = "2.46.15" azuresdk-bom = "1.3.7" awssdk-s3accessgrants = "2.4.1" bouncycastle = "1.84" From a2fb64bff8ea2c299d56f81d666778bf4365fdac Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 29 Jun 2026 14:24:00 -0700 Subject: [PATCH 60/73] Spec: Add spec for expressions (#16652) --- format/expressions-spec.md | 311 +++++++++++++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 format/expressions-spec.md diff --git a/format/expressions-spec.md b/format/expressions-spec.md new file mode 100644 index 000000000000..01aa7d8b353d --- /dev/null +++ b/format/expressions-spec.md @@ -0,0 +1,311 @@ +--- +title: "Expressions Spec" +--- + + +# Iceberg Expressions + +This document defines the structure and behavior of expressions for use in Iceberg specifications. The purpose is to define a common structure that enables simple expressions to be stored and exchanged. + +Stored expressions are needed for use cases like data validations (`CHECK` constraints) and default values (for instance, `current_timestamp()`). Expressions are exchanged in use cases like server-side scan planning in the catalog protocol. + + +## Overview + +The goal of this specification is to define a simple expression structure and avoid complexity. + +To remain simple, the expressions that can be represented are deliberately constrained to value expressions (constants, references, and function calls) and predicates (comparisons that produce true or false). + +This approach is intended to keep focus on the logical structure of expressions. Complexity is pushed to the functions that are called, which are a limited set of well-defined and portable functions (like Iceberg partition transforms) or [user-defined functions][udf-spec] that can use the full range of SQL capabilities. Multi-dialect UDFs are responsible for any SQL constructs that are specific to an engine, rather than importing and duplicating dialects in Iceberg expressions. + +This is consistent with Iceberg's conservative approach in other specs. Expressions and predicates are an important part of Iceberg implementation APIs, but have been deliberately limited in specifications. For example, sort orders and partition fields are strictly limited to a small set of transforms over well-defined inputs (source field IDs). This spec is widening what can be expressed, but depends on function calls for complex tasks. + +This specification covers the structure of Iceberg expressions and includes appendices that specify serialization as JSON and a set of portable functions defined by Iceberg specifications. + +[udf-spec]: https://iceberg.apache.org/udf-spec + + +## Structure + +Iceberg expressions have two types: + +* **Value expressions** represent data values and transformations of values (function calls) that produce any Iceberg type +* **Predicates** represent comparisons of value expressions as well as combinations of predicates with boolean logic (and, or, not) + + +### Value expressions + +A value expression is an expression that produces a typed value. + +Value expressions can be one of three types: a constant value, a field reference, or a function applied to zero or more value expressions. + + +#### Constant values + +A constant or literal is the simplest type of value expression that represents a specific typed value. + + +#### Field reference + +A field reference represents the value of a specific field in a row. When an expression is evaluated on a row, it returns the value of the field. + +Field references may be named references (unbound) or ID references (bound). ID references identify a field by field ID from a schema. Named references identify a field by name that must be resolved to an ID (bound to a schema) to access the field. + +ID references are used for stored expressions, where the identity of the column is determined when the stored expression is created. For example, column constraints are tied to field IDs so that renaming a column does not invalidate the reference in its stored constraint. + +Named references are used when the identity of the column is determined when the expression is evaluated. For example, query filters are resolved each time a query runs so server-side planning uses unbound named references. + +The context in which an expression is used determines the type of references that are valid. Iceberg specifications should document whether ID references, named references, or both are allowed. + + +#### Apply function + +An apply expression represents the result of a function applied to (or called on) zero or more values produced by child value expressions or predicates. + +Functions are referenced using a catalog and a function identifier. + +* The function identifier consists of 0 or more namespace names followed by the function name. At least one part, the function name, is required. +* Catalog is optional and is assumed to be the catalog in which the referencing object is stored if it is not present or is null + +The catalog name identifies the catalog where the function definition can be loaded or is a reserved name that identifies a set of functions. As in the view and UDF specs, catalog names represent connection configurations that may differ across environments. Omitting catalog names is recommended to avoid depending on consistent environments. For example, if a table has a CHECK constraint that references a UDF without a catalog name (missing or null), the UDF should be loaded from the table's catalog. + +The reserved names used to identify sets are: + +* `sql_functions` is used for functions defined by the SQL standard +* `iceberg_functions` is used for functions defined in this specification + +Engines may document and use a catalog name to identify their built-in functions that are not part of the SQL spec, like `spark_builtin_functions.to_utc_timestamp`. + +Function references are unambiguous and are not interpreted using session context. Producers are responsible for resolving catalog, namespace, and name if the session is relevant. For example, if a SQL engine uses its current catalog and namespace to find a function, the resolved catalog and namespace must be used to produce an unambiguous function reference. + + +#### Value expression types + +The type produced by a value expression may change. For example, an ID reference may produce a widened type after the underlying column's type is promoted. + +A value expression's result type is determined when it is bound to a specific input schema. + +Function calls may produce different types when function definitions change, and type changes may change the definition that is resolved for a function name. For example, if the input field passed to `identity(int) -> int` is promoted from `int` to `long`, the resolved `identity` function can change to `identity(long) -> long` if it is defined. + +If types are incompatible at runtime, implementations binding or evaluating expressions may apply type promotion to align types for predicates and to resolve functions. Implementations may choose when to promote values to accommodate engines that differ in casting behavior. However, implementations must fail rather than insert unsafe casts. + + +### Predicates + +A predicate is a boolean expression that produces true or false. + +Predicates can be constants (true or false), tests of a value expression, comparisons of value expressions, or logical combinations of predicates (AND, OR, NOT). + +Value expressions are not valid predicates, even when the expression is expected to return a boolean value. Value expressions must be compared or tested to produce a predicate. For example, `is_empty(str_col)` is not a valid predicate because it may produce `null`, but `is_empty(str_col) = true` is a valid predicate. + + +#### Tests + +Tests are predicates that test a single value expression, optionally using a constant or set of constants. Constants must all have the same type and must be non-null and non-NaN. Tests are: + +| Test | Allowed types | Constant type | Description | +|-------------------------|---------------|---------------|-------------| +| `IS NULL` | any | | true iff the value is null | +| `IS NOT NULL` | any | | true iff the value is not null | +| `IS NaN` | float, double | | true iff the value is an IEEE 754 NaN | +| `IS NOT NaN` | float, double | | true iff the value is not an IEEE 754 NaN | +| `STARTS WITH const` | string | string | true iff the constant is a prefix of the value | +| `NOT STARTS WITH const` | string | string | true iff the constant is not a prefix of the value | +| `IN (constant set)` | any primitive | same as value | true iff the value is equal to any constant | +| `NOT IN (constant set)` | any primitive | same as value | true iff the value is not equal to any constant | + + +#### Comparisons + +Comparisons are predicates that compare two value expressions with the same primitive type. + +If value expression types in a comparison are incompatible, implementations should align types using type promotion. For instance, `int_col > 5.0` should promote int values to float. If the types cannot be aligned according to type promotion rules (for instance, `"goats" > -Infinity`), the predicate cannot be evaluated and implementations must fail. + +Comparisons are: + +| Comparison | Description | +|-------------|-------------| +| `=` | Is equal (is not distinct from) | +| `!=` | Is not equal | +| `<` | Less than | +| `<=` | Less than or equal | +| `>` | Greater than | +| `>=` | Greater than or equal | + +Comparisons must be null-safe. For any two operands a and b: + +* `a = b` is true if both are null, or both are non-null and equal; otherwise false +* `a != b` is the boolean negation of `a = b` +* `a < b` and `a > b` are false when either operand is null; otherwise they use the order defined above +* `a <= b` is `(a = b) OR (a < b)`; `a >= b` is `(a = b) OR (a > b)`; both are true when both operands are null and false when only one operand is null + +This table shows examples of these rules after evaluating value expressions to constants: + +| Comparison | Result | +|----------------|---------| +| `null = null` | `true` | +| `34 = null` | `false` | +| `null != null` | `false` | +| `34 != null` | `true` | +| `null < null` | `false` | +| `null <= null` | `true` | +| `34 < null` | `false` | + +Value expressions that are the direct child of a comparison must not be either a null or NaN constant. However, comparisons must handle null and NaN values that are the result of evaluating a value expression. For example, `x = get_item(map, "key")` is valid although `get_item` may return a null value, but `x = null` must be rejected because `x IS NULL` is the correct unambiguous predicate. Similarly, `multiply(a, b)` may produce NaN for `a=0.0` and `b=Infinity` and is valid, but `x = NaN` must be rejected because `x IS NaN` is the correct test. + +Primitive types are compared using signed comparison, except for the following types: + +* `false` is less than `true` for `boolean` +* `fixed` and `binary` use unsigned byte-wise comparison +* `string` uses unsigned byte-wise comparison of the UTF-8 representation; it is not the Unicode Collation Algorithm +* `uuid` uses unsigned byte-wise comparison of the UUID bytes +* `decimal` uses signed comparison independent of scale; this is equivalent to comparison of unscaled values because type alignment produces values with the same scale +* `float` and `double` use IEEE 754 order for all non-NaN values; see below for NaN comparison rules + +For floating point values, comparison with NaN behaves similarly to comparison of values with null. NaN should be specifically handled using `IS NaN` and `IS NOT NaN` tests. However, when value expressions produce a NaN value, the following rules must be applied: + +* `a = b` is true if both are NaN, or both are non-NaN and equal; false otherwise +* `a != b` is the boolean negation of `a = b` +* `a < b` and `a > b` are false when either operand is NaN; otherwise the IEEE 754 order is used +* `a <= b` is `(a = b) OR (a < b)`; `a >= b` is `(a = b) OR (a > b)`; both are true when both operands are NaN and false when only one operand is NaN + + +#### Boolean logic + +Predicates must use 2-valued boolean logic. Evaluation of all predicates must produce `true` or `false`. + +Engines that implement SQL 3-valued boolean logic must add `IS NULL` and `IS NOT NULL` to produce the 2-valued equivalent. This avoids bugs in engines and languages that do not natively implement 3-valued logic. For example, the SQL predicate `x < 10` should be passed as `x < 10 AND x IS NOT NULL` for a SQL `WHERE` condition (or `x < 10`; see null-safe comparisons below). For a `CHECK` constraint, the expression is passed as `x < 10 OR x IS NULL`. This ensures that implementations will make the correct determination, rather than depending on context to interpret a null result (`WHERE` vs `CHECK`). + +Logical combinations are boolean operators applied to predicates. `AND` and `OR` are binary operations and `NOT` is a unary operation. `AND`, `OR`, and `NOT` do not accept null values because predicates cannot produce them. + + +### Compatibility with REST catalog expressions + +Prior to this spec, REST APIs used a more restrictive, term-based form of predicates and references. Those forms are now deprecated, but should be supported for backward compatibility to allow older clients to interact with newer REST catalog services. + +The deprecated expressions were passed in 3 places: + +* As `filter` passed to server-side scan planning +* As `filter` passed to the service in `ScanReport` +* As `residual` passed to the client with a scan task + +Both server-side scan planning and the report endpoint should continue to accept filters from older clients by parsing term-based expressions (see [Appendix B: JSON serialization](#backward-compatibility)). + +Residuals passed from services back to clients that do not use the new syntax would cause clients to fail. Services are allowed to omit the residual so that it is calculated on the client side (intended to avoid duplicating large IN filters). For compatibility, REST services should omit residuals from tasks, but may include them if the service detects support for newer predicates (for example, via client version). + + +## Appendix A: Iceberg functions + +This section defines the functions in the `iceberg_functions` reserved catalog name. + +* `if_else(condition: predicate, when_true: T, when_false: T) -> T`: returns the value of `when_true` when `condition` is true and `when_false` otherwise + +### Partition transforms + +Iceberg partition transforms are also defined as functions (other than `void`). + +All partition transforms produce `null` for a `null` input value. + +| Function name | Description | Source types | Result type | +|-------------------|--------------------------------------------------------------|----------------------------------------------------------------------|-------------| +| `identity(value)` | Source value, unmodified | Any primitive except for `geometry`, `geography`, and `variant` | Source type | +| `year(value)` | Extract a date or timestamp year, as years from 1970 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | +| `month(value)` | Extract a date or timestamp month, as months from 1970-01-01 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | +| `day(value)` | Extract a date or timestamp day, as days from 1970-01-01 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `date` | +| `hour(value)` | Extract a timestamp hour, as hours from 1970-01-01 00:00:00 | `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | + +Note that `year`, `month`, and `hour` transforms produce ordinal values and not human-readable values. For example, `year(2018-05-13)` produces `48`, not `2018`. + +`bucket` and `truncate` are called as 2-argument functions. The first argument is an `int` parameter (`N` or `W` from the table spec) and the second argument is the value to transform. For example, `bucket(256, id)` calls `bucket[256]`. + +| Parameterized function name | Description | Source types | Result type | +|-----------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------| +| `bucket(N, value)` | Hash of value, mod `N` (see [table spec details][bucket-ref]) | Any primitive except for `geometry`, `geography`, `variant`, `boolean`, `float`, or `double` | `int` | +| `truncate(W, value)` | Value truncated to width `W` (see [table spec details][truncate-ref]) | `int`, `long`, `decimal`, `string`, `binary` | Source type | + +[bucket-ref]: spec/#bucket-transform-details +[truncate-ref]: spec/#truncate-transform-details + + +## Appendix B: JSON serialization + +Iceberg expressions are serialized as JSON objects in table, view, and UDF metadata, and in the REST protocol for catalogs. + +### Value expressions + +``` +EXPR: LITERAL | REFERENCE | APPLY + +LITERAL: VALUE + | { "type": "literal", "value": VALUE } + | { "type": "literal", "value": VALUE, "data-type": DATA_TYPE } +LITERALS: [ LITERAL* ] + | { "type": "literals", "values": [ VALUE* ], "data-type": DATA_TYPE } + +REFERENCE: BOUND_REF | UNBOUND_REF +BOUND_REF: { "type": "reference", "id": ID } +UNBOUND_REF: { "type": "reference", "name": NAME } + +APPLY: { "type": "apply", "function": FUNC_REF, "arguments": [ FUNC_ARG* ] } +FUNC_ARG: EXPR | PREDICATE +FUNC_REF: NAME + | [ NAME* ] + | { "identifier": [ NAME* ] } + | { "catalog": NAME, "identifier": [ NAME* ] } + +ID: integer +NAME: string + +VALUE: single value JSON from the table spec +DATA_TYPE: Iceberg type from the spec +``` + +If a function reference is a string, that string is the one-part identifier and the catalog is missing/null. +If a function reference is a list of strings, it is the function identifier and the catalog is missing/null. + +### Predicates + +``` +PREDICATE: true | false + | { "type": "not", "child": PREDICATE } + | { "type": BINARY_OP, "left": PREDICATE, "right": PREDICATE } + | { "type": UNARY_OP, "child": EXPR } + | { "type": CMP_OP, "left": EXPR, "right": EXPR } + | { "type": SET_OP, "child": EXPR, "values": LITERALS } + | DEPRECATED_PREDICATE + +BINARY_OP: "and" | "or" +UNARY_OP: "is-null" | "not-null" | "is-nan" | "not-nan" +CMP_OP: "lt" | "lt-eq" | "gt" | "gt-eq" | "eq" | "not-eq" + | "starts-with" | "not-starts-with" +SET_OP: "in" | "not-in" +``` + +### Backward compatibility + +``` +DEPRECATED_PREDICATE: + | { "type": UNARY_OP, "term": TERM } + | { "type": CMP_OP, "term": TERM, "value": LITERAL } + | { "type": SET_OP, "term": TERM, "values": LITERALS } + +DEPRECATED_REF: { "type": "reference", "term": NAME } + +TERM: NAME | DEPRECATED_REF | TRANSFORM +TRANSFORM: { "type": "transform", "transform": NAME, "term": TERM } +``` From 8c1ee9d9df4184a7889f85f6a9e1f82f959e0eaf Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Mon, 29 Jun 2026 18:43:57 -0700 Subject: [PATCH 61/73] Core, Spark: Migrate Spark table properties to Spark module (#15875) --- .../org/apache/iceberg/TableProperties.java | 41 +++++++++++++++++-- .../SparkRowLevelOperationsTestBase.java | 4 +- .../iceberg/spark/SparkTableProperties.java | 38 +++++++++++++++++ .../apache/iceberg/spark/SparkWriteConf.java | 4 +- .../iceberg/spark/source/SparkTable.java | 5 ++- .../TestSparkDistributionAndOrderingUtil.java | 6 +-- .../spark/source/TestDataFrameWriterV2.java | 18 ++++---- .../spark/source/TestSparkDataWrite.java | 4 +- .../SparkRowLevelOperationsTestBase.java | 4 +- .../iceberg/spark/SparkTableProperties.java | 38 +++++++++++++++++ .../apache/iceberg/spark/SparkWriteConf.java | 4 +- .../iceberg/spark/source/SparkTable.java | 5 ++- .../TestSparkDistributionAndOrderingUtil.java | 6 +-- .../spark/source/TestDataFrameWriterV2.java | 18 ++++---- .../spark/source/TestSparkDataWrite.java | 4 +- .../SparkRowLevelOperationsTestBase.java | 4 +- .../extensions/TestMergeSchemaEvolution.java | 4 +- .../iceberg/spark/SparkTableProperties.java | 38 +++++++++++++++++ .../apache/iceberg/spark/SparkWriteConf.java | 4 +- .../iceberg/spark/source/SparkTable.java | 10 ++--- .../TestSparkDistributionAndOrderingUtil.java | 6 +-- .../spark/source/TestDataFrameWriterV2.java | 18 ++++---- .../spark/source/TestSparkDataWrite.java | 4 +- 23 files changed, 219 insertions(+), 68 deletions(-) create mode 100644 spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java create mode 100644 spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java create mode 100644 spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 881c00b71263..56d381632607 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -357,16 +357,51 @@ private TableProperties() {} public static final String DELETE_TARGET_FILE_SIZE_BYTES = "write.delete.target-file-size-bytes"; public static final long DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB + /** + * @deprecated will be removed in 1.14.0, use + * SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED in iceberg-spark instead. + */ + @Deprecated public static final String SPARK_WRITE_PARTITIONED_FANOUT_ENABLED = "write.spark.fanout.enabled"; - public static final boolean SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + /** + * @deprecated will be removed in 1.14.0, use + * SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT in iceberg-spark instead. + */ + @Deprecated public static final boolean SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + + /** + * @deprecated will be removed in 1.14.0, use SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA in + * iceberg-spark instead. + */ + @Deprecated public static final String SPARK_WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema"; - public static final boolean SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + /** + * @deprecated will be removed in 1.14.0, use SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA_DEFAULT + * in iceberg-spark instead. + */ + @Deprecated public static final boolean SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + + /** + * @deprecated will be removed in 1.14.0, use SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION in + * iceberg-spark instead. + */ + @Deprecated public static final String SPARK_WRITE_AUTO_SCHEMA_EVOLUTION = "write.spark.auto-schema-evolution.enabled"; - public static final boolean SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + /** + * @deprecated will be removed in 1.14.0, use + * SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT in iceberg-spark instead. + */ + @Deprecated public static final boolean SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + + /** + * @deprecated will be removed in 1.14.0, use + * SparkTableProperties.WRITE_ADVISORY_PARTITION_SIZE_BYTES in iceberg-spark instead. + */ + @Deprecated public static final String SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES = "write.spark.advisory-partition-size-bytes"; diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index 72988ae0ed9e..36f28acd9955 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -34,11 +34,11 @@ import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.ORC_VECTORIZATION_ENABLED; import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -172,7 +172,7 @@ protected void initTable() { fileFormat, WRITE_DISTRIBUTION_MODE, distributionMode, - SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, + WRITE_PARTITIONED_FANOUT_ENABLED, String.valueOf(fanoutEnabled), DATA_PLANNING_MODE, planningMode.modeName(), diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java new file mode 100644 index 000000000000..6ca02220a8d8 --- /dev/null +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +/** Spark-specific table properties that control Spark integration behavior. */ +public class SparkTableProperties { + + private SparkTableProperties() {} + + public static final String WRITE_PARTITIONED_FANOUT_ENABLED = "write.spark.fanout.enabled"; + public static final boolean WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + + public static final String WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema"; + public static final boolean WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + + public static final String WRITE_AUTO_SCHEMA_EVOLUTION = + "write.spark.auto-schema-evolution.enabled"; + public static final boolean WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + + public static final String WRITE_ADVISORY_PARTITION_SIZE_BYTES = + "write.spark.advisory-partition-size-bytes"; +} diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 9da48ae51e5c..537a14e4ab52 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -228,7 +228,7 @@ private boolean fanoutWriterEnabled(boolean defaultValue) { return confParser .booleanConf() .option(SparkWriteOptions.FANOUT_ENABLED) - .tableProperty(TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED) + .tableProperty(SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED) .defaultValue(defaultValue) .parse(); } @@ -717,7 +717,7 @@ private long advisoryPartitionSize(long defaultValue) { .longConf() .option(SparkWriteOptions.ADVISORY_PARTITION_SIZE) .sessionConf(SparkSQLProperties.ADVISORY_PARTITION_SIZE) - .tableProperty(TableProperties.SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES) + .tableProperty(SparkTableProperties.WRITE_ADVISORY_PARTITION_SIZE_BYTES) .defaultValue(defaultValue) .parse(); } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java index 1348afff6475..3cb5d1809f95 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java @@ -59,6 +59,7 @@ import org.apache.iceberg.spark.SparkReadOptions; import org.apache.iceberg.spark.SparkSQLProperties; import org.apache.iceberg.spark.SparkSchemaUtil; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.SparkTableUtil; import org.apache.iceberg.spark.SparkUtil; import org.apache.iceberg.spark.SparkV2Filters; @@ -156,8 +157,8 @@ public SparkTable( boolean acceptAnySchema = PropertyUtil.propertyAsBoolean( icebergTable.properties(), - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA, - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA, + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); this.capabilities = acceptAnySchema ? CAPABILITIES_WITH_ACCEPT_ANY_SCHEMA : CAPABILITIES; this.isTableRewrite = isTableRewrite; } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java index ca86350346cd..e1592525f31e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java @@ -20,12 +20,12 @@ import static org.apache.iceberg.TableProperties.DELETE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.MERGE_DISTRIBUTION_MODE; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.UPDATE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.MERGE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.UPDATE; @@ -3020,10 +3020,10 @@ private void checkPositionDeltaDistributionAndOrdering( } private void disableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); } private void enableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java index 0a51d5b16f50..240d5bcf5bc6 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java @@ -25,9 +25,9 @@ import java.math.BigDecimal; import java.util.List; import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.spark.Spark3Util; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.TestBaseWithCatalog; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -60,7 +60,7 @@ public void removeTables() { public void testMergeSchemaFailsWithoutWriterOption() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -119,7 +119,7 @@ public void testMergeSchemaWithoutAcceptAnySchema() throws Exception { public void testMergeSchemaSparkProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -153,7 +153,7 @@ public void testMergeSchemaSparkProperty() throws Exception { public void testMergeSchemaIcebergProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -190,7 +190,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars .sql( String.format( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA)) + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA)) .collect(); String schema = "ID bigint, DaTa string"; @@ -220,7 +220,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars public void testMergeSchemaSparkConfiguration() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( "id bigint, data string", @@ -255,7 +255,7 @@ public void testMergeSchemaSparkConfiguration() throws Exception { public void testMergeSchemaIgnoreCastingLongToInt() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset bigintDF = jsonToDF( @@ -297,7 +297,7 @@ public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception { sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset doubleDF = jsonToDF( @@ -339,7 +339,7 @@ public void testMergeSchemaIgnoreCastingDecimalToDecimalWithNarrowerPrecision() sql("CREATE TABLE %s (id decimal(6,2), data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset decimalPrecision6DF = jsonToDF( diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java index 70f3b986d23b..7f2dc287ff5c 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java @@ -18,7 +18,7 @@ */ package org.apache.iceberg.spark.source; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -624,7 +624,7 @@ public void partitionedCreateWithTargetFileSizeViaOption(IcebergOptionsType opti .save(location.toString()); break; case TABLE: - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); df.select("id", "data") .write() .format("iceberg") diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index 72988ae0ed9e..36f28acd9955 100644 --- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -34,11 +34,11 @@ import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.ORC_VECTORIZATION_ENABLED; import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -172,7 +172,7 @@ protected void initTable() { fileFormat, WRITE_DISTRIBUTION_MODE, distributionMode, - SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, + WRITE_PARTITIONED_FANOUT_ENABLED, String.valueOf(fanoutEnabled), DATA_PLANNING_MODE, planningMode.modeName(), diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java new file mode 100644 index 000000000000..6ca02220a8d8 --- /dev/null +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +/** Spark-specific table properties that control Spark integration behavior. */ +public class SparkTableProperties { + + private SparkTableProperties() {} + + public static final String WRITE_PARTITIONED_FANOUT_ENABLED = "write.spark.fanout.enabled"; + public static final boolean WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + + public static final String WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema"; + public static final boolean WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + + public static final String WRITE_AUTO_SCHEMA_EVOLUTION = + "write.spark.auto-schema-evolution.enabled"; + public static final boolean WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + + public static final String WRITE_ADVISORY_PARTITION_SIZE_BYTES = + "write.spark.advisory-partition-size-bytes"; +} diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index add12e6040b0..71a4a7878fef 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -232,7 +232,7 @@ private boolean fanoutWriterEnabled(boolean defaultValue) { return confParser .booleanConf() .option(SparkWriteOptions.FANOUT_ENABLED) - .tableProperty(TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED) + .tableProperty(SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED) .defaultValue(defaultValue) .parse(); } @@ -731,7 +731,7 @@ private long advisoryPartitionSize(long defaultValue) { .longConf() .option(SparkWriteOptions.ADVISORY_PARTITION_SIZE) .sessionConf(SparkSQLProperties.ADVISORY_PARTITION_SIZE) - .tableProperty(TableProperties.SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES) + .tableProperty(SparkTableProperties.WRITE_ADVISORY_PARTITION_SIZE_BYTES) .defaultValue(defaultValue) .parse(); } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java index 9e3c9a7e69e6..6060a2ee5c21 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java @@ -60,6 +60,7 @@ import org.apache.iceberg.spark.SparkReadOptions; import org.apache.iceberg.spark.SparkSQLProperties; import org.apache.iceberg.spark.SparkSchemaUtil; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.SparkTableUtil; import org.apache.iceberg.spark.SparkUtil; import org.apache.iceberg.spark.SparkV2Filters; @@ -157,8 +158,8 @@ public SparkTable( boolean acceptAnySchema = PropertyUtil.propertyAsBoolean( icebergTable.properties(), - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA, - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA, + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); this.capabilities = acceptAnySchema ? CAPABILITIES_WITH_ACCEPT_ANY_SCHEMA : CAPABILITIES; this.isTableRewrite = isTableRewrite; } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java index ca86350346cd..e1592525f31e 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java @@ -20,12 +20,12 @@ import static org.apache.iceberg.TableProperties.DELETE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.MERGE_DISTRIBUTION_MODE; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.UPDATE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.MERGE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.UPDATE; @@ -3020,10 +3020,10 @@ private void checkPositionDeltaDistributionAndOrdering( } private void disableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); } private void enableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); } } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java index 186d0b2b5204..c1adc405b89e 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java @@ -25,10 +25,10 @@ import java.math.BigDecimal; import java.util.List; import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.spark.Spark3Util; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.TestBaseWithCatalog; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -61,7 +61,7 @@ public void removeTables() { public void testMergeSchemaFailsWithoutWriterOption() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -120,7 +120,7 @@ public void testMergeSchemaWithoutAcceptAnySchema() throws Exception { public void testMergeSchemaSparkProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -154,7 +154,7 @@ public void testMergeSchemaSparkProperty() throws Exception { public void testMergeSchemaIcebergProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -196,7 +196,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars .sql( String.format( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA)) + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA)) .collect(); String schema = "ID bigint, DaTa string"; @@ -226,7 +226,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars public void testMergeSchemaSparkConfiguration() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( "id bigint, data string", @@ -261,7 +261,7 @@ public void testMergeSchemaSparkConfiguration() throws Exception { public void testMergeSchemaIgnoreCastingLongToInt() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset bigintDF = jsonToDF( @@ -303,7 +303,7 @@ public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception { sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset doubleDF = jsonToDF( @@ -345,7 +345,7 @@ public void testMergeSchemaIgnoreCastingDecimalToDecimalWithNarrowerPrecision() sql("CREATE TABLE %s (id decimal(6,2), data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset decimalPrecision6DF = jsonToDF( diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java index 70f3b986d23b..7f2dc287ff5c 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java @@ -18,7 +18,7 @@ */ package org.apache.iceberg.spark.source; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -624,7 +624,7 @@ public void partitionedCreateWithTargetFileSizeViaOption(IcebergOptionsType opti .save(location.toString()); break; case TABLE: - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); df.select("id", "data") .write() .format("iceberg") diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index 72988ae0ed9e..36f28acd9955 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -34,11 +34,11 @@ import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.ORC_VECTORIZATION_ENABLED; import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -172,7 +172,7 @@ protected void initTable() { fileFormat, WRITE_DISTRIBUTION_MODE, distributionMode, - SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, + WRITE_PARTITIONED_FANOUT_ENABLED, String.valueOf(fanoutEnabled), DATA_PLANNING_MODE, planningMode.modeName(), diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMergeSchemaEvolution.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMergeSchemaEvolution.java index 782321b588a7..fa8314ecaea4 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMergeSchemaEvolution.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMergeSchemaEvolution.java @@ -24,9 +24,9 @@ import java.util.Map; import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.spark.SparkTableProperties; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestTemplate; @@ -272,7 +272,7 @@ public void testMergeWithSchemaEvolutionDisabledByTableProperty() { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s' = 'false')", - tableName, TableProperties.SPARK_WRITE_AUTO_SCHEMA_EVOLUTION); + tableName, SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION); createOrReplaceView( "source", diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java new file mode 100644 index 000000000000..6ca02220a8d8 --- /dev/null +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +/** Spark-specific table properties that control Spark integration behavior. */ +public class SparkTableProperties { + + private SparkTableProperties() {} + + public static final String WRITE_PARTITIONED_FANOUT_ENABLED = "write.spark.fanout.enabled"; + public static final boolean WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + + public static final String WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema"; + public static final boolean WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + + public static final String WRITE_AUTO_SCHEMA_EVOLUTION = + "write.spark.auto-schema-evolution.enabled"; + public static final boolean WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + + public static final String WRITE_ADVISORY_PARTITION_SIZE_BYTES = + "write.spark.advisory-partition-size-bytes"; +} diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 80f93427805a..bf0474db6245 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -239,7 +239,7 @@ private boolean fanoutWriterEnabled(boolean defaultValue) { return confParser .booleanConf() .option(SparkWriteOptions.FANOUT_ENABLED) - .tableProperty(TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED) + .tableProperty(SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED) .defaultValue(defaultValue) .parse(); } @@ -706,7 +706,7 @@ private long advisoryPartitionSize(long defaultValue) { .longConf() .option(SparkWriteOptions.ADVISORY_PARTITION_SIZE) .sessionConf(SparkSQLProperties.ADVISORY_PARTITION_SIZE) - .tableProperty(TableProperties.SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES) + .tableProperty(SparkTableProperties.WRITE_ADVISORY_PARTITION_SIZE_BYTES) .defaultValue(defaultValue) .parse(); } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java index 80a40d72c8d1..9cd170f352eb 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java @@ -31,7 +31,6 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Evaluator; @@ -49,6 +48,7 @@ import org.apache.iceberg.spark.CommitMetadata; import org.apache.iceberg.spark.Spark3Util; import org.apache.iceberg.spark.SparkReadConf; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.SparkTableUtil; import org.apache.iceberg.spark.SparkUtil; import org.apache.iceberg.spark.SparkV2Filters; @@ -375,15 +375,15 @@ private static Set computeCapabilities(Table table) { private static boolean acceptAnySchema(Table table) { return PropertyUtil.propertyAsBoolean( table.properties(), - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA, - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA, + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); } private static boolean autoSchemaEvolution(Table table) { return PropertyUtil.propertyAsBoolean( table.properties(), - TableProperties.SPARK_WRITE_AUTO_SCHEMA_EVOLUTION, - TableProperties.SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT); + SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION, + SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT); } // returns latest snapshot for branch or current snapshot if branch is yet to be created diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java index 21b67b89a99b..5fa9abde1225 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java @@ -20,12 +20,12 @@ import static org.apache.iceberg.TableProperties.DELETE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.MERGE_DISTRIBUTION_MODE; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.UPDATE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.MERGE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.UPDATE; @@ -3019,10 +3019,10 @@ private void checkPositionDeltaDistributionAndOrdering( } private void disableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); } private void enableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); } } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java index 186d0b2b5204..c1adc405b89e 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java @@ -25,10 +25,10 @@ import java.math.BigDecimal; import java.util.List; import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.spark.Spark3Util; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.TestBaseWithCatalog; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -61,7 +61,7 @@ public void removeTables() { public void testMergeSchemaFailsWithoutWriterOption() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -120,7 +120,7 @@ public void testMergeSchemaWithoutAcceptAnySchema() throws Exception { public void testMergeSchemaSparkProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -154,7 +154,7 @@ public void testMergeSchemaSparkProperty() throws Exception { public void testMergeSchemaIcebergProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -196,7 +196,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars .sql( String.format( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA)) + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA)) .collect(); String schema = "ID bigint, DaTa string"; @@ -226,7 +226,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars public void testMergeSchemaSparkConfiguration() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( "id bigint, data string", @@ -261,7 +261,7 @@ public void testMergeSchemaSparkConfiguration() throws Exception { public void testMergeSchemaIgnoreCastingLongToInt() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset bigintDF = jsonToDF( @@ -303,7 +303,7 @@ public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception { sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset doubleDF = jsonToDF( @@ -345,7 +345,7 @@ public void testMergeSchemaIgnoreCastingDecimalToDecimalWithNarrowerPrecision() sql("CREATE TABLE %s (id decimal(6,2), data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset decimalPrecision6DF = jsonToDF( diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java index c2f5afef0e8c..aacdabfe6047 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java @@ -18,7 +18,7 @@ */ package org.apache.iceberg.spark.source; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -697,7 +697,7 @@ public void partitionedCreateWithTargetFileSizeViaOption(IcebergOptionsType opti .save(location.toString()); break; case TABLE: - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); df.select("id", "data") .write() .format("iceberg") From be27af46df4316b41787a62382c5aa330a173c1e Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Tue, 30 Jun 2026 22:28:42 +0200 Subject: [PATCH 62/73] REST: Fix schema of data-access object in REST spec (#16594) The current spec defines this object as a single enum constant, not a comma-separated list of enum constants. --- open-api/rest-catalog-open-api.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index b5fe7f69e37d..419041d12a87 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -2152,10 +2152,12 @@ components: required: false schema: - type: string - enum: - - vended-credentials - - remote-signing + type: array + items: + type: string + enum: + - vended-credentials + - remote-signing style: simple explode: false example: "vended-credentials,remote-signing" From c9e61fa55da1b2f47d94a1988b4a5a358248e8c9 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 1 Jul 2026 11:22:22 -0400 Subject: [PATCH 63/73] CVE Scan: Test PR failure reporting UI (#16962) * CVE scan: improve PR failure reporting Keep the Trivy scan step green so GitHub opens the reporting step that prints the actionable CVE findings, then fail PR runs from that reporting step. Also remove the Spark 3.5 Jackson CVE ignore entries on this test branch so the PR run exercises the failure UI. Generated-by: GPT-5 Codex * CVE scan: format Trivy findings as a table Parse Trivy SARIF messages into a compact table and keep the PR annotation concise so the failed reporting step is easier to read. Generated-by: GPT-5 Codex * CVE scan: simplify Trivy report step Extract SARIF parsing and report rendering into small shell helpers so the report step is easier to read without changing behavior. Generated-by: GPT-5 Codex * CVE scan: restore Spark 3.5 Trivy ignore Restore the Spark 3.5 Jackson CVE ignore entries that were removed only to exercise the PR failure UI. Generated-by: GPT-5 Codex * CVE scan: document PR and push behavior Clarify that Trivy always writes SARIF, PRs fail from the reporting step on findings, push runs keep findings informational, and missing or unparseable SARIF is still an error. Generated-by: GPT-5 Codex --- .github/workflows/cve-scan.yml | 117 +++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cve-scan.yml b/.github/workflows/cve-scan.yml index 18b0f1e0a569..2365f86837fb 100644 --- a/.github/workflows/cve-scan.yml +++ b/.github/workflows/cve-scan.yml @@ -88,12 +88,13 @@ jobs: # Trivy CVE scan — scans bundled jars for known vulnerabilities. # # Behaviour: - # - On PRs: the scan blocks CI if CVEs are found (exit-code 1). - # SARIF upload is skipped because GitHub's Security tab only - # accepts results from default/protected branches. - # - On push to main/release branches: the scan is informational - # (exit-code 0) and results are uploaded as SARIF to the GitHub - # Security tab for ongoing tracking. + # - Trivy always writes SARIF with exit-code 0. The reporting step owns + # finding-based failures so GitHub opens the step with actionable details. + # - On PRs: the reporting step blocks CI if HIGH/CRITICAL CVEs are found. + # - On push to main/release branches and release tags: findings are + # informational, then SARIF is uploaded to the GitHub Security tab. + # - Missing or unparseable SARIF is still a failure on every event because + # it means the scan did not produce usable results. # ------------------------------------------------------------------ cve-scan: runs-on: ubuntu-24.04 @@ -220,19 +221,107 @@ jobs: severity: 'HIGH,CRITICAL' trivyignores: ${{ matrix.trivyignores }} limit-severities-for-sarif: true - # Block PRs on CVE findings; on main/release branches report without failing - exit-code: ${{ github.event_name == 'pull_request' && '1' || '0' }} + # Let Trivy generate SARIF without failing on findings. GitHub opens the + # failed step by default, so PRs fail later in the reporting step that + # prints the actionable CVE details. + exit-code: '0' format: 'sarif' output: 'trivy-results.sarif' trivy-image: ${{ env.TRIVY_IMAGE }} - - name: Print Trivy scan results + - name: Report Trivy scan results if: always() + env: + # PRs block on findings; push runs report findings without failing so + # SARIF can be uploaded to GitHub Security for tracking. + FAIL_ON_FINDINGS: ${{ github.event_name == 'pull_request' && 'true' || 'false' }} run: | - if [ -f trivy-results.sarif ]; then - echo "## Trivy CVE Scan Results — ${{ matrix.distribution }}" - jq -r '.runs[].results[] | "- \(.ruleId): \(.message.text)"' trivy-results.sarif 2>/dev/null || echo "No findings or unable to parse SARIF." - else - echo "No SARIF file found — scan may have failed to install." + results_file="trivy-results.sarif" + summary="${GITHUB_STEP_SUMMARY:-/dev/null}" + + log() { + printf '%s\n' "$*" | tee -a "${summary}" + } + + markdown_escape() { + value="$1" + value="${value//|/\\|}" + printf '%s' "${value}" + } + + escape_annotation() { + value="$1" + value="${value//'%'/%25}" + value="${value//$'\r'/%0D}" + value="${value//$'\n'/%0A}" + printf '%s' "${value}" + } + + extract_findings() { + jq -r ' + def field($prefix): + (.message.text | split("\n") | map(select(startswith($prefix))) | first // "") + | ltrimstr($prefix); + + .runs[].results[]? + | [ + .ruleId, + field("Severity: "), + field("Package: "), + field("Installed Version: "), + field("Fixed Version: "), + field("Link: ") + ] + | @tsv + ' "${results_file}" + } + + report_findings() { + log "Found ${finding_count} HIGH/CRITICAL vulnerabilities." + log "" + log "| CVE | Severity | Package | Installed | Fixed | Link |" + log "| --- | --- | --- | --- | --- | --- |" + + while IFS=$'\t' read -r cve severity package installed fixed link; do + cve="$(markdown_escape "${cve}")" + severity="$(markdown_escape "${severity}")" + package="$(markdown_escape "${package}")" + installed="$(markdown_escape "${installed}")" + fixed="$(markdown_escape "${fixed}")" + link="$(markdown_escape "${link}")" + log "| ${cve} | ${severity} | \`${package}\` | \`${installed}\` | ${fixed} | ${link} |" + done <<< "${findings}" + } + + if [ ! -f "${results_file}" ]; then + log "No SARIF file found — scan may have failed to run." + exit 1 + fi + + if ! findings="$(extract_findings)"; then + log "Unable to parse Trivy SARIF results." + exit 1 + fi + + log "## Trivy CVE Scan Results — ${{ matrix.distribution }}" + + if [ -z "${findings}" ]; then + log "No HIGH or CRITICAL vulnerabilities found." + exit 0 + fi + + finding_count="$(printf '%s\n' "${findings}" | awk 'END { print NR }')" + finding_ids="$(printf '%s\n' "${findings}" | cut -f1 | awk 'BEGIN { sep="" } { printf "%s%s", sep, $0; sep=", " } END { print "" }')" + + report_findings + + if [ "${FAIL_ON_FINDINGS}" = "true" ]; then + # Surface findings in the PR checks UI, not just in the workflow logs. + annotation_message="Trivy found ${finding_count} HIGH/CRITICAL vulnerabilities in ${{ matrix.distribution }}: ${finding_ids}. See the 'Report Trivy scan results' step for details." + annotation="$(escape_annotation "${annotation_message}")" + echo "::error title=Trivy CVE scan failed::${annotation}" + log "" + log "Failing because ${finding_count} HIGH/CRITICAL vulnerabilities were found." + exit 1 fi - name: Upload Trivy results to GitHub Security tab if: always() && github.event_name == 'push' From f1ed57c1776b7f6cc96e0d434aa6dbdea6407348 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Thu, 2 Jul 2026 02:22:50 +0800 Subject: [PATCH 64/73] Core: Include row lineage and key-id in snapshot value methods (#17015) Include snapshot row-lineage fields and manifest list key IDs in BaseSnapshot equals/hashCode and toString. This keeps comparisons and hashes aligned with metadata that changes row ID assignment or manifest-list encryption key selection, and makes those values visible in debug output. Co-authored-by: Codex --- .../java/org/apache/iceberg/BaseSnapshot.java | 17 +++++++++-- .../java/org/apache/iceberg/TestSnapshot.java | 30 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java index b8ea6db22938..826b9624c0e6 100644 --- a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java +++ b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java @@ -340,7 +340,10 @@ public boolean equals(Object o) { && Objects.equal(this.parentId, other.parentId()) && this.sequenceNumber == other.sequenceNumber() && this.timestampMillis == other.timestampMillis() - && Objects.equal(this.schemaId, other.schemaId()); + && Objects.equal(this.schemaId, other.schemaId()) + && Objects.equal(this.firstRowId, other.firstRowId()) + && Objects.equal(this.addedRows, other.addedRows()) + && Objects.equal(this.keyId, other.keyId()); } return false; @@ -349,7 +352,14 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hashCode( - this.snapshotId, this.parentId, this.sequenceNumber, this.timestampMillis, this.schemaId); + this.snapshotId, + this.parentId, + this.sequenceNumber, + this.timestampMillis, + this.schemaId, + this.firstRowId, + this.addedRows, + this.keyId); } @Override @@ -361,6 +371,9 @@ public String toString() { .add("summary", summary) .add("manifest-list", manifestListLocation) .add("schema-id", schemaId) + .add("first-row-id", firstRowId) + .add("added-rows", addedRows) + .add("key-id", keyId) .toString(); } } diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshot.java b/core/src/test/java/org/apache/iceberg/TestSnapshot.java index ff9aebffc3a6..398bce3e94d9 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshot.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshot.java @@ -30,6 +30,36 @@ @ExtendWith(ParameterizedTestExtension.class) public class TestSnapshot extends TestBase { + @TestTemplate + public void snapshotValueMethodsIncludeMetadataFields() { + Snapshot snapshot = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-1"); + Snapshot same = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-1"); + Snapshot differentFirstRowId = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 11L, 5L, "key-1"); + Snapshot differentAddedRows = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 6L, "key-1"); + Snapshot differentKeyId = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-2"); + Snapshot withoutMetadataFields = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null); + + assertThat(snapshot).isEqualTo(same); + assertThat(snapshot.hashCode()).isEqualTo(same.hashCode()); + assertThat(snapshot).isNotEqualTo(differentFirstRowId); + assertThat(snapshot).isNotEqualTo(differentAddedRows); + assertThat(snapshot).isNotEqualTo(differentKeyId); + assertThat(snapshot).isNotEqualTo(withoutMetadataFields); + assertThat(snapshot.toString()).contains("first-row-id=10", "added-rows=5", "key-id=key-1"); + } + @TestTemplate public void testAppendFilesFromTable() { table.newFastAppend().appendFile(FILE_A).appendFile(FILE_B).commit(); From e4074709c509bf41bf2b4a112773786105707765 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 1 Jul 2026 12:06:05 -0700 Subject: [PATCH 65/73] API: Add indexStatsNames to create field names for content stats (#17010) --- .../org/apache/iceberg/types/IndexByName.java | 21 ++++-- .../org/apache/iceberg/types/TypeUtil.java | 13 ++++ .../apache/iceberg/types/TestTypeUtil.java | 69 ++++++++++++++++++- 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/IndexByName.java b/api/src/main/java/org/apache/iceberg/types/IndexByName.java index 60258f5c5c3e..9ca2a1d3396c 100644 --- a/api/src/main/java/org/apache/iceberg/types/IndexByName.java +++ b/api/src/main/java/org/apache/iceberg/types/IndexByName.java @@ -32,20 +32,26 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; public class IndexByName extends TypeUtil.SchemaVisitor> { - private static final Joiner DOT = Joiner.on("."); - private final Deque fieldNames = Lists.newLinkedList(); private final Deque shortFieldNames = Lists.newLinkedList(); private final Map nameToId = Maps.newHashMap(); private final Map shortNameToId = Maps.newHashMap(); + private final Joiner joiner; private final Function quotingFunc; + private final boolean useShortNames; public IndexByName() { this(Function.identity()); } public IndexByName(Function quotingFunc) { + this(".", quotingFunc, false); + } + + IndexByName(String separator, Function quotingFunc, boolean useShortNames) { + this.joiner = Joiner.on(separator); this.quotingFunc = quotingFunc; + this.useShortNames = useShortNames; } /** @@ -76,8 +82,15 @@ public Map byName() { * @return a map from field ID to name */ public Map byId() { + // a builder is used to swap key and value ImmutableMap.Builder builder = ImmutableMap.builder(); nameToId.forEach((key, value) -> builder.put(value, key)); + + if (useShortNames) { + shortNameToId.forEach((key, value) -> builder.put(value, key)); + return builder.buildKeepingLast(); + } + return builder.build(); } @@ -193,7 +206,7 @@ private void addField(String name, int fieldId) { if (!fieldNames.isEmpty()) { Iterator quotedFieldNames = Iterators.transform(fieldNames.descendingIterator(), quotingFunc::apply); - fullName = DOT.join(DOT.join(quotedFieldNames), quotedName); + fullName = joiner.join(joiner.join(quotedFieldNames), quotedName); } Integer existingFieldId = nameToId.put(fullName, fieldId); @@ -208,7 +221,7 @@ private void addField(String name, int fieldId) { if (!shortFieldNames.isEmpty()) { Iterator quotedShortFieldNames = Iterators.transform(shortFieldNames.descendingIterator(), quotingFunc::apply); - String shortName = DOT.join(DOT.join(quotedShortFieldNames), quotedName); + String shortName = joiner.join(joiner.join(quotedShortFieldNames), quotedName); if (!shortNameToId.containsKey(shortName)) { shortNameToId.put(shortName, fieldId); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index 22e461a39ef9..a3bee3e3d860 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -195,6 +195,19 @@ public static Map indexQuotedNameById( return indexer.byId(); } + /** + * Indexes the fields of a struct from ID to a flattened name that can be used for stats struct + * field names. + * + * @param struct a struct type + * @return an index from field ID to short names, joined by _ + */ + public static Map indexStatsNames(Types.StructType struct) { + IndexByName indexer = new IndexByName("_", Function.identity(), true /* use short names */); + visit(struct, indexer); + return indexer.byId(); + } + /** * Creates a mapping from lower-case field names to their corresponding field IDs. * diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index dd8afebab84d..b7da4b3108e6 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -571,7 +571,7 @@ public void testReassignIdsIllegalArgumentException() { } @Test - public void testValidateSchemaViaIndexByName() { + public void testValidateSchemaViaByName() { Types.NestedField nestedType = Types.NestedField.required( 1, @@ -970,4 +970,71 @@ public void ancestorFieldsInNestedSchema() { assertThat(TypeUtil.ancestorFields(schema, 16)).containsExactly(pointsElement, points); assertThat(TypeUtil.ancestorFields(schema, 17)).containsExactly(pointsElement, points); } + + @Test + public void testIndexStatsNames() { + Schema schema = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "data", Types.StringType.get()), + optional( + 3, + "location", + Types.StructType.of( + required(10, "lat", Types.FloatType.get()), + required(11, "long", Types.FloatType.get()))), + optional(4, "values", Types.ListType.ofOptional(12, IntegerType.get())), + optional( + 5, + "points", + Types.ListType.ofRequired( + 13, + Types.StructType.of( + required(14, "x", IntegerType.get()), + required(15, "y", IntegerType.get())))), + optional( + 6, + "properties", + Types.MapType.ofOptional(16, 17, Types.StringType.get(), Types.StringType.get())), + optional( + 7, + "addresses", + Types.MapType.ofRequired( + 18, + 19, + Types.StringType.get(), + Types.StructType.of( + required(20, "street", Types.StringType.get()), + optional(21, "city", Types.StringType.get()), + optional(22, "state", Types.StringType.get()), + required(23, "zip", Types.IntegerType.get()), + required(24, "value", Types.IntegerType.get()))))); + + Map statsNameIndex = TypeUtil.indexStatsNames(schema.asStruct()); + + assertThat(statsNameIndex) + .containsEntry(1, "id") + .containsEntry(2, "data") + .containsEntry(3, "location") + .containsEntry(10, "location_lat") + .containsEntry(11, "location_long") + .containsEntry(4, "values") + .containsEntry(12, "values_element") + .containsEntry(5, "points") + .containsEntry(13, "points_element") + .containsEntry(14, "points_x") + .containsEntry(15, "points_y") + .containsEntry(6, "properties") + .containsEntry(16, "properties_key") + .containsEntry(17, "properties_value") + .containsEntry(7, "addresses") + .containsEntry(18, "addresses_key") + .containsEntry(19, "addresses_value") + .containsEntry(20, "addresses_street") + .containsEntry(21, "addresses_city") + .containsEntry(22, "addresses_state") + .containsEntry(23, "addresses_zip") + .containsEntry(24, "addresses_value") // the leaf takes precedence + .hasSize(22); + } } From 41113b1f691d34a7b62c933e2303836c029f2142 Mon Sep 17 00:00:00 2001 From: gaborkaszab Date: Wed, 1 Jul 2026 22:08:13 +0200 Subject: [PATCH 66/73] Core: Remove unused TestTrackedFileStruct.CONTENT_STATS_ORDINAL (#17031) --- .../test/java/org/apache/iceberg/TestTrackedFileStruct.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 8e0d5c06824b..c3391bcbbe44 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -49,10 +49,7 @@ class TestTrackedFileStruct { private static final int FILE_SIZE_IN_BYTES_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.FILE_SIZE_IN_BYTES); private static final int SPEC_ID_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.SPEC_ID); - // partition and content_stats are rebuilt with the supplied struct types inside - // schemaWithContentStats, so their ordinals are looked up by field ID. private static final int PARTITION_ORDINAL = ordinalOf(TrackedFile.PARTITION_ID); - private static final int CONTENT_STATS_ORDINAL = ordinalOf(TrackedFile.CONTENT_STATS_ID); private static final int SORT_ORDER_ID_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.SORT_ORDER_ID); private static final int DELETION_VECTOR_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.DELETION_VECTOR); From d6aa0bf5c244cf6934eaa1820f1a88d099610f7b Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Wed, 1 Jul 2026 15:22:22 -0700 Subject: [PATCH 67/73] Spec: Add optional specific-name to UDF definition model (#16727) --- format/udf-spec.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/format/udf-spec.md b/format/udf-spec.md index a11f10ffd2fd..7a5e89010cac 100644 --- a/format/udf-spec.md +++ b/format/udf-spec.md @@ -80,6 +80,7 @@ must produce values of the declared `return-type`. | Requirement | Field name | Type | Description | |-------------|----------------------|--------------------------------|---------------------------------------------------------------------------------------------------| | *required* | `definition-id` | `string` | An identifier derived from canonical parameter-type tuple (see [Definition ID](#definition-id)). | +| *optional* | `specific-name` | `string` | A user-assignable name for this definition; must be unique (see [Specific Name](#specific-name)). | | *required* | `parameters` | `list` | Ordered list of [function parameters](#parameter). Invocation order **must** match this list. | | *required* | `return-type` | `string` or `object` | Declared return type using [Types](#types). | | *optional* | `return-nullable` | `boolean` | A hint to indicate whether the return value is nullable or not. Default: `true`. | @@ -131,6 +132,13 @@ Examples of complete definition-id signatures: * `int,string` – two parameters: int and string * `int,list,struct` – three parameters: an int, a list and a struct +#### Specific Name + +The `specific-name` is an optional, user-assignable name for a single definition, analogous to the SQL standard's +routine *specific name*. It provides a stable, human-readable handle for a definition that is independent of its +signature (e.g., for SQL statements such as `DROP SPECIFIC FUNCTION`). +When present, `specific-name` **must** be unique among all definitions within the UDF metadata. + ### Definition Version Each definition can evolve over time by introducing new versions. From da8ff447a23447733ea6231625cc4d468245b090 Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Thu, 2 Jul 2026 13:51:42 +0700 Subject: [PATCH 68/73] Docs: Document nightly snapshots (#16544) --- site/docs/developer-snapshot-testing.md | 112 ++++++++++++++++++++++++ site/docs/multi-engine-support.md | 2 +- site/mkdocs-dev.yml | 1 + site/nav.yml | 1 + 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 site/docs/developer-snapshot-testing.md diff --git a/site/docs/developer-snapshot-testing.md b/site/docs/developer-snapshot-testing.md new file mode 100644 index 000000000000..6365e66e590b --- /dev/null +++ b/site/docs/developer-snapshot-testing.md @@ -0,0 +1,112 @@ +--- +title: "Developer Snapshot Testing" +--- + + +{# Mirrors getProjectVersion() in build.gradle for released versions: bump minor, reset patch to 0, and append -SNAPSHOT. #} +{% set icebergVersionParts = icebergVersion.split('.') %} +{% set snapshotMinorVersion = (icebergVersionParts[1] | int) + 1 %} +{% set snapshotVersion = icebergVersionParts[0] ~ '.' ~ snapshotMinorVersion ~ '.0-SNAPSHOT' %} + +# Developer Snapshot Testing + +!!! warning "For Iceberg developers only" + + Nightly snapshots are **unreleased** development artifacts. They are **not** + official Apache releases, are not intended for general use, may change or + break at any time, and **must not be used in production**. If you are not + actively participating in Iceberg development or following dev list + discussions, use an official [release](releases.md) instead. + +Every night, Apache Iceberg publishes snapshots of every module from unreleased +changes on `main` to support active Iceberg developers, engine maintainers, and +automated testing. +Per the ASF [release policy](https://apache.org/legal/release-policy#publication), +unreleased artifacts are developer resources for testing ongoing development and +are not a substitute for official releases. + +## Snapshot version + +Snapshots are published daily at 00:00 UTC under the version `{{ snapshotVersion }}`, +which tracks the next unreleased development version by incrementing the latest +release minor version and resetting the patch version to 0. + +## Development-only usage + +Active developers who need to validate unreleased Iceberg changes can use the +[Apache snapshot repository](https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/) +when a development task or [dev list](community.md#mailing-lists) discussion +calls for testing snapshots. Add snapshot dependencies only to temporary local +test builds; do not commit them to production applications, user documentation, +or release validation workflows. + +The examples below use `iceberg-core` as the module under test. Replace it only +with the Iceberg module needed for the development task you are validating. + +=== "Gradle (development only)" + + ```gradle + repositories { + mavenCentral() + maven { + url = uri("https://repository.apache.org/content/repositories/snapshots") + } + } + + dependencies { + implementation "org.apache.iceberg:iceberg-core:{{ snapshotVersion }}" + } + ``` + +=== "Maven (development only)" + + ```xml + + + apache-snapshots + https://repository.apache.org/content/repositories/snapshots + + true + + + + + + + org.apache.iceberg + iceberg-core + {{ snapshotVersion }} + + + ``` + +=== "sbt (development only)" + + ```scala + resolvers += "Apache Snapshots" at "https://repository.apache.org/content/repositories/snapshots" + + libraryDependencies += "org.apache.iceberg" % "iceberg-core" % "{{ snapshotVersion }}" + ``` + +=== "Spark (development only)" + + ```sh + spark-shell \ + --repositories https://repository.apache.org/content/repositories/snapshots \ + --packages org.apache.iceberg:iceberg-spark-runtime-{{ sparkVersionMajor }}:{{ snapshotVersion }} + ``` diff --git a/site/docs/multi-engine-support.md b/site/docs/multi-engine-support.md index bdfe87123aa3..d153c6104bba 100644 --- a/site/docs/multi-engine-support.md +++ b/site/docs/multi-engine-support.md @@ -135,6 +135,6 @@ This allows the Iceberg support to evolve with the engine. Projects such as [Trino](https://trino.io/docs/current/connector/iceberg.html) and [Presto](https://prestodb.io/docs/current/connector/iceberg.html) are good examples of such support strategy. In this approach, an Iceberg version upgrade is needed for an engine to consume new Iceberg features. -To facilitate engine development against unreleased Iceberg features, a daily snapshot is published in the [Apache snapshot repository](https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/). +To facilitate engine development against unreleased Iceberg features, [developer snapshot testing](developer-snapshot-testing.md) is available for active developers and engine maintainers testing ongoing development. If bringing an engine directly to the Iceberg main repository is needed, please raise a discussion thread in the [Iceberg community](community.md). diff --git a/site/mkdocs-dev.yml b/site/mkdocs-dev.yml index f8d4d71bf316..1557a73f041a 100644 --- a/site/mkdocs-dev.yml +++ b/site/mkdocs-dev.yml @@ -82,6 +82,7 @@ nav: - Project: - Contributing: contribute.md - Multi-engine support: multi-engine-support.md + - Developer snapshot testing: developer-snapshot-testing.md - Benchmarks: benchmarks.md - Security: security.md - How to release: how-to-release.md diff --git a/site/nav.yml b/site/nav.yml index c5e9fa5ed0ba..a83f66165acc 100644 --- a/site/nav.yml +++ b/site/nav.yml @@ -98,6 +98,7 @@ nav: - Project: - Contributing: contribute.md - Multi-engine support: multi-engine-support.md + - Developer snapshot testing: developer-snapshot-testing.md - Benchmarks: benchmarks.md - Security: security.md - How to release: how-to-release.md From 744e811036c87727b41aa211fab9010bab48721f Mon Sep 17 00:00:00 2001 From: Xin Huang <42597328+huan233usc@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:36:18 -0700 Subject: [PATCH 69/73] Parquet: Read and write geometry and geography WKB values (#16982) --- .../org/apache/iceberg/util/RandomUtil.java | 23 +++++ .../apache/iceberg/InternalTestHelpers.java | 2 + .../apache/iceberg/RandomInternalData.java | 2 + .../apache/iceberg/data/DataTestHelpers.java | 2 + .../iceberg/data/RandomGenericData.java | 2 + .../iceberg/data/parquet/TestGenericData.java | 5 ++ .../data/parquet/BaseParquetReaders.java | 14 ++++ .../data/parquet/BaseParquetWriter.java | 10 +-- .../iceberg/parquet/TestInternalParquet.java | 5 ++ .../parquet/TestParquetDataWriter.java | 84 ++++++++++++------- 10 files changed, 114 insertions(+), 35 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/util/RandomUtil.java b/api/src/test/java/org/apache/iceberg/util/RandomUtil.java index b437b0bbf51c..4dd3ac887044 100644 --- a/api/src/test/java/org/apache/iceberg/util/RandomUtil.java +++ b/api/src/test/java/org/apache/iceberg/util/RandomUtil.java @@ -20,6 +20,8 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -157,6 +159,12 @@ public static Object generatePrimitive(Type.PrimitiveType primitive, Random rand BigDecimal bigDecimal = new BigDecimal(unscaled, type.scale()); return negate(choice) ? bigDecimal.negate() : bigDecimal; + case GEOMETRY: + case GEOGRAPHY: + // geometry and geography values are stored as WKB + return wkbPoint( + (random.nextDouble() * 360.0) - 180.0, (random.nextDouble() * 180.0) - 90.0); + default: throw new IllegalArgumentException( "Cannot generate random value for unknown type: " + primitive); @@ -202,12 +210,27 @@ public static Object generateDictionaryEncodablePrimitive( byte[] uuidBytes = new byte[16]; random.nextBytes(uuidBytes); return uuidBytes; + case GEOMETRY: + case GEOGRAPHY: + // a small set of distinct points so the WKB column stays dictionary encodable + return wkbPoint(value, value); default: throw new IllegalArgumentException( "Cannot generate random value for unknown type: " + primitive); } } + /** Encodes a point as little-endian WKB, the on-disk representation for geo values. */ + public static byte[] wkbPoint(double xCoord, double yCoord) { + return ByteBuffer.allocate(21) + .order(ByteOrder.LITTLE_ENDIAN) + .put((byte) 1) // byte order: little endian + .putInt(1) // WKB geometry type: Point + .putDouble(xCoord) + .putDouble(yCoord) + .array(); + } + private static final long FIFTY_YEARS_IN_MICROS = (50L * (365 * 3 + 366) * 24 * 60 * 60 * 1_000_000) / 4; private static final long ABOUT_TEN_YEARS_IN_NANOS = 10L * 365 * 24 * 60 * 60 * 1_000_000_000; diff --git a/core/src/test/java/org/apache/iceberg/InternalTestHelpers.java b/core/src/test/java/org/apache/iceberg/InternalTestHelpers.java index 781051f11d7b..226f105f5310 100644 --- a/core/src/test/java/org/apache/iceberg/InternalTestHelpers.java +++ b/core/src/test/java/org/apache/iceberg/InternalTestHelpers.java @@ -91,6 +91,8 @@ private static void assertEquals(Type type, Object expected, Object actual) { case FIXED: case BINARY: case DECIMAL: + case GEOMETRY: + case GEOGRAPHY: assertThat(actual).as("Primitive value should be equal to expected").isEqualTo(expected); break; case STRUCT: diff --git a/core/src/test/java/org/apache/iceberg/RandomInternalData.java b/core/src/test/java/org/apache/iceberg/RandomInternalData.java index e458ce054cc0..a0638ad010a9 100644 --- a/core/src/test/java/org/apache/iceberg/RandomInternalData.java +++ b/core/src/test/java/org/apache/iceberg/RandomInternalData.java @@ -99,6 +99,8 @@ public Object primitive(Type.PrimitiveType primitive) { switch (primitive.typeId()) { case FIXED: case BINARY: + case GEOMETRY: + case GEOGRAPHY: return ByteBuffer.wrap((byte[]) result); case UUID: return UUID.nameUUIDFromBytes((byte[]) result); diff --git a/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java b/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java index f2e2b4e7fa34..6321d876a985 100644 --- a/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java +++ b/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java @@ -128,6 +128,8 @@ private static void assertEquals(Type type, Object expected, Object actual) { case UUID: case BINARY: case DECIMAL: + case GEOMETRY: + case GEOGRAPHY: assertThat(actual) .as("Primitive value should be equal to expected for type " + type) .isEqualTo(expected); diff --git a/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java b/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java index 4963052e0877..583513d3f90d 100644 --- a/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java +++ b/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java @@ -268,6 +268,8 @@ public Object primitive(Type.PrimitiveType primitive) { Object result = randomValue(primitive, random); switch (primitive.typeId()) { case BINARY: + case GEOMETRY: + case GEOGRAPHY: return ByteBuffer.wrap((byte[]) result); case UUID: return UUID.nameUUIDFromBytes((byte[]) result); diff --git a/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java b/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java index 8c0e2e903ab7..63b6e4769631 100644 --- a/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java +++ b/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java @@ -68,6 +68,11 @@ protected boolean supportsRowLineage() { return true; } + @Override + protected boolean supportsGeospatial() { + return true; + } + @Override protected void writeAndValidate(Schema schema) throws IOException { writeAndValidate(schema, schema); diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java index b1fd8f43a578..89024951ccaa 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java @@ -224,6 +224,20 @@ public Optional> visit( return Optional.of(ParquetValueReaders.byteBuffers(desc)); } + @Override + public Optional> visit( + LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryLogicalType) { + // geometry values are pure WKB stored in a BINARY column + return Optional.of(ParquetValueReaders.byteBuffers(desc)); + } + + @Override + public Optional> visit( + LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyLogicalType) { + // geography values are pure WKB stored in a BINARY column + return Optional.of(ParquetValueReaders.byteBuffers(desc)); + } + @Override public Optional> visit( LogicalTypeAnnotation.UUIDLogicalTypeAnnotation uuidLogicalType) { diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java index 1f3c7ab31f1c..dcc93f939d8e 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java @@ -268,17 +268,15 @@ public Optional> visit( @Override public Optional> visit( LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryType) { - // reject geometry so it does not silently fall through to the generic binary writer; the - // geospatial value path is a separate follow-up - throw new UnsupportedOperationException("Cannot write geometry value to Parquet"); + // geometry values are pure WKB stored in a BINARY column + return Optional.of(ParquetValueWriters.byteBuffers(desc)); } @Override public Optional> visit( LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyType) { - // reject geography so it does not silently fall through to the generic binary writer; the - // geospatial value path is a separate follow-up - throw new UnsupportedOperationException("Cannot write geography value to Parquet"); + // geography values are pure WKB stored in a BINARY column + return Optional.of(ParquetValueWriters.byteBuffers(desc)); } @Override diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestInternalParquet.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestInternalParquet.java index a53ac8972528..f193cab5c552 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestInternalParquet.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestInternalParquet.java @@ -56,6 +56,11 @@ protected boolean supportsVariant() { return true; } + @Override + protected boolean supportsGeospatial() { + return true; + } + @Override protected void writeAndValidate(Schema schema) throws IOException { List expected = RandomInternalData.generate(schema, 100, 1376L); diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java index 0c17a871677b..00891b507eef 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java @@ -20,7 +20,6 @@ import static org.apache.iceberg.parquet.ParquetWritingTestUtils.createTempFile; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.IOException; import java.nio.ByteBuffer; @@ -56,6 +55,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.RandomUtil; import org.apache.iceberg.variants.Variant; import org.apache.iceberg.variants.VariantMetadata; import org.apache.iceberg.variants.VariantTestUtil; @@ -104,36 +104,62 @@ public void testDataWriter() throws IOException { } @Test - public void testGeospatialWriteIsRejected() { - Schema geometrySchema = + public void testGeospatialRoundTrip() throws IOException { + Schema schema = new Schema( Types.NestedField.required(1, "id", Types.LongType.get()), - Types.NestedField.optional(2, "geom", Types.GeometryType.crs84())); - assertThatThrownBy( - () -> - Parquet.writeData(Files.localOutput(createTempFile(temp))) - .schema(geometrySchema) - .createWriterFunc(GenericParquetWriter::create) - .overwrite() - .withSpec(PartitionSpec.unpartitioned()) - .build()) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("Cannot write geometry value to Parquet"); - - Schema geographySchema = - new Schema( - Types.NestedField.required(1, "id", Types.LongType.get()), - Types.NestedField.optional(2, "geog", Types.GeographyType.crs84())); - assertThatThrownBy( - () -> - Parquet.writeData(Files.localOutput(createTempFile(temp))) - .schema(geographySchema) - .createWriterFunc(GenericParquetWriter::create) - .overwrite() - .withSpec(PartitionSpec.unpartitioned()) - .build()) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("Cannot write geography value to Parquet"); + Types.NestedField.optional(2, "geom", Types.GeometryType.crs84()), + Types.NestedField.optional(3, "geog", Types.GeographyType.crs84())); + + GenericRecord record = GenericRecord.create(schema); + List geoRecords = + ImmutableList.of( + record.copy( + ImmutableMap.of("id", 1L, "geom", wkbPoint(30, 10), "geog", wkbPoint(-5, 40))), + // geog is left null + record.copy(ImmutableMap.of("id", 2L, "geom", wkbPoint(0, 0))), + // both geo columns are left null + record.copy(ImmutableMap.of("id", 3L))); + + OutputFile file = Files.localOutput(createTempFile(temp)); + DataWriter dataWriter = + Parquet.writeData(file) + .schema(schema) + .createWriterFunc(GenericParquetWriter::create) + .overwrite() + .withSpec(PartitionSpec.unpartitioned()) + .build(); + try (dataWriter) { + for (Record geoRecord : geoRecords) { + dataWriter.write(geoRecord); + } + } + + assertThat(dataWriter.toDataFile().recordCount()).isEqualTo(geoRecords.size()); + + List writtenRecords; + try (CloseableIterable reader = + Parquet.read(file.toInputFile()) + .project(schema) + .createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema)) + .build()) { + writtenRecords = Lists.newArrayList(reader); + } + + assertThat(writtenRecords).hasSameSizeAs(geoRecords); + for (int i = 0; i < geoRecords.size(); i++) { + assertThat(writtenRecords.get(i).getField("id")).isEqualTo(geoRecords.get(i).getField("id")); + assertThat(writtenRecords.get(i).getField("geom")) + .as("geometry WKB should round-trip unchanged") + .isEqualTo(geoRecords.get(i).getField("geom")); + assertThat(writtenRecords.get(i).getField("geog")) + .as("geography WKB should round-trip unchanged") + .isEqualTo(geoRecords.get(i).getField("geog")); + } + } + + private static ByteBuffer wkbPoint(double xCoord, double yCoord) { + return ByteBuffer.wrap(RandomUtil.wkbPoint(xCoord, yCoord)); } private void testDataWriter(Schema schema, VariantShreddingFunction variantShreddingFunc) From 035fc1e405d8902b272f4c5af42a24919d2ea625 Mon Sep 17 00:00:00 2001 From: Xin Huang <42597328+huan233usc@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:37:38 -0700 Subject: [PATCH 70/73] Spark 4.1: Map geo Spark types (#16851) --- .../spark/PruneColumnsWithoutReordering.java | 32 +++++ .../apache/iceberg/spark/SparkTypeToType.java | 33 +++++ .../apache/iceberg/spark/TypeToSparkType.java | 38 ++++++ .../iceberg/spark/TestSparkSchemaUtil.java | 123 ++++++++++++++++++ 4 files changed, 226 insertions(+) diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java index f4323f1c0350..90fa68594ade 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java @@ -37,7 +37,10 @@ import org.apache.spark.sql.types.DateType$; import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.DoubleType$; +import org.apache.spark.sql.types.EdgeInterpolationAlgorithm; import org.apache.spark.sql.types.FloatType$; +import org.apache.spark.sql.types.GeographyType; +import org.apache.spark.sql.types.GeometryType; import org.apache.spark.sql.types.IntegerType$; import org.apache.spark.sql.types.LongType$; import org.apache.spark.sql.types.MapType; @@ -224,6 +227,33 @@ public Type primitive(Type.PrimitiveType primitive) { requestedDecimal.precision(), decimal.precision()); break; + case GEOMETRY: + Types.GeometryType geometry = (Types.GeometryType) primitive; + GeometryType requestedGeometry = (GeometryType) current; + Preconditions.checkArgument( + geometry.crs().equalsIgnoreCase(requestedGeometry.crs()), + "Cannot project geometry with incompatible CRS: %s != %s", + requestedGeometry.crs(), + geometry.crs()); + break; + case GEOGRAPHY: + Types.GeographyType geography = (Types.GeographyType) primitive; + GeographyType requestedGeography = (GeographyType) current; + Preconditions.checkArgument( + geography.crs().equalsIgnoreCase(requestedGeography.crs()), + "Cannot project geography with incompatible CRS: %s != %s", + requestedGeography.crs(), + geography.crs()); + // algorithm() is EdgeAlgorithm on Iceberg and EdgeInterpolationAlgorithm on Spark, so + // translate the table's algorithm into Spark's type and compare within one type system. + EdgeInterpolationAlgorithm tableAlgorithm = + TypeToSparkType.convertAlgorithm(geography.algorithm()); + Preconditions.checkArgument( + tableAlgorithm == requestedGeography.algorithm(), + "Cannot project geography with incompatible edge algorithm: %s != %s", + requestedGeography.algorithm(), + tableAlgorithm); + break; default: } @@ -244,6 +274,8 @@ public Type primitive(Type.PrimitiveType primitive) { .put(TypeID.STRING, ImmutableSet.of(StringType$.class)) .put(TypeID.FIXED, ImmutableSet.of(BinaryType$.class)) .put(TypeID.BINARY, ImmutableSet.of(BinaryType$.class)) + .put(TypeID.GEOMETRY, ImmutableSet.of(GeometryType.class)) + .put(TypeID.GEOGRAPHY, ImmutableSet.of(GeographyType.class)) .put(TypeID.UNKNOWN, ImmutableSet.of(NullType$.class)) .buildOrThrow(); } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java index 54ad899ade77..6ce085a1b7a1 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java @@ -19,7 +19,9 @@ package org.apache.iceberg.spark; import java.util.List; +import java.util.Locale; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.types.ArrayType; @@ -31,7 +33,10 @@ import org.apache.spark.sql.types.DateType; import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.DoubleType; +import org.apache.spark.sql.types.EdgeInterpolationAlgorithm; import org.apache.spark.sql.types.FloatType; +import org.apache.spark.sql.types.GeographyType; +import org.apache.spark.sql.types.GeometryType; import org.apache.spark.sql.types.IntegerType; import org.apache.spark.sql.types.LongType; import org.apache.spark.sql.types.MapType; @@ -162,10 +167,38 @@ public Type atomic(DataType atomic) { ((DecimalType) atomic).precision(), ((DecimalType) atomic).scale()); } else if (atomic instanceof BinaryType) { return Types.BinaryType.get(); + } else if (atomic instanceof GeometryType) { + GeometryType geometry = (GeometryType) atomic; + if (geometry.isMixedSrid()) { + throw new UnsupportedOperationException( + "Cannot convert Spark geometry with mixed SRID to Iceberg"); + } + return Types.GeometryType.of(geometry.crs()); + } else if (atomic instanceof GeographyType) { + GeographyType geography = (GeographyType) atomic; + if (geography.isMixedSrid()) { + throw new UnsupportedOperationException( + "Cannot convert Spark geography with mixed SRID to Iceberg"); + } + return Types.GeographyType.of(geography.crs(), convertAlgorithm(geography.algorithm())); } else if (atomic instanceof NullType) { return Types.UnknownType.get(); } throw new UnsupportedOperationException("Not a supported type: " + atomic.catalogString()); } + + // Translates Spark's edge-interpolation algorithm to Iceberg's, mirroring + // TypeToSparkType#convertAlgorithm. Spark supports only the spherical algorithm today; anything + // else is rejected loudly rather than silently defaulting, so a new Spark algorithm surfaces here + // instead of being dropped. + private static EdgeAlgorithm convertAlgorithm(EdgeInterpolationAlgorithm algorithm) { + switch (algorithm.toString().toUpperCase(Locale.ROOT)) { + case "SPHERICAL": + return EdgeAlgorithm.SPHERICAL; + default: + throw new UnsupportedOperationException( + "Iceberg does not support Spark geography edge algorithm: " + algorithm); + } + } } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java index 09c89bbba813..dc077937577c 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java @@ -22,6 +22,7 @@ import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -33,7 +34,11 @@ import org.apache.spark.sql.types.DateType$; import org.apache.spark.sql.types.DecimalType$; import org.apache.spark.sql.types.DoubleType$; +import org.apache.spark.sql.types.EdgeInterpolationAlgorithm; +import org.apache.spark.sql.types.EdgeInterpolationAlgorithm.SPHERICAL$; import org.apache.spark.sql.types.FloatType$; +import org.apache.spark.sql.types.GeographyType$; +import org.apache.spark.sql.types.GeometryType$; import org.apache.spark.sql.types.IntegerType$; import org.apache.spark.sql.types.LongType$; import org.apache.spark.sql.types.MapType$; @@ -52,6 +57,9 @@ class TypeToSparkType extends TypeUtil.SchemaVisitor { public static final String METADATA_COL_ATTR_KEY = "__metadata_col"; + // Spark's only edge-interpolation algorithm. + private static final EdgeInterpolationAlgorithm SPARK_SPHERICAL = SPHERICAL$.MODULE$; + @Override public DataType schema(Schema schema, DataType structType) { return structType; @@ -145,6 +153,10 @@ public DataType primitive(Type.PrimitiveType primitive) { return BinaryType$.MODULE$; case BINARY: return BinaryType$.MODULE$; + case GEOMETRY: + return geometryType((Types.GeometryType) primitive); + case GEOGRAPHY: + return geographyType((Types.GeographyType) primitive); case DECIMAL: Types.DecimalType decimal = (Types.DecimalType) primitive; return DecimalType$.MODULE$.apply(decimal.precision(), decimal.scale()); @@ -156,6 +168,32 @@ public DataType primitive(Type.PrimitiveType primitive) { } } + private DataType geometryType(Types.GeometryType geometry) { + // The spec lets a geometry CRS be any string identifying a CRS, but Spark recognizes only a + // fixed set; a CRS Spark cannot resolve throws SparkIllegalArgumentException (an + // IllegalArgumentException). + return GeometryType$.MODULE$.apply(geometry.crs()); + } + + private DataType geographyType(Types.GeographyType geography) { + // The spec requires a geography CRS to be geographic; Spark recognizes only OGC:CRS84, so any + // other CRS throws SparkIllegalArgumentException (an IllegalArgumentException). + return GeographyType$.MODULE$.apply(geography.crs(), convertAlgorithm(geography.algorithm())); + } + + // Translates Iceberg's edge-interpolation algorithm to Spark's. Spark supports only the spherical + // algorithm (the Iceberg default); every other algorithm is rejected with a clear error. Shared + // with PruneColumnsWithoutReordering, which compares algorithms across the two type systems. + static EdgeInterpolationAlgorithm convertAlgorithm(EdgeAlgorithm algorithm) { + switch (algorithm) { + case SPHERICAL: + return SPARK_SPHERICAL; + default: + throw new UnsupportedOperationException( + "Spark does not support geography edge algorithm: " + algorithm); + } + } + private Metadata fieldMetadata(int fieldId) { if (MetadataColumns.metadataFieldIds().contains(fieldId)) { return new MetadataBuilder().putBoolean(METADATA_COL_ATTR_KEY, true).build(); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java index d5f407a715ef..f107167a7a24 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java @@ -20,6 +20,7 @@ import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -29,12 +30,19 @@ import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.catalyst.expressions.AttributeReference; import org.apache.spark.sql.catalyst.expressions.MetadataAttribute; import org.apache.spark.sql.catalyst.types.DataTypeUtils; import org.apache.spark.sql.catalyst.util.ResolveDefaultColumnsUtils$; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.GeographyType; +import org.apache.spark.sql.types.GeographyType$; +import org.apache.spark.sql.types.GeometryType; +import org.apache.spark.sql.types.GeometryType$; import org.apache.spark.sql.types.Metadata; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; @@ -93,6 +101,121 @@ public void testSchemaConversionWithMetaDataColumnSchema() { } } + @Test + public void testGeospatialTypeConversion() { + // a default-CRS geometry round-trips through the null <-> OGC:CRS84 normalization + Types.GeometryType defaultGeometry = Types.GeometryType.crs84(); + DataType sparkDefaultGeometry = SparkSchemaUtil.convert(defaultGeometry); + assertThat(sparkDefaultGeometry).isInstanceOf(GeometryType.class); + assertThat(((GeometryType) sparkDefaultGeometry).crs()) + .isEqualTo(Types.GeometryType.DEFAULT_CRS); + assertThat(SparkSchemaUtil.convert(sparkDefaultGeometry)).isEqualTo(defaultGeometry); + + // a default-CRS geography round-trips through the null <-> OGC:CRS84 normalization + Types.GeographyType defaultGeography = Types.GeographyType.crs84(); + DataType sparkDefaultGeography = SparkSchemaUtil.convert(defaultGeography); + assertThat(sparkDefaultGeography).isInstanceOf(GeographyType.class); + assertThat(((GeographyType) sparkDefaultGeography).crs()) + .isEqualTo(Types.GeographyType.DEFAULT_CRS); + assertThat(SparkSchemaUtil.convert(sparkDefaultGeography)).isEqualTo(defaultGeography); + + Types.GeometryType geometry = Types.GeometryType.of("EPSG:3857"); + DataType sparkGeometry = SparkSchemaUtil.convert(geometry); + assertThat(sparkGeometry).isInstanceOf(GeometryType.class); + assertThat(((GeometryType) sparkGeometry).crs()).isEqualTo("EPSG:3857"); + assertThat(SparkSchemaUtil.convert(sparkGeometry)).isEqualTo(geometry); + + Types.GeographyType geography = Types.GeographyType.of("OGC:CRS84"); + DataType sparkGeography = SparkSchemaUtil.convert(geography); + assertThat(sparkGeography).isInstanceOf(GeographyType.class); + assertThat(((GeographyType) sparkGeography).crs()).isEqualTo("OGC:CRS84"); + assertThat(SparkSchemaUtil.convert(sparkGeography)).isEqualTo(geography); + + assertThat(SparkSchemaUtil.convert(GeometryType$.MODULE$.apply("EPSG:3857"))) + .isEqualTo(geometry); + assertThat(SparkSchemaUtil.convert(GeographyType$.MODULE$.apply("OGC:CRS84"))) + .isEqualTo(geography); + + Types.GeographyType vincentyGeography = + Types.GeographyType.of("OGC:CRS84", EdgeAlgorithm.VINCENTY); + assertThatThrownBy(() -> SparkSchemaUtil.convert(vincentyGeography)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Spark does not support geography edge algorithm: vincenty"); + } + + @Test + public void testGeospatialCrsUnsupportedBySparkIsRejected() { + // Iceberg permits any non-empty CRS, but Spark only recognizes a fixed set; a CRS Spark cannot + // resolve is rejected by Spark's SparkIllegalArgumentException (an IllegalArgumentException). + Types.GeometryType geometry = Types.GeometryType.of("EPSG:4269"); + assertThatThrownBy(() -> SparkSchemaUtil.convert(geometry)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("EPSG:4269"); + + Types.GeographyType geography = Types.GeographyType.of("EPSG:4269"); + assertThatThrownBy(() -> SparkSchemaUtil.convert(geography)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("EPSG:4269"); + } + + @Test + public void testGeospatialMixedSridIsRejected() { + // Spark models a mixed-SRID column with a sentinel CRS ("SRID:ANY"); Iceberg requires a + // concrete column-level CRS, so a mixed-SRID Spark type must be rejected, not persisted. + DataType mixedGeometry = GeometryType$.MODULE$.apply("ANY"); + assertThatThrownBy(() -> SparkSchemaUtil.convert(mixedGeometry)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot convert Spark geometry with mixed SRID to Iceberg"); + + DataType mixedGeography = GeographyType$.MODULE$.apply("ANY"); + assertThatThrownBy(() -> SparkSchemaUtil.convert(mixedGeography)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot convert Spark geography with mixed SRID to Iceberg"); + } + + @Test + public void testPruneGeospatialTypes() { + Schema schema = + new Schema( + optional(1, "geom", Types.GeometryType.of("EPSG:3857")), + optional(2, "geog", Types.GeographyType.of("OGC:CRS84")), + optional(3, "id", Types.LongType.get())); + + StructType requestedType = SparkSchemaUtil.convert(schema); + Schema pruned = SparkSchemaUtil.prune(schema, requestedType); + + assertThat(pruned.asStruct()).isEqualTo(schema.asStruct()); + } + + @Test + public void testPruneGeospatialTypeWithIncompatibleRequestedType() { + Schema schema = new Schema(optional(1, "geom", Types.GeometryType.of("EPSG:3857"))); + + // requesting a non-geo Spark type for a geometry column must be rejected + StructType incompatibleType = new StructType().add("geom", DataTypes.BinaryType, true); + + assertThatThrownBy(() -> SparkSchemaUtil.prune(schema, incompatibleType)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot project") + .hasMessageContaining("incompatible type"); + } + + @Test + public void testPruneGeospatialTypeWithIncompatibleCrs() { + // Spark normally copies the type from the table schema, so this defends against a requested geo + // type whose CRS disagrees with the table's rather than a case reachable through normal reads. + // Both CRS values must be ones Spark recognizes (else construction fails first); OGC:CRS84 and + // EPSG:3857 are both valid and differ, exercising the prune-time CRS check. + Schema schema = new Schema(optional(1, "geom", Types.GeometryType.of("EPSG:3857"))); + + StructType mismatchedCrs = + new StructType().add("geom", GeometryType$.MODULE$.apply("OGC:CRS84"), true); + + assertThatThrownBy(() -> SparkSchemaUtil.prune(schema, mismatchedCrs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot project geometry with incompatible CRS"); + } + @Test public void testSchemaConversionWithOnlyWriteDefault() { Schema schema = From 49b89a8c59d7d88290c6e925f41b65fa9fc99a88 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Fri, 3 Jul 2026 08:33:17 +0200 Subject: [PATCH 71/73] Flink: Hold back equality delete converter watermark until completion (#17038) This PR fixes the CI test flakiness seen in: ``` TestConvertEqualityDeletesE2E > testConvertEqualityDeletesE2E(String) > [1] staging FAILED org.opentest4j.AssertionFailedError: expected: 2L but was: 1L at app//org.apache.iceberg.flink.maintenance.api.TestConvertEqualityDeletesE2E.lambda$testConvertEqualityDeletesE2E$1(TestConvertEqualityDeletesE2E.java:127) ``` e.g.: https://github.com/apache/iceberg/actions/runs/28499724055/job/84473933155?pr=16293 Flink's table maintenance framework maintains a lock to prevent concurrent execution of maintenance tasks. The component responsible for removing the lock (LockRemover) releases the task lock once a watermark reaches it past the task's start timestamp. EqualityConvertPlanner emits phase watermarks in the middle of its execution, and the committer forwarded them, so the lock was released before the run completed. The maintenance framework then started a next cycle concurrently, and both re-processed the same uncommitted staging snapshot. The overlapping commits conflicted, and one advanced its commit marker while dropping its deletion vector, causing the test flakiness. The solution is to forward watermarks from the committer only after it finishes the conversion cycle, to ensure mutual exclusive execution of the maintenance tasks. --- .../operator/EqualityConvertCommitter.java | 44 ++++++++++-------- .../TestEqualityConvertCommitter.java | 45 +++++++++++++++++++ 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java index b3e8fb4d8938..95c9ccd5de1c 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -50,6 +50,12 @@ * *

The commit is gated on the plan's done-timestamp watermark. * + *

Watermarks are forwarded only after the cycle commits, never mid-cycle. The {@link + * LockRemover} releases the maintenance lock once a watermark past the trigger's start epoch + * reaches it. The planner emits phase watermarks in the middle of a cycle; forwarding those would + * release the lock before this commit, letting the TriggerManager start a concurrent cycle that + * re-processes the same uncommitted staging snapshot. + * *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for * the Aggregator. @@ -139,27 +145,29 @@ public void processElement2(StreamRecord record) { @Override public void processWatermark(Watermark mark) throws Exception { - if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { - try { - commitIfNeeded(); - } catch (Exception e) { - LOG.error( - "Failed to commit equality convert result for table {} task {}", - tableName, - taskName, - e); - output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); - errorCounter.inc(); - } - - // Emit Trigger for the Aggregator (even on error or no-op). - output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + if (planResult == null || mark.getTimestamp() < planResult.doneTimestamp()) { + // Hold back watermarks until the cycle commits so the LockRemover keeps the maintenance lock + // for the whole cycle. Forwarding the planner's mid-cycle phase watermarks will release the + // lock early and could let the next trigger run a concurrent cycle on the same staging + // snapshot. + return; + } - bufferedResults.clear(); - planResult = null; + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); } - // Always forward watermarks to prevent stalling downstream. + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + super.processWatermark(mark); } diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java index f74570c878db..54f0b1758d90 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.List; +import java.util.stream.Collectors; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; @@ -83,6 +84,43 @@ void commitsDataFilesToMainBranch() throws Exception { } } + @Test + void holdsBackWatermarkUntilCommit() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 2, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 2)); + + // A phase watermark before the plan's done timestamp must not be forwarded: it would let the + // LockRemover release the maintenance lock before this cycle commits. + harness.processBothWatermarks(new Watermark(doneTs - 1)); + assertThat(harness.extractOutputValues()).isEmpty(); + assertThat(watermarks(harness)).isEmpty(); + + // The done-timestamp watermark commits the cycle; the watermark is forwarded only now. + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(watermarks(harness)).containsExactly(new Watermark(doneTs)); + } + } + @Test void skipsCommitForEmptyCycle() throws Exception { Table table = createTable(3, FileFormat.PARQUET); @@ -476,6 +514,13 @@ void removesRewrittenStagingDvOnSharedBranch() throws Exception { SnapshotRef.MAIN_BRANCH)); } + private static List watermarks(TwoInputStreamOperatorTestHarness harness) { + return harness.getOutput().stream() + .filter(Watermark.class::isInstance) + .map(Watermark.class::cast) + .collect(Collectors.toList()); + } + private static List deletesForDataFile(Table table, String dataFilePath) { List deletes = Lists.newArrayList(); for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { From 11706a286bc94f852aa6046d2d1ec6c3b85039a5 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Fri, 3 Jul 2026 12:13:24 +0200 Subject: [PATCH 72/73] Flink: Backport: Hold back equality delete converter watermark until completion (#17038) (#17067) --- .../operator/EqualityConvertCommitter.java | 44 ++++++++++-------- .../TestEqualityConvertCommitter.java | 45 +++++++++++++++++++ .../operator/EqualityConvertCommitter.java | 44 ++++++++++-------- .../TestEqualityConvertCommitter.java | 45 +++++++++++++++++++ 4 files changed, 142 insertions(+), 36 deletions(-) diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java index b3e8fb4d8938..95c9ccd5de1c 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -50,6 +50,12 @@ * *

The commit is gated on the plan's done-timestamp watermark. * + *

Watermarks are forwarded only after the cycle commits, never mid-cycle. The {@link + * LockRemover} releases the maintenance lock once a watermark past the trigger's start epoch + * reaches it. The planner emits phase watermarks in the middle of a cycle; forwarding those would + * release the lock before this commit, letting the TriggerManager start a concurrent cycle that + * re-processes the same uncommitted staging snapshot. + * *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for * the Aggregator. @@ -139,27 +145,29 @@ public void processElement2(StreamRecord record) { @Override public void processWatermark(Watermark mark) throws Exception { - if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { - try { - commitIfNeeded(); - } catch (Exception e) { - LOG.error( - "Failed to commit equality convert result for table {} task {}", - tableName, - taskName, - e); - output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); - errorCounter.inc(); - } - - // Emit Trigger for the Aggregator (even on error or no-op). - output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + if (planResult == null || mark.getTimestamp() < planResult.doneTimestamp()) { + // Hold back watermarks until the cycle commits so the LockRemover keeps the maintenance lock + // for the whole cycle. Forwarding the planner's mid-cycle phase watermarks will release the + // lock early and could let the next trigger run a concurrent cycle on the same staging + // snapshot. + return; + } - bufferedResults.clear(); - planResult = null; + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); } - // Always forward watermarks to prevent stalling downstream. + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + super.processWatermark(mark); } diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java index f74570c878db..54f0b1758d90 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.List; +import java.util.stream.Collectors; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; @@ -83,6 +84,43 @@ void commitsDataFilesToMainBranch() throws Exception { } } + @Test + void holdsBackWatermarkUntilCommit() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 2, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 2)); + + // A phase watermark before the plan's done timestamp must not be forwarded: it would let the + // LockRemover release the maintenance lock before this cycle commits. + harness.processBothWatermarks(new Watermark(doneTs - 1)); + assertThat(harness.extractOutputValues()).isEmpty(); + assertThat(watermarks(harness)).isEmpty(); + + // The done-timestamp watermark commits the cycle; the watermark is forwarded only now. + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(watermarks(harness)).containsExactly(new Watermark(doneTs)); + } + } + @Test void skipsCommitForEmptyCycle() throws Exception { Table table = createTable(3, FileFormat.PARQUET); @@ -476,6 +514,13 @@ void removesRewrittenStagingDvOnSharedBranch() throws Exception { SnapshotRef.MAIN_BRANCH)); } + private static List watermarks(TwoInputStreamOperatorTestHarness harness) { + return harness.getOutput().stream() + .filter(Watermark.class::isInstance) + .map(Watermark.class::cast) + .collect(Collectors.toList()); + } + private static List deletesForDataFile(Table table, String dataFilePath) { List deletes = Lists.newArrayList(); for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java index b3e8fb4d8938..95c9ccd5de1c 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -50,6 +50,12 @@ * *

The commit is gated on the plan's done-timestamp watermark. * + *

Watermarks are forwarded only after the cycle commits, never mid-cycle. The {@link + * LockRemover} releases the maintenance lock once a watermark past the trigger's start epoch + * reaches it. The planner emits phase watermarks in the middle of a cycle; forwarding those would + * release the lock before this commit, letting the TriggerManager start a concurrent cycle that + * re-processes the same uncommitted staging snapshot. + * *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for * the Aggregator. @@ -139,27 +145,29 @@ public void processElement2(StreamRecord record) { @Override public void processWatermark(Watermark mark) throws Exception { - if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { - try { - commitIfNeeded(); - } catch (Exception e) { - LOG.error( - "Failed to commit equality convert result for table {} task {}", - tableName, - taskName, - e); - output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); - errorCounter.inc(); - } - - // Emit Trigger for the Aggregator (even on error or no-op). - output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + if (planResult == null || mark.getTimestamp() < planResult.doneTimestamp()) { + // Hold back watermarks until the cycle commits so the LockRemover keeps the maintenance lock + // for the whole cycle. Forwarding the planner's mid-cycle phase watermarks will release the + // lock early and could let the next trigger run a concurrent cycle on the same staging + // snapshot. + return; + } - bufferedResults.clear(); - planResult = null; + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); } - // Always forward watermarks to prevent stalling downstream. + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + super.processWatermark(mark); } diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java index f74570c878db..54f0b1758d90 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.List; +import java.util.stream.Collectors; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; @@ -83,6 +84,43 @@ void commitsDataFilesToMainBranch() throws Exception { } } + @Test + void holdsBackWatermarkUntilCommit() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 2, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 2)); + + // A phase watermark before the plan's done timestamp must not be forwarded: it would let the + // LockRemover release the maintenance lock before this cycle commits. + harness.processBothWatermarks(new Watermark(doneTs - 1)); + assertThat(harness.extractOutputValues()).isEmpty(); + assertThat(watermarks(harness)).isEmpty(); + + // The done-timestamp watermark commits the cycle; the watermark is forwarded only now. + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(watermarks(harness)).containsExactly(new Watermark(doneTs)); + } + } + @Test void skipsCommitForEmptyCycle() throws Exception { Table table = createTable(3, FileFormat.PARQUET); @@ -476,6 +514,13 @@ void removesRewrittenStagingDvOnSharedBranch() throws Exception { SnapshotRef.MAIN_BRANCH)); } + private static List watermarks(TwoInputStreamOperatorTestHarness harness) { + return harness.getOutput().stream() + .filter(Watermark.class::isInstance) + .map(Watermark.class::cast) + .collect(Collectors.toList()); + } + private static List deletesForDataFile(Table table, String dataFilePath) { List deletes = Lists.newArrayList(); for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { From 8af1339dbdb2bfdb7061bdd19b6946232d638a0b Mon Sep 17 00:00:00 2001 From: Christof Schablinski Date: Fri, 19 Jun 2026 11:40:33 +0200 Subject: [PATCH 73/73] #15824 Include sources and Javadoc of subprojects of spark-runtime-4.0_2.13 into its source and javadoc-jar --- deploy.gradle | 12 +++++++++--- spark/v3.5/build.gradle | 39 +++++++++++++++++++++++++++++++++++++++ spark/v4.0/build.gradle | 39 +++++++++++++++++++++++++++++++++++++++ spark/v4.1/build.gradle | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) diff --git a/deploy.gradle b/deploy.gradle index 65836bf1b3f1..fc380daa71e6 100644 --- a/deploy.gradle +++ b/deploy.gradle @@ -47,15 +47,17 @@ subprojects { from sourceSets.test.output } + def sourceArtifactTask = tasks.findByName('shadowSourceJar') ?: sourceJar + artifacts { - archives sourceJar + archives sourceArtifactTask archives javadocJar archives testJar testArtifacts testJar } // add LICENSE and NOTICE - [jar, sourceJar, javadocJar, testJar].each { task -> + [jar, sourceArtifactTask, javadocJar, testJar].each { task -> task.dependsOn rootProject.tasks.buildInfo task.from("${rootDir}/build") { include 'iceberg-build.properties' @@ -81,8 +83,12 @@ subprojects { } else { project.shadow.component(it) } + if (tasks.matching({task -> task.name == 'shadowSourceJar'}).isEmpty()) { + artifact sourceJar + } else { + artifact shadowSourceJar + } - artifact sourceJar artifact javadocJar artifact testJar diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle index 68bdb1c21a98..964b8af9a52b 100644 --- a/spark/v3.5/build.gradle +++ b/spark/v3.5/build.gradle @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle index 8557115cedd1..fce8483aeb20 100644 --- a/spark/v4.0/build.gradle +++ b/spark/v4.0/build.gradle @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v4.1/build.gradle b/spark/v4.1/build.gradle index 36e7f2672262..9b6cac429efe 100644 --- a/spark/v4.1/build.gradle +++ b/spark/v4.1/build.gradle @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" }