Skip to content

Commit 3c9d13d

Browse files
lishuxushuxu.li
andauthored
feat(update): add OverwriteFiles for overwrite snapshot commits (#741)
Summary: Add a production OverwriteFiles builder that brings iceberg-cpp to semantic parity with Java's BaseOverwriteFiles. It supports explicit file replacement (DeleteFile + AddFile) and range-based replacement (OverwriteByRowFilter + AddFile) with the same family of pre-commit concurrency validations. The builder is a thin subclass of MergingSnapshotUpdate and reuses the existing commit kernel (Apply/summary/retry/cleanup) unchanged. Changes: - New OverwriteFiles class (src/iceberg/update/overwrite_files.{h,cc}) and Table::NewOverwrite() / Transaction::NewOverwrite() entry points. - Builder surface: AddFile, DeleteFile, bulk DeleteFiles, OverwriteByRowFilter, ValidateFromSnapshot, ConflictDetectionFilter, ValidateNoConflictingData, ValidateNoConflictingDeletes, ValidateAddedFilesMatchOverwriteFilter, WithCaseSensitivity. - Validate(): conflict-filter resolution, concurrent add/delete conflict checks, and strict added-file range validation (projection + StrictMetricsEvaluator). - Tests (overwrite_files_test.cc, 45 cases) and CMake/meson wiring. Behavior alignment with Java: - operation() returns append/delete/overwrite from builder content. - Conflict-filter resolution mirrors BaseOverwriteFiles (explicit -> row filter -> AlwaysTrue); replaced-file delete checks honor ConflictDetectionFilter. - Strict added-file validation uses a single DataSpec(), rejecting multi-spec and empty added-file sets. - Deviations: public WithCaseSensitivity (vs caseSensitive) to avoid a protected-name clash; ValidateFromSnapshot rejects negative ids early. --------- Co-authored-by: shuxu.li <shuxu.li@clickzetta.com>
1 parent ed051a7 commit 3c9d13d

20 files changed

Lines changed: 1301 additions & 75 deletions

src/iceberg/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ set(ICEBERG_SOURCES
105105
update/fast_append.cc
106106
update/merge_append.cc
107107
update/merging_snapshot_update.cc
108+
update/overwrite_files.cc
108109
update/pending_update.cc
109110
update/row_delta.cc
110111
update/set_snapshot.cc

src/iceberg/meson.build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ iceberg_sources = files(
130130
'update/fast_append.cc',
131131
'update/merge_append.cc',
132132
'update/merging_snapshot_update.cc',
133+
'update/overwrite_files.cc',
133134
'update/pending_update.cc',
134135
'update/row_delta.cc',
135136
'update/set_snapshot.cc',

src/iceberg/table.cc

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include "iceberg/update/expire_snapshots.h"
3636
#include "iceberg/update/fast_append.h"
3737
#include "iceberg/update/merge_append.h"
38+
#include "iceberg/update/overwrite_files.h"
3839
#include "iceberg/update/row_delta.h"
3940
#include "iceberg/update/set_snapshot.h"
4041
#include "iceberg/update/snapshot_manager.h"
@@ -238,6 +239,12 @@ Result<std::shared_ptr<RowDelta>> Table::NewRowDelta() {
238239
return RowDelta::Make(name().name, std::move(ctx));
239240
}
240241

242+
Result<std::shared_ptr<OverwriteFiles>> Table::NewOverwrite() {
243+
ICEBERG_ASSIGN_OR_RAISE(
244+
auto ctx, TransactionContext::Make(shared_from_this(), TransactionKind::kUpdate));
245+
return OverwriteFiles::Make(name().name, std::move(ctx));
246+
}
247+
241248
Result<std::shared_ptr<UpdateStatistics>> Table::NewUpdateStatistics() {
242249
ICEBERG_ASSIGN_OR_RAISE(
243250
auto ctx, TransactionContext::Make(shared_from_this(), TransactionKind::kUpdate));
@@ -349,6 +356,10 @@ Result<std::shared_ptr<RowDelta>> StaticTable::NewRowDelta() {
349356
return NotSupported("Cannot create a row delta for a static table");
350357
}
351358

359+
Result<std::shared_ptr<OverwriteFiles>> StaticTable::NewOverwrite() {
360+
return NotSupported("Cannot create an overwrite for a static table");
361+
}
362+
352363
Result<std::shared_ptr<SnapshotManager>> StaticTable::NewSnapshotManager() {
353364
return NotSupported("Cannot create a snapshot manager for a static table");
354365
}

src/iceberg/table.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,9 @@ class ICEBERG_EXPORT Table : public std::enable_shared_from_this<Table> {
185185
/// \brief Create a new RowDelta to add rows and row-level deletes.
186186
virtual Result<std::shared_ptr<RowDelta>> NewRowDelta();
187187

188+
/// \brief Create a new OverwriteFiles to overwrite data files and commit the changes.
189+
virtual Result<std::shared_ptr<OverwriteFiles>> NewOverwrite();
190+
188191
/// \brief Create a new SnapshotManager to manage snapshots and snapshot references.
189192
virtual Result<std::shared_ptr<SnapshotManager>> NewSnapshotManager();
190193

@@ -258,6 +261,8 @@ class ICEBERG_EXPORT StaticTable : public Table {
258261

259262
Result<std::shared_ptr<RowDelta>> NewRowDelta() override;
260263

264+
Result<std::shared_ptr<OverwriteFiles>> NewOverwrite() override;
265+
261266
Result<std::shared_ptr<SnapshotManager>> NewSnapshotManager() override;
262267

263268
private:

src/iceberg/test/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,8 @@ if(ICEBERG_BUILD_BUNDLE)
239239
update_properties_test.cc
240240
update_schema_test.cc
241241
update_sort_order_test.cc
242-
update_statistics_test.cc)
242+
update_statistics_test.cc
243+
overwrite_files_test.cc)
243244

244245
add_iceberg_test(data_test
245246
USE_BUNDLE

0 commit comments

Comments
 (0)