feat: add rewrite manifests

Author: zhjwpku

src/iceberg/CMakeLists.txt

@@ -108,6 +108,7 @@ set(ICEBERG_SOURCES
     update/merging_snapshot_update.cc
     update/overwrite_files.cc
     update/pending_update.cc
+    update/rewrite_manifests.cc
     update/row_delta.cc
     update/set_snapshot.cc
     update/snapshot_manager.cc

src/iceberg/manifest/rolling_manifest_writer.cc

@@ -54,6 +54,13 @@ Status RollingManifestWriter::WriteExistingEntry(
   return {};
 }
 
+Status RollingManifestWriter::WriteExistingEntry(const ManifestEntry& entry) {
+  ICEBERG_ASSIGN_OR_RAISE(auto* writer, CurrentWriter());
+  ICEBERG_RETURN_UNEXPECTED(writer->WriteExistingEntry(entry));
+  current_file_rows_++;
+  return {};
+}
+
 Status RollingManifestWriter::WriteDeletedEntry(
     std::shared_ptr<DataFile> file, int64_t data_sequence_number,
     std::optional<int64_t> file_sequence_number) {

src/iceberg/manifest/rolling_manifest_writer.h

@@ -76,6 +76,9 @@ class ICEBERG_EXPORT RollingManifestWriter {
                             int64_t data_sequence_number,
                             std::optional<int64_t> file_sequence_number = std::nullopt);
 
+  /// \brief Add an existing entry while preserving snapshot and sequence fields.
+  Status WriteExistingEntry(const ManifestEntry& entry);
+
   /// \brief Add a delete entry for a file.
   ///
   /// \param file a deleted data file

src/iceberg/manifest/v3_metadata.cc

@@ -194,13 +194,20 @@ Status ManifestFileAdapterV3::Init() {
 }
 
 Status ManifestFileAdapterV3::Append(const ManifestFile& file) {
-  ICEBERG_RETURN_UNEXPECTED(AppendInternal(file));
   if (WrapFirstRowId(file)) {
     if (!next_row_id_.has_value()) {
       return InvalidManifestList("Missing next-row-id for file: {}", file.manifest_path);
     }
-    next_row_id_ = next_row_id_.value() + file.existing_rows_count.value_or(0) +
-                   file.added_rows_count.value_or(0);
+    if (!file.existing_rows_count.has_value() || !file.added_rows_count.has_value()) {
+      return InvalidManifestList("Missing row counts for file: {}", file.manifest_path);
+    }
+  }
+
+  ICEBERG_RETURN_UNEXPECTED(AppendInternal(file));
+
+  if (WrapFirstRowId(file)) {
+    next_row_id_ = next_row_id_.value() + file.existing_rows_count.value() +
+                   file.added_rows_count.value();
   }
   return {};
 }

src/iceberg/meson.build

@@ -133,6 +133,7 @@ iceberg_sources = files(
     'update/merging_snapshot_update.cc',
     'update/overwrite_files.cc',
     'update/pending_update.cc',
+    'update/rewrite_manifests.cc',
     'update/row_delta.cc',
     'update/set_snapshot.cc',
     'update/snapshot_manager.cc',

src/iceberg/table.cc

@@ -36,6 +36,7 @@
 #include "iceberg/update/fast_append.h"
 #include "iceberg/update/merge_append.h"
 #include "iceberg/update/overwrite_files.h"
+#include "iceberg/update/rewrite_manifests.h"
 #include "iceberg/update/row_delta.h"
 #include "iceberg/update/set_snapshot.h"
 #include "iceberg/update/snapshot_manager.h"
@@ -245,6 +246,12 @@ Result<std::shared_ptr<OverwriteFiles>> Table::NewOverwrite() {
   return OverwriteFiles::Make(name().name, std::move(ctx));
 }
 
+Result<std::shared_ptr<RewriteManifests>> Table::NewRewriteManifests() {
+  ICEBERG_ASSIGN_OR_RAISE(
+      auto ctx, TransactionContext::Make(shared_from_this(), TransactionKind::kUpdate));
+  return RewriteManifests::Make(name().name, std::move(ctx));
+}
+
 Result<std::shared_ptr<UpdateStatistics>> Table::NewUpdateStatistics() {
   ICEBERG_ASSIGN_OR_RAISE(
       auto ctx, TransactionContext::Make(shared_from_this(), TransactionKind::kUpdate));
@@ -360,6 +367,10 @@ Result<std::shared_ptr<OverwriteFiles>> StaticTable::NewOverwrite() {
   return NotSupported("Cannot create an overwrite for a static table");
 }
 
+Result<std::shared_ptr<RewriteManifests>> StaticTable::NewRewriteManifests() {
+  return NotSupported("Cannot create a rewrite manifests for a static table");
+}
+
 Result<std::shared_ptr<SnapshotManager>> StaticTable::NewSnapshotManager() {
   return NotSupported("Cannot create a snapshot manager for a static table");
 }

src/iceberg/table.h

@@ -188,6 +188,9 @@ class ICEBERG_EXPORT Table : public std::enable_shared_from_this<Table> {
   /// \brief Create a new OverwriteFiles to overwrite data files and commit the changes.
   virtual Result<std::shared_ptr<OverwriteFiles>> NewOverwrite();
 
+  /// \brief Create a new RewriteManifests to rewrite manifest layout.
+  virtual Result<std::shared_ptr<RewriteManifests>> NewRewriteManifests();
+
   /// \brief Create a new SnapshotManager to manage snapshots and snapshot references.
   virtual Result<std::shared_ptr<SnapshotManager>> NewSnapshotManager();
 
@@ -263,6 +266,8 @@ class ICEBERG_EXPORT StaticTable : public Table {
 
   Result<std::shared_ptr<OverwriteFiles>> NewOverwrite() override;
 
+  Result<std::shared_ptr<RewriteManifests>> NewRewriteManifests() override;
+
   Result<std::shared_ptr<SnapshotManager>> NewSnapshotManager() override;
 
  private:

src/iceberg/test/CMakeLists.txt

@@ -234,6 +234,7 @@ if(ICEBERG_BUILD_BUNDLE)
                    merge_append_test.cc
                    merging_snapshot_update_test.cc
                    name_mapping_update_test.cc
+                   rewrite_manifests_test.cc
                    row_delta_test.cc
                    snapshot_manager_test.cc
                    transaction_test.cc

src/iceberg/test/merge_append_test.cc

@@ -88,10 +88,9 @@ class MergeAppendTestBase : public UpdateTestBase {
     auto metadata_location = std::format("{}/metadata/00001-{}.metadata.json",
                                          table_location_, Uuid::GenerateV7().ToString());
     ICEBERG_UNWRAP_OR_FAIL(
-        auto metadata, ReadTableMetadataFromResource("TableMetadataV2ValidMinimal.json"));
+        auto metadata, ReadTableMetadataFromResource("TableMetadataV3ValidMinimal.json"));
     metadata->format_version = format_version;
     metadata->location = table_location_;
-    metadata->next_row_id = TableMetadata::kInitialRowId;
 
     ASSERT_THAT(TableMetadataUtil::Write(*file_io_, metadata_location, *metadata),
                 IsOk());