feat: add manifest&manifest list writer

[dongxiao1198]

1 add v1v2v3 writer definition
2 add v1v2v3 metadata wrapper

[HeartLinked]

sequence_number is required for V2 manifest list, so here needs check validation for v2?

[wgtmac]

What about this? ManifestListWriter does not need to be an abstract class. Only ManifestFileAdapter should have subclasses from v1 ~ v4 to deal with different schemas and value-filling logics. @dongxiao1198 @HeartLinked

// Implemented by different versions with different schemas to
// append a list of `ManifestFile`s to an `ArrowArray`.
class ManifestFileAdapter {
 public:
  virtual Status StartAppending() const = 0;
  virtual Status Append(const ManifestFile& file) const = 0;
  virtual Result<ArrowArray> FinishAppending() const = 0;
  int64_t size() const { return size_; }

 private:
  std::shared_ptr<Schema> manifest_list_schema_;
  ArrowSchema schema_;  // converted from manifest_list_schema_
  ArrowArray array_;    // array to append `ManifestFile`s
  int64_t size_;
};

class ICEBERG_EXPORT ManifestListWriter {
 public:
  static Result<std::unique_ptr<ManifestListWriter>> Make(
      std::string_view manifest_list_location, std::shared_ptr<FileIO> file_io,
      const std::unordered_map<std::string, std::string>& meta);

  Status AddAll(const std::vector<ManifestFile>& files) const {
    for (const auto& file : files) {
      Add(file);
    }
    return {};
  }

  Status Add(const ManifestFile& file) const {
    if (adapter_->size() >= kBatchSize) {
      ICEBERG_ASSIGN_OR_RAISE(auto array, adapter_->FinishAppending());
      ICEBERG_RETURN_UNEXPECTED(writer_->Write(array));
      ICEBERG_RETURN_UNEXPECTED(adapter_->StartAppending());
    }
    return adapter_->Append(file);
  }

  Status Close() const;

 private:
  static constexpr int64_t kBatchSize = 1024;
  std::unique_ptr<Writer> writer_;
  std::unique_ptr<ManifestFileAdapter> adapter_;
};

[dongxiao1198]

What about this? ManifestListWriter does not need to be an abstract class. Only ManifestFileAdapter should have subclasses from v1 ~ v4 to deal with different schemas and value-filling logics. @dongxiao1198 @HeartLinked

// Implemented by different versions with different schemas to
// append a list of `ManifestFile`s to an `ArrowArray`.
class ManifestFileAdapter {
 public:
  virtual Status StartAppending() const = 0;
  virtual Status Append(const ManifestFile& file) const = 0;
  virtual Result<ArrowArray> FinishAppending() const = 0;
  int64_t size() const { return size_; }

 private:
  std::shared_ptr<Schema> manifest_list_schema_;
  ArrowSchema schema_;  // converted from manifest_list_schema_
  ArrowArray array_;    // array to append `ManifestFile`s
  int64_t size_;
};

class ICEBERG_EXPORT ManifestListWriter {
 public:
  static Result<std::unique_ptr<ManifestListWriter>> Make(
      std::string_view manifest_list_location, std::shared_ptr<FileIO> file_io,
      const std::unordered_map<std::string, std::string>& meta);

  Status AddAll(const std::vector<ManifestFile>& files) const {
    for (const auto& file : files) {
      Add(file);
    }
    return {};
  }

  Status Add(const ManifestFile& file) const {
    if (adapter_->size() >= kBatchSize) {
      ICEBERG_ASSIGN_OR_RAISE(auto array, adapter_->FinishAppending());
      ICEBERG_RETURN_UNEXPECTED(writer_->Write(array));
      ICEBERG_RETURN_UNEXPECTED(adapter_->StartAppending());
    }
    return adapter_->Append(file);
  }

  Status Close() const;

 private:
  static constexpr int64_t kBatchSize = 1024;
  std::unique_ptr<Writer> writer_;
  std::unique_ptr<ManifestFileAdapter> adapter_;
};

The latest patch add a new header for this adapter at src/iceberg/metadata_adapter.h.
Split writer maker function into 4 independent builder function.

[wgtmac]

+1

Left some nits.