feat: adopt partition summary

Author: zhjwpku

src/iceberg/CMakeLists.txt

@@ -39,6 +39,7 @@ set(ICEBERG_SOURCES
     name_mapping.cc
     partition_field.cc
     partition_spec.cc
+    partition_summary_internal.cc
     row/arrow_array_wrapper.cc
     row/manifest_wrapper.cc
     schema.cc

src/iceberg/expression/literal.cc

@@ -504,6 +504,17 @@ bool Literal::IsAboveMax() const { return std::holds_alternative<AboveMax>(value
 
 bool Literal::IsNull() const { return std::holds_alternative<std::monostate>(value_); }
 
+bool Literal::IsNan() const {
+  if (type_->type_id() == TypeId::kFloat) {
+    auto val = std::get<float>(value_);
+    return std::isnan(val);
+  } else if (type_->type_id() == TypeId::kDouble) {
+    auto val = std::get<double>(value_);
+    return std::isnan(val);
+  }
+  return false;
+}
+
 // LiteralCaster implementation
 
 Result<Literal> LiteralCaster::CastTo(const Literal& literal,

src/iceberg/expression/literal.h

@@ -150,10 +150,14 @@ class ICEBERG_EXPORT Literal : public util::Formattable {
   /// \return true if this literal represents a BelowMin value, false otherwise
   bool IsBelowMin() const;
 
-  /// Check if this literal is null.
+  /// \brief Check if this literal is null.
   /// \return true if this literal is null, false otherwise
   bool IsNull() const;
 
+  /// \brief Check if this literal is NaN (Not a Number).
+  /// \return true if this literal is NaN, false otherwise
+  bool IsNan() const;
+
   std::string ToString() const override;
 
  private:

src/iceberg/manifest_adapter.cc

@@ -26,6 +26,7 @@
 #include "iceberg/arrow/nanoarrow_status_internal.h"
 #include "iceberg/manifest_entry.h"
 #include "iceberg/manifest_list.h"
+#include "iceberg/partition_summary_internal.h"
 #include "iceberg/result.h"
 #include "iceberg/schema.h"
 #include "iceberg/schema_internal.h"
@@ -248,6 +249,7 @@ Status ManifestEntryAdapter::AppendDataFile(
     ArrowArray* array, const std::shared_ptr<StructType>& data_file_type,
     const DataFile& file) {
   auto fields = data_file_type->fields();
+  ICEBERG_RETURN_UNEXPECTED(partition_summary_->Update(file.partition));
   for (size_t i = 0; i < fields.size(); i++) {
     const auto& field = fields[i];
     auto child_array = array->children[i];

src/iceberg/manifest_adapter.h

@@ -25,7 +25,6 @@
 #include <memory>
 #include <optional>
 #include <unordered_map>
-#include <unordered_set>
 #include <vector>
 
 #include "iceberg/arrow_c_data.h"
@@ -95,6 +94,14 @@ class ICEBERG_EXPORT ManifestEntryAdapter : public ManifestAdapter {
   std::shared_ptr<Schema> current_schema_;
   std::shared_ptr<Schema> manifest_schema_;
   const ManifestContent content_;
+  int32_t add_files_count_{0};
+  int32_t existing_files_count_{0};
+  int32_t delete_files_count_{0};
+  int64_t add_rows_count_{0L};
+  int64_t existing_rows_count_{0L};
+  int64_t delete_rows_count_{0L};
+  std::optional<int64_t> min_data_sequence_number_{std::nullopt};
+  std::unique_ptr<PartitionSummary> partition_summary_;
 };
 
 /// \brief Adapter for appending a list of `ManifestFile`s to an `ArrowArray`.

src/iceberg/manifest_entry.h

@@ -57,15 +57,6 @@ ICEBERG_EXPORT constexpr Result<ManifestStatus> ManifestStatusFromInt(
   }
 }
 
-enum class ManifestContent {
-  kData = 0,
-  kDeletes = 1,
-};
-
-ICEBERG_EXPORT constexpr std::string_view ToString(ManifestContent content) noexcept;
-ICEBERG_EXPORT constexpr Result<ManifestContent> ManifestContentFromString(
-    std::string_view str) noexcept;
-
 /// \brief DataFile carries data file path, partition tuple, metrics, ...
 struct ICEBERG_EXPORT DataFile {
   /// \brief Content of a data file
@@ -315,6 +306,11 @@ struct ICEBERG_EXPORT ManifestEntry {
   inline static const SchemaField kFileSequenceNumber =
       SchemaField::MakeOptional(4, "file_sequence_number", iceberg::int64());
 
+  /// \brief Check if this manifest entry is deleted.
+  constexpr bool IsAlive() const {
+    return status == ManifestStatus::kAdded || status == ManifestStatus::kExisting;
+  }
+
   bool operator==(const ManifestEntry& other) const;
 
   static std::shared_ptr<StructType> TypeFromPartitionType(

src/iceberg/manifest_list.h

@@ -25,7 +25,6 @@
 #include <optional>
 #include <string>
 #include <string_view>
-#include <utility>
 
 #include "iceberg/iceberg_export.h"
 #include "iceberg/partition_spec.h"
@@ -72,17 +71,17 @@ struct ICEBERG_EXPORT PartitionFieldSummary {
   static const StructType& Type();
 };
 
+/// \brief The type of files tracked by the manifest, either data or delete files; 0 for
+/// all v1 manifests
+enum class ManifestContent {
+  /// The manifest content is data.
+  kData = 0,
+  /// The manifest content is deletes.
+  kDeletes = 1,
+};
+
 /// \brief Entry in a manifest list.
 struct ICEBERG_EXPORT ManifestFile {
-  /// \brief The type of files tracked by the manifest, either data or delete files; 0 for
-  /// all v1 manifests
-  enum class Content {
-    /// The manifest content is data.
-    kData = 0,
-    /// The manifest content is deletes.
-    kDeletes = 1,
-  };
-
   /// Field id: 500
   /// Location of the manifest file
   std::string manifest_path;
@@ -96,7 +95,7 @@ struct ICEBERG_EXPORT ManifestFile {
   /// Field id: 517
   /// The type of files tracked by the manifest, either data or delete files; 0 for all v1
   /// manifests
-  Content content = Content::kData;
+  ManifestContent content = ManifestContent::kData;
   /// Field id: 515
   /// The sequence number when the manifest was added to the table; use 0 when reading v1
   /// manifest lists
@@ -218,21 +217,20 @@ struct ICEBERG_EXPORT ManifestList {
 };
 
 /// \brief Get the relative manifest content type name
-ICEBERG_EXPORT constexpr std::string_view ToString(ManifestFile::Content type) noexcept {
+ICEBERG_EXPORT constexpr std::string_view ToString(ManifestContent type) noexcept {
   switch (type) {
-    case ManifestFile::Content::kData:
+    case ManifestContent::kData:
       return "data";
-    case ManifestFile::Content::kDeletes:
+    case ManifestContent::kDeletes:
       return "deletes";
   }
-  std::unreachable();
 }
 
 /// \brief Get the relative manifest content type from name
-ICEBERG_EXPORT constexpr Result<ManifestFile::Content> ManifestFileContentFromString(
+ICEBERG_EXPORT constexpr Result<ManifestContent> ManifestContentFromString(
     std::string_view str) noexcept {
-  if (str == "data") return ManifestFile::Content::kData;
-  if (str == "deletes") return ManifestFile::Content::kDeletes;
+  if (str == "data") return ManifestContent::kData;
+  if (str == "deletes") return ManifestContent::kDeletes;
   return InvalidArgument("Invalid manifest content type: {}", str);
 }
 

src/iceberg/manifest_reader_internal.cc

@@ -237,7 +237,7 @@ Result<std::vector<ManifestFile>> ParseManifestList(ArrowSchema* schema,
         break;
       case ManifestFileField::kContent:
         PARSE_PRIMITIVE_FIELD(manifest_files[row_idx].content, view_of_column,
-                              ManifestFile::Content);
+                              ManifestContent);
         break;
       case ManifestFileField::kSequenceNumber:
         PARSE_PRIMITIVE_FIELD(manifest_files[row_idx].sequence_number, view_of_column,

src/iceberg/meson.build

@@ -61,6 +61,7 @@ iceberg_sources = files(
     'name_mapping.cc',
     'partition_field.cc',
     'partition_spec.cc',
+    'partition_summary_internal.cc',
     'row/arrow_array_wrapper.cc',
     'row/manifest_wrapper.cc',
     'schema.cc',

src/iceberg/partition_summary_internal.cc

@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/partition_summary_internal.h"
+
+#include <memory>
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/result.h"
+
+namespace iceberg {
+
+Status PartitionFieldStats::Update(const Literal& value) {
+  if (type_->type_id() != value.type()->type_id()) {
+    return InvalidArgument("value is not compatible with type");
+  }
+
+  if (value.IsNull()) {
+    contains_null_ = true;
+    return {};
+  }
+
+  if (value.IsNan()) {
+    contains_nan_ = true;
+    return {};
+  }
+
+  if (!lower_bound_ || value < *lower_bound_) {
+    lower_bound_ = value;
+  }
+  if (!upper_bound_ || value > *upper_bound_) {
+    upper_bound_ = value;
+  }
+  return {};
+}
+
+PartitionFieldSummary PartitionFieldStats::Finish() const {
+  PartitionFieldSummary summary;
+  summary.contains_null = contains_null_;
+  summary.contains_nan = contains_nan_;
+  if (lower_bound_) {
+    summary.lower_bound = lower_bound_->Serialize().value();
+  }
+  if (upper_bound_) {
+    summary.upper_bound = upper_bound_->Serialize().value();
+  }
+  return summary;
+}
+
+Status PartitionSummary::Update(const std::vector<Literal>& partition) {
+  if (partition.size() != field_stats_.size()) {
+    return InvalidArgument("partition size does not match field stats size");
+  }
+
+  for (size_t i = 0; i < partition.size(); i++) {
+    ICEBERG_RETURN_UNEXPECTED(field_stats_[i].Update(partition[i]));
+  }
+  return {};
+}
+
+std::vector<PartitionFieldSummary> PartitionSummary::Summaries() const {
+  std::vector<PartitionFieldSummary> summaries;
+  for (const auto& field_stat : field_stats_) {
+    summaries.push_back(field_stat.Finish());
+  }
+  return summaries;
+}
+
+Result<std::unique_ptr<PartitionSummary>> PartitionSummary::Make(
+    const StructType& partition_type) {
+  std::vector<PartitionFieldStats> field_stats;
+  for (const auto& field : partition_type.fields()) {
+    field_stats.emplace_back(field.type());
+  }
+  return std::unique_ptr<PartitionSummary>(new PartitionSummary(std::move(field_stats)));
+}
+
+}  // namespace iceberg