feat: adopt partition summary

Author: zhjwpku

src/iceberg/expression/literal.cc

@@ -504,6 +504,17 @@ bool Literal::IsAboveMax() const { return std::holds_alternative<AboveMax>(value
 
 bool Literal::IsNull() const { return std::holds_alternative<std::monostate>(value_); }
 
+bool Literal::IsNan() const {
+  if (type_->type_id() == TypeId::kFloat) {
+    auto val = std::get<float>(value_);
+    return std::isnan(val);
+  } else if (type_->type_id() == TypeId::kDouble) {
+    auto val = std::get<double>(value_);
+    return std::isnan(val);
+  }
+  return false;
+}
+
 // LiteralCaster implementation
 
 Result<Literal> LiteralCaster::CastTo(const Literal& literal,

src/iceberg/expression/literal.h

@@ -150,10 +150,14 @@ class ICEBERG_EXPORT Literal : public util::Formattable {
   /// \return true if this literal represents a BelowMin value, false otherwise
   bool IsBelowMin() const;
 
-  /// Check if this literal is null.
+  /// \brief Check if this literal is null.
   /// \return true if this literal is null, false otherwise
   bool IsNull() const;
 
+  /// \brief Check if this literal is NaN (Not a Number).
+  /// \return true if this literal is NaN, false otherwise
+  bool IsNan() const;
+
   std::string ToString() const override;
 
  private:

src/iceberg/manifest_adapter.h

@@ -25,7 +25,6 @@
 #include <memory>
 #include <optional>
 #include <unordered_map>
-#include <unordered_set>
 #include <vector>
 
 #include "iceberg/arrow_c_data.h"
@@ -95,6 +94,13 @@ class ICEBERG_EXPORT ManifestEntryAdapter : public ManifestAdapter {
   std::shared_ptr<Schema> current_schema_;
   std::shared_ptr<Schema> manifest_schema_;
   const ManifestContent content_;
+  int32_t add_files_count_{0};
+  int32_t existing_files_count_{0};
+  int32_t delete_files_count_{0};
+  int64_t add_rows_count_{0L};
+  int64_t existing_rows_count_{0L};
+  int64_t delete_rows_count_{0L};
+  std::optional<int64_t> min_data_sequence_number_{std::nullopt};
 };
 
 /// \brief Adapter for appending a list of `ManifestFile`s to an `ArrowArray`.

src/iceberg/manifest_entry.h

@@ -57,15 +57,6 @@ ICEBERG_EXPORT constexpr Result<ManifestStatus> ManifestStatusFromInt(
   }
 }
 
-enum class ManifestContent {
-  kData = 0,
-  kDeletes = 1,
-};
-
-ICEBERG_EXPORT constexpr std::string_view ToString(ManifestContent content) noexcept;
-ICEBERG_EXPORT constexpr Result<ManifestContent> ManifestContentFromString(
-    std::string_view str) noexcept;
-
 /// \brief DataFile carries data file path, partition tuple, metrics, ...
 struct ICEBERG_EXPORT DataFile {
   /// \brief Content of a data file
@@ -315,6 +306,11 @@ struct ICEBERG_EXPORT ManifestEntry {
   inline static const SchemaField kFileSequenceNumber =
       SchemaField::MakeOptional(4, "file_sequence_number", iceberg::int64());
 
+  /// \brief Check if this manifest entry is deleted.
+  constexpr bool IsAlive() const {
+    return status == ManifestStatus::kAdded || status == ManifestStatus::kExisting;
+  }
+
   bool operator==(const ManifestEntry& other) const;
 
   static std::shared_ptr<StructType> TypeFromPartitionType(

src/iceberg/manifest_list.h

@@ -25,7 +25,6 @@
 #include <optional>
 #include <string>
 #include <string_view>
-#include <utility>
 
 #include "iceberg/iceberg_export.h"
 #include "iceberg/partition_spec.h"
@@ -72,17 +71,17 @@ struct ICEBERG_EXPORT PartitionFieldSummary {
   static const StructType& Type();
 };
 
+/// \brief The type of files tracked by the manifest, either data or delete files; 0 for
+/// all v1 manifests
+enum class ManifestContent {
+  /// The manifest content is data.
+  kData = 0,
+  /// The manifest content is deletes.
+  kDeletes = 1,
+};
+
 /// \brief Entry in a manifest list.
 struct ICEBERG_EXPORT ManifestFile {
-  /// \brief The type of files tracked by the manifest, either data or delete files; 0 for
-  /// all v1 manifests
-  enum class Content {
-    /// The manifest content is data.
-    kData = 0,
-    /// The manifest content is deletes.
-    kDeletes = 1,
-  };
-
   /// Field id: 500
   /// Location of the manifest file
   std::string manifest_path;
@@ -96,7 +95,7 @@ struct ICEBERG_EXPORT ManifestFile {
   /// Field id: 517
   /// The type of files tracked by the manifest, either data or delete files; 0 for all v1
   /// manifests
-  Content content = Content::kData;
+  ManifestContent content = ManifestContent::kData;
   /// Field id: 515
   /// The sequence number when the manifest was added to the table; use 0 when reading v1
   /// manifest lists
@@ -218,21 +217,20 @@ struct ICEBERG_EXPORT ManifestList {
 };
 
 /// \brief Get the relative manifest content type name
-ICEBERG_EXPORT constexpr std::string_view ToString(ManifestFile::Content type) noexcept {
+ICEBERG_EXPORT constexpr std::string_view ToString(ManifestContent type) noexcept {
   switch (type) {
-    case ManifestFile::Content::kData:
+    case ManifestContent::kData:
       return "data";
-    case ManifestFile::Content::kDeletes:
+    case ManifestContent::kDeletes:
       return "deletes";
   }
-  std::unreachable();
 }
 
 /// \brief Get the relative manifest content type from name
-ICEBERG_EXPORT constexpr Result<ManifestFile::Content> ManifestFileContentFromString(
+ICEBERG_EXPORT constexpr Result<ManifestContent> ManifestContentFromString(
     std::string_view str) noexcept {
-  if (str == "data") return ManifestFile::Content::kData;
-  if (str == "deletes") return ManifestFile::Content::kDeletes;
+  if (str == "data") return ManifestContent::kData;
+  if (str == "deletes") return ManifestContent::kDeletes;
   return InvalidArgument("Invalid manifest content type: {}", str);
 }
 

src/iceberg/manifest_reader_internal.cc

@@ -237,7 +237,7 @@ Result<std::vector<ManifestFile>> ParseManifestList(ArrowSchema* schema,
         break;
       case ManifestFileField::kContent:
         PARSE_PRIMITIVE_FIELD(manifest_files[row_idx].content, view_of_column,
-                              ManifestFile::Content);
+                              ManifestContent);
         break;
       case ManifestFileField::kSequenceNumber:
         PARSE_PRIMITIVE_FIELD(manifest_files[row_idx].sequence_number, view_of_column,

src/iceberg/partition_summary_internal.cc

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/partition_summary_internal.h"
+
+#include "iceberg/expression/literal.h"
+
+namespace iceberg {
+
+Status PartitionFieldStats::Update(const Literal& value) {
+  if (type_->type_id() != value.type()->type_id()) {
+    return InvalidArgument("value is not compatible with type");
+  }
+
+  if (value.IsNull()) {
+    contains_null_ = true;
+    return {};
+  }
+
+  if (value.IsNan()) {
+    contains_nan_ = true;
+    return {};
+  }
+
+  if (!lower_bound_ || value < *lower_bound_) {
+    lower_bound_ = value;
+  }
+  if (!upper_bound_ || value > *upper_bound_) {
+    upper_bound_ = value;
+  }
+  return {};
+}
+
+}  // namespace iceberg

src/iceberg/partition_summary_internal.h

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include "iceberg/expression/literal.h"
+#include "iceberg/result.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class PartitionFieldStats {
+ public:
+  explicit PartitionFieldStats(const std::shared_ptr<PrimitiveType>& type)
+      : type_(type) {}
+
+  Status Update(const Literal& value);
+
+  PartitionFieldSummary Finish() const {
+    PartitionFieldSummary summary;
+    summary.contains_null = contains_null_;
+    summary.contains_nan = contains_nan_;
+    if (lower_bound_) {
+      summary.lower_bound = lower_bound_->Serialize().value();
+    }
+    if (upper_bound_) {
+      summary.upper_bound = upper_bound_->Serialize().value();
+    }
+    return summary;
+  }
+
+ private:
+  std::shared_ptr<PrimitiveType> type_{nullptr};
+  bool contains_null_{false};
+  bool contains_nan_{false};
+  std::optional<Literal> lower_bound_;
+  std::optional<Literal> upper_bound_;
+};
+
+class PartitionSummary {};
+
+}  // namespace iceberg

src/iceberg/test/manifest_list_reader_writer_test.cc

@@ -64,7 +64,7 @@ class ManifestListReaderWriterTestBase : public TempFileTestBase {
     for (const auto& manifest : manifest_files) {
       ASSERT_EQ(manifest.partition_spec_id, 0);
       ASSERT_TRUE(manifest.partitions.empty());
-      ASSERT_EQ(manifest.content, ManifestFile::Content::kData);
+      ASSERT_EQ(manifest.content, ManifestContent::kData);
     }
   }
 
@@ -242,7 +242,7 @@ class ManifestListReaderWriterV2Test : public ManifestListReaderWriterTestBase {
       manifest_file.manifest_path = test_dir_prefix + paths[i];
       manifest_file.manifest_length = file_size[i];
       manifest_file.partition_spec_id = 0;
-      manifest_file.content = ManifestFile::Content::kData;
+      manifest_file.content = ManifestContent::kData;
       manifest_file.sequence_number = 4 - i;
       manifest_file.min_sequence_number = 4 - i;
       manifest_file.added_snapshot_id = snapshot_id[i];
@@ -282,7 +282,7 @@ class ManifestListReaderWriterV2Test : public ManifestListReaderWriterTestBase {
       manifest_file.manifest_path = test_dir_prefix + paths[i];
       manifest_file.manifest_length = file_size[i];
       manifest_file.partition_spec_id = 0;
-      manifest_file.content = ManifestFile::Content::kData;
+      manifest_file.content = ManifestContent::kData;
       manifest_file.sequence_number = 4 - i;
       manifest_file.min_sequence_number = 4 - i;
       manifest_file.added_snapshot_id = snapshot_id[i];

src/iceberg/test/manifest_list_versions_test.cc

@@ -59,7 +59,7 @@ const static auto kTestManifest = ManifestFile{
     .manifest_path = kPath,
     .manifest_length = kLength,
     .partition_spec_id = kSpecId,
-    .content = ManifestFile::Content::kData,
+    .content = ManifestContent::kData,
     .sequence_number = kSeqNum,
     .min_sequence_number = kMinSeqNum,
     .added_snapshot_id = kSnapshotId,
@@ -78,7 +78,7 @@ const static auto kDeleteManifest = ManifestFile{
     .manifest_path = kPath,
     .manifest_length = kLength,
     .partition_spec_id = kSpecId,
-    .content = ManifestFile::Content::kDeletes,
+    .content = ManifestContent::kDeletes,
     .sequence_number = kSeqNum,
     .min_sequence_number = kMinSeqNum,
     .added_snapshot_id = kSnapshotId,
@@ -227,7 +227,7 @@ TEST_F(TestManifestListVersions, TestV1Write) {
   EXPECT_EQ(manifest.manifest_path, kPath);
   EXPECT_EQ(manifest.manifest_length, kLength);
   EXPECT_EQ(manifest.partition_spec_id, kSpecId);
-  EXPECT_EQ(manifest.content, ManifestFile::Content::kData);
+  EXPECT_EQ(manifest.content, ManifestContent::kData);
   EXPECT_EQ(manifest.added_snapshot_id, kSnapshotId);
   EXPECT_EQ(manifest.added_files_count, kAddedFiles);
   EXPECT_EQ(manifest.existing_files_count, kExistingFiles);
@@ -247,7 +247,7 @@ TEST_F(TestManifestListVersions, TestV2Write) {
   EXPECT_EQ(manifest.manifest_path, kPath);
   EXPECT_EQ(manifest.manifest_length, kLength);
   EXPECT_EQ(manifest.partition_spec_id, kSpecId);
-  EXPECT_EQ(manifest.content, ManifestFile::Content::kData);
+  EXPECT_EQ(manifest.content, ManifestContent::kData);
   EXPECT_EQ(manifest.sequence_number, kSeqNum);
   EXPECT_EQ(manifest.min_sequence_number, kMinSeqNum);
   EXPECT_EQ(manifest.added_snapshot_id, kSnapshotId);
@@ -266,7 +266,7 @@ TEST_F(TestManifestListVersions, TestV3Write) {
   EXPECT_EQ(manifest.manifest_path, kPath);
   EXPECT_EQ(manifest.manifest_length, kLength);
   EXPECT_EQ(manifest.partition_spec_id, kSpecId);
-  EXPECT_EQ(manifest.content, ManifestFile::Content::kData);
+  EXPECT_EQ(manifest.content, ManifestContent::kData);
   EXPECT_EQ(manifest.sequence_number, kSeqNum);
   EXPECT_EQ(manifest.min_sequence_number, kMinSeqNum);
   EXPECT_EQ(manifest.added_snapshot_id, kSnapshotId);
@@ -292,7 +292,7 @@ TEST_F(TestManifestListVersions, TestV3WriteFirstRowIdAssignment) {
   EXPECT_EQ(manifest.manifest_path, kPath);
   EXPECT_EQ(manifest.manifest_length, kLength);
   EXPECT_EQ(manifest.partition_spec_id, kSpecId);
-  EXPECT_EQ(manifest.content, ManifestFile::Content::kData);
+  EXPECT_EQ(manifest.content, ManifestContent::kData);
   EXPECT_EQ(manifest.sequence_number, kSeqNum);
   EXPECT_EQ(manifest.min_sequence_number, kMinSeqNum);
   EXPECT_EQ(manifest.added_snapshot_id, kSnapshotId);
@@ -323,7 +323,7 @@ TEST_F(TestManifestListVersions, TestV3WriteMixedRowIdAssignment) {
     EXPECT_EQ(manifest.manifest_path, kPath);
     EXPECT_EQ(manifest.manifest_length, kLength);
     EXPECT_EQ(manifest.partition_spec_id, kSpecId);
-    EXPECT_EQ(manifest.content, ManifestFile::Content::kData);
+    EXPECT_EQ(manifest.content, ManifestContent::kData);
     EXPECT_EQ(manifest.sequence_number, kSeqNum);
     EXPECT_EQ(manifest.min_sequence_number, kMinSeqNum);
     EXPECT_EQ(manifest.added_snapshot_id, kSnapshotId);
@@ -442,7 +442,7 @@ TEST_F(TestManifestListVersions, TestManifestsPartitionSummary) {
       .manifest_path = kPath,
       .manifest_length = kLength,
       .partition_spec_id = kSpecId,
-      .content = ManifestFile::Content::kData,
+      .content = ManifestContent::kData,
       .sequence_number = kSeqNum,
       .min_sequence_number = kMinSeqNum,
       .added_snapshot_id = kSnapshotId,