feat(puffin): add basic data structures and constants of puffin (#588)

zhaoxuan1994 · web-flow · commit 743c3185848d · 2026-03-24T09:55:45.000+08:00
Add the foundational types for Puffin file format support:
- Blob, BlobMetadata, FileMetadata structs
- PuffinCompressionCodec enum with codec name conversion
- StandardBlobTypes and StandardPuffinProperties constants
- ToString functions for all types
diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt
@@ -62,6 +62,7 @@ set(ICEBERG_SOURCES
     partition_field.cc
     partition_spec.cc
     partition_summary.cc
+    puffin/file_metadata.cc
     row/arrow_array_wrapper.cc
     row/manifest_wrapper.cc
     row/partition_values.cc
@@ -167,6 +168,7 @@ add_subdirectory(catalog)
 add_subdirectory(data)
 add_subdirectory(expression)
 add_subdirectory(manifest)
+add_subdirectory(puffin)
 add_subdirectory(row)
 add_subdirectory(update)
 add_subdirectory(util)
diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build
@@ -80,6 +80,7 @@ iceberg_sources = files(
     'partition_field.cc',
     'partition_spec.cc',
     'partition_summary.cc',
+    'puffin/file_metadata.cc',
     'row/arrow_array_wrapper.cc',
     'row/manifest_wrapper.cc',
     'row/partition_values.cc',
@@ -222,6 +223,7 @@ install_headers(
 subdir('catalog')
 subdir('expression')
 subdir('manifest')
+subdir('puffin')
 subdir('row')
 subdir('update')
 subdir('util')
diff --git a/src/iceberg/puffin/CMakeLists.txt b/src/iceberg/puffin/CMakeLists.txt
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+iceberg_install_all_headers(iceberg/puffin)
diff --git a/src/iceberg/puffin/file_metadata.cc b/src/iceberg/puffin/file_metadata.cc
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/puffin/file_metadata.h"
+
+#include <format>
+#include <utility>
+
+#include "iceberg/util/formatter_internal.h"
+
+namespace iceberg::puffin {
+
+namespace {
+constexpr std::string_view kLz4CodecName = "lz4";
+constexpr std::string_view kZstdCodecName = "zstd";
+}  // namespace
+
+std::string_view CodecName(PuffinCompressionCodec codec) {
+  switch (codec) {
+    case PuffinCompressionCodec::kNone:
+      return "";
+    case PuffinCompressionCodec::kLz4:
+      return kLz4CodecName;
+    case PuffinCompressionCodec::kZstd:
+      return kZstdCodecName;
+  }
+  std::unreachable();
+}
+
+Result<PuffinCompressionCodec> PuffinCompressionCodecFromName(
+    std::string_view codec_name) {
+  if (codec_name.empty()) {
+    return PuffinCompressionCodec::kNone;
+  }
+  if (codec_name == kLz4CodecName) {
+    return PuffinCompressionCodec::kLz4;
+  }
+  if (codec_name == kZstdCodecName) {
+    return PuffinCompressionCodec::kZstd;
+  }
+  return InvalidArgument("Unknown codec name: {}", codec_name);
+}
+
+std::string ToString(PuffinCompressionCodec codec) {
+  return std::string(CodecName(codec));
+}
+
+std::string ToString(const Blob& blob) {
+  std::string repr = "Blob[";
+  std::format_to(std::back_inserter(repr), "type='{}',inputFields={},", blob.type,
+                 blob.input_fields);
+  std::format_to(std::back_inserter(repr), "snapshotId={},sequenceNumber={},",
+                 blob.snapshot_id, blob.sequence_number);
+  std::format_to(std::back_inserter(repr), "dataSize={}", blob.data.size());
+  if (blob.requested_compression.has_value()) {
+    std::format_to(std::back_inserter(repr), ",requestedCompression={}",
+                   ToString(*blob.requested_compression));
+  }
+  if (!blob.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", blob.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+std::string ToString(const BlobMetadata& blob_metadata) {
+  std::string repr = "BlobMetadata[";
+  std::format_to(std::back_inserter(repr), "type='{}',inputFields={},",
+                 blob_metadata.type, blob_metadata.input_fields);
+  std::format_to(std::back_inserter(repr), "snapshotId={},sequenceNumber={},",
+                 blob_metadata.snapshot_id, blob_metadata.sequence_number);
+  std::format_to(std::back_inserter(repr), "offset={},length={}", blob_metadata.offset,
+                 blob_metadata.length);
+  if (!blob_metadata.compression_codec.empty()) {
+    std::format_to(std::back_inserter(repr), ",compressionCodec='{}'",
+                   blob_metadata.compression_codec);
+  }
+  if (!blob_metadata.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", blob_metadata.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+std::string ToString(const FileMetadata& file_metadata) {
+  std::string repr = "FileMetadata[";
+  std::format_to(std::back_inserter(repr), "blobs=[");
+  for (size_t i = 0; i < file_metadata.blobs.size(); ++i) {
+    if (i > 0) {
+      std::format_to(std::back_inserter(repr), ",");
+    }
+    std::format_to(std::back_inserter(repr), "{}", ToString(file_metadata.blobs[i]));
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  if (!file_metadata.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", file_metadata.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+}  // namespace iceberg::puffin
diff --git a/src/iceberg/puffin/file_metadata.h b/src/iceberg/puffin/file_metadata.h
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/puffin/file_metadata.h
+/// Data structures for Puffin files.
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/result.h"
+
+namespace iceberg::puffin {
+
+/// \brief Compression codecs supported by Puffin files.
+enum class PuffinCompressionCodec {
+  kNone,
+  kLz4,
+  kZstd,
+};
+
+ICEBERG_EXPORT std::string_view CodecName(PuffinCompressionCodec codec);
+
+ICEBERG_EXPORT Result<PuffinCompressionCodec> PuffinCompressionCodecFromName(
+    std::string_view codec_name);
+
+ICEBERG_EXPORT std::string ToString(PuffinCompressionCodec codec);
+
+/// \brief Standard blob types defined by the Iceberg specification.
+struct StandardBlobTypes {
+  /// A serialized form of a "compact" Theta sketch produced by the
+  /// Apache DataSketches library.
+  static constexpr std::string_view kApacheDatasketchesThetaV1 =
+      "apache-datasketches-theta-v1";
+
+  /// A serialized deletion vector according to the Iceberg spec.
+  static constexpr std::string_view kDeletionVectorV1 = "deletion-vector-v1";
+};
+
+/// \brief Standard file-level properties for Puffin files.
+struct StandardPuffinProperties {
+  /// Human-readable identification of the application writing the file,
+  /// along with its version.
+  static constexpr std::string_view kCreatedBy = "created-by";
+};
+
+/// \brief A blob in a Puffin file.
+struct ICEBERG_EXPORT Blob {
+  /// See StandardBlobTypes for known types.
+  std::string type;
+  /// Ordered list of field IDs the blob was computed from.
+  std::vector<int32_t> input_fields;
+  /// ID of the Iceberg table's snapshot the blob was computed from.
+  int64_t snapshot_id;
+  /// Sequence number of the Iceberg table's snapshot the blob was computed from.
+  int64_t sequence_number;
+  std::vector<uint8_t> data;
+  /// If not set, the writer's default codec will be used.
+  std::optional<PuffinCompressionCodec> requested_compression;
+  std::unordered_map<std::string, std::string> properties;
+
+  friend bool operator==(const Blob& lhs, const Blob& rhs) = default;
+};
+
+ICEBERG_EXPORT std::string ToString(const Blob& blob);
+
+/// \brief Metadata about a blob stored in a Puffin file footer.
+struct ICEBERG_EXPORT BlobMetadata {
+  /// See StandardBlobTypes for known types.
+  std::string type;
+  /// Ordered list of field IDs the blob was computed from.
+  std::vector<int32_t> input_fields;
+  /// ID of the Iceberg table's snapshot the blob was computed from.
+  int64_t snapshot_id;
+  /// Sequence number of the Iceberg table's snapshot the blob was computed from.
+  int64_t sequence_number;
+  int64_t offset;
+  int64_t length;
+  /// Codec name (e.g. "lz4", "zstd"), or empty if not compressed.
+  std::string compression_codec;
+  std::unordered_map<std::string, std::string> properties;
+
+  friend bool operator==(const BlobMetadata& lhs, const BlobMetadata& rhs) = default;
+};
+
+ICEBERG_EXPORT std::string ToString(const BlobMetadata& blob_metadata);
+
+/// \brief Metadata about a Puffin file.
+struct ICEBERG_EXPORT FileMetadata {
+  std::vector<BlobMetadata> blobs;
+  std::unordered_map<std::string, std::string> properties;
+
+  friend bool operator==(const FileMetadata& lhs, const FileMetadata& rhs) = default;
+};
+
+ICEBERG_EXPORT std::string ToString(const FileMetadata& file_metadata);
+
+}  // namespace iceberg::puffin
diff --git a/src/iceberg/puffin/meson.build b/src/iceberg/puffin/meson.build
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+install_headers(['file_metadata.h'], subdir: 'iceberg/puffin')
