feat(puffin): add basic data structures and constants

Add the foundational types for Puffin file format support:
- Blob, BlobMetadata, FileMetadata structs
- PuffinCompressionCodec enum with codec name conversion
- StandardBlobTypes and StandardPuffinProperties constants
- ToString functions for all types
- 24 unit tests covering all public APIs

Author: xuan.zhao

src/iceberg/CMakeLists.txt

@@ -62,6 +62,8 @@ set(ICEBERG_SOURCES
     partition_field.cc
     partition_spec.cc
     partition_summary.cc
+    puffin/file_metadata.cc
+    puffin/puffin_compression_codec.cc
     row/arrow_array_wrapper.cc
     row/manifest_wrapper.cc
     row/partition_values.cc
@@ -166,6 +168,7 @@ add_subdirectory(catalog)
 add_subdirectory(data)
 add_subdirectory(expression)
 add_subdirectory(manifest)
+add_subdirectory(puffin)
 add_subdirectory(row)
 add_subdirectory(update)
 add_subdirectory(util)

src/iceberg/meson.build

@@ -80,6 +80,8 @@ iceberg_sources = files(
     'partition_field.cc',
     'partition_spec.cc',
     'partition_summary.cc',
+    'puffin/file_metadata.cc',
+    'puffin/puffin_compression_codec.cc',
     'row/arrow_array_wrapper.cc',
     'row/manifest_wrapper.cc',
     'row/partition_values.cc',
@@ -221,6 +223,7 @@ install_headers(
 subdir('catalog')
 subdir('expression')
 subdir('manifest')
+subdir('puffin')
 subdir('row')
 subdir('update')
 subdir('util')

src/iceberg/puffin/CMakeLists.txt

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+iceberg_install_all_headers(iceberg/puffin)

src/iceberg/puffin/file_metadata.cc

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/puffin/file_metadata.h"
+
+#include <format>
+
+#include "iceberg/util/formatter_internal.h"
+
+namespace iceberg::puffin {
+
+std::string ToString(const Blob& blob) {
+  std::string repr = "Blob[";
+  std::format_to(std::back_inserter(repr), "type='{}',inputFields={},", blob.type,
+                 blob.input_fields);
+  std::format_to(std::back_inserter(repr), "snapshotId={},sequenceNumber={},",
+                 blob.snapshot_id, blob.sequence_number);
+  std::format_to(std::back_inserter(repr), "dataSize={}", blob.data.size());
+  if (blob.requested_compression.has_value()) {
+    std::format_to(std::back_inserter(repr), ",requestedCompression={}",
+                   iceberg::puffin::ToString(*blob.requested_compression));
+  }
+  if (!blob.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", blob.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+std::string ToString(const BlobMetadata& blob_metadata) {
+  std::string repr = "BlobMetadata[";
+  std::format_to(std::back_inserter(repr), "type='{}',inputFields={},",
+                 blob_metadata.type, blob_metadata.input_fields);
+  std::format_to(std::back_inserter(repr), "snapshotId={},sequenceNumber={},",
+                 blob_metadata.snapshot_id, blob_metadata.sequence_number);
+  std::format_to(std::back_inserter(repr), "offset={},length={}", blob_metadata.offset,
+                 blob_metadata.length);
+  if (!blob_metadata.compression_codec.empty()) {
+    std::format_to(std::back_inserter(repr), ",compressionCodec='{}'",
+                   blob_metadata.compression_codec);
+  }
+  if (!blob_metadata.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", blob_metadata.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+std::string ToString(const FileMetadata& file_metadata) {
+  std::string repr = "FileMetadata[";
+  std::format_to(std::back_inserter(repr), "blobs=[");
+  for (size_t i = 0; i < file_metadata.blobs.size(); ++i) {
+    if (i > 0) {
+      std::format_to(std::back_inserter(repr), ",");
+    }
+    std::format_to(std::back_inserter(repr), "{}", ToString(file_metadata.blobs[i]));
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  if (!file_metadata.properties.empty()) {
+    std::format_to(std::back_inserter(repr), ",properties={}", file_metadata.properties);
+  }
+  std::format_to(std::back_inserter(repr), "]");
+  return repr;
+}
+
+}  // namespace iceberg::puffin

src/iceberg/puffin/file_metadata.h

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/puffin/file_metadata.h
+/// Data structures for Puffin files: Blob, BlobMetadata, and FileMetadata.
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/puffin/puffin_compression_codec.h"
+
+namespace iceberg::puffin {
+
+/// \brief A blob in a Puffin file.
+struct ICEBERG_EXPORT Blob {
+  /// \brief Type of the blob. See StandardBlobTypes for known types.
+  std::string type;
+  /// \brief List of field IDs the blob was computed for.
+  std::vector<int32_t> input_fields;
+  /// \brief ID of the Iceberg table's snapshot the blob was computed from.
+  int64_t snapshot_id;
+  /// \brief Sequence number of the Iceberg table's snapshot the blob was computed from.
+  int64_t sequence_number;
+  /// \brief The uncompressed blob data.
+  std::vector<uint8_t> data;
+  /// \brief Requested compression codec. If not set, the writer's default will be used.
+  std::optional<PuffinCompressionCodec> requested_compression;
+  /// \brief Additional properties of the blob.
+  std::unordered_map<std::string, std::string> properties;
+
+  /// \brief Compare two Blobs for equality.
+  friend bool operator==(const Blob& lhs, const Blob& rhs) = default;
+};
+
+/// \brief Returns a string representation of a Blob.
+ICEBERG_EXPORT std::string ToString(const Blob& blob);
+
+/// \brief Metadata about a blob stored in a Puffin file.
+struct ICEBERG_EXPORT BlobMetadata {
+  /// \brief Type of the blob. See StandardBlobTypes for known types.
+  std::string type;
+  /// \brief List of field IDs the blob was computed for.
+  std::vector<int32_t> input_fields;
+  /// \brief ID of the Iceberg table's snapshot the blob was computed from.
+  int64_t snapshot_id;
+  /// \brief Sequence number of the Iceberg table's snapshot the blob was computed from.
+  int64_t sequence_number;
+  /// \brief Offset in the file where the blob data starts.
+  int64_t offset;
+  /// \brief Length of the blob data in the file.
+  int64_t length;
+  /// \brief Compression codec name (e.g. "lz4", "zstd"), or empty if not compressed.
+  std::string compression_codec;
+  /// \brief Additional properties of the blob.
+  std::unordered_map<std::string, std::string> properties;
+
+  /// \brief Compare two BlobMetadatas for equality.
+  friend bool operator==(const BlobMetadata& lhs, const BlobMetadata& rhs) = default;
+};
+
+/// \brief Returns a string representation of a BlobMetadata.
+ICEBERG_EXPORT std::string ToString(const BlobMetadata& blob_metadata);
+
+/// \brief Metadata about a Puffin file.
+struct ICEBERG_EXPORT FileMetadata {
+  /// \brief List of blob metadata for all blobs in the file.
+  std::vector<BlobMetadata> blobs;
+  /// \brief File-level properties.
+  std::unordered_map<std::string, std::string> properties;
+
+  /// \brief Compare two FileMetadatas for equality.
+  friend bool operator==(const FileMetadata& lhs, const FileMetadata& rhs) = default;
+};
+
+/// \brief Returns a string representation of a FileMetadata.
+ICEBERG_EXPORT std::string ToString(const FileMetadata& file_metadata);
+
+}  // namespace iceberg::puffin

src/iceberg/puffin/meson.build

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+install_headers(
+    ['file_metadata.h', 'puffin_compression_codec.h', 'types.h'],
+    subdir: 'iceberg/puffin',
+)

src/iceberg/puffin/puffin_compression_codec.cc

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/puffin/puffin_compression_codec.h"
+
+#include <format>
+#include <utility>
+
+namespace iceberg::puffin {
+
+namespace {
+constexpr std::string_view kLz4CodecName = "lz4";
+constexpr std::string_view kZstdCodecName = "zstd";
+}  // namespace
+
+std::string_view CodecName(PuffinCompressionCodec codec) {
+  switch (codec) {
+    case PuffinCompressionCodec::kNone:
+      return "";
+    case PuffinCompressionCodec::kLz4:
+      return kLz4CodecName;
+    case PuffinCompressionCodec::kZstd:
+      return kZstdCodecName;
+  }
+  std::unreachable();
+}
+
+Result<PuffinCompressionCodec> PuffinCompressionCodecFromName(
+    std::string_view codec_name) {
+  if (codec_name.empty()) {
+    return PuffinCompressionCodec::kNone;
+  }
+  if (codec_name == kLz4CodecName) {
+    return PuffinCompressionCodec::kLz4;
+  }
+  if (codec_name == kZstdCodecName) {
+    return PuffinCompressionCodec::kZstd;
+  }
+  return InvalidArgument("Unknown codec name: {}", codec_name);
+}
+
+std::string ToString(PuffinCompressionCodec codec) {
+  auto name = CodecName(codec);
+  if (!name.empty()) {
+    return std::string(name);
+  }
+  return "none";
+}
+
+}  // namespace iceberg::puffin

src/iceberg/puffin/puffin_compression_codec.h

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/puffin/puffin_compression_codec.h
+/// Compression codec definitions for Puffin files.
+
+#include <string>
+#include <string_view>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/result.h"
+
+namespace iceberg::puffin {
+
+/// \brief Compression codecs supported by Puffin files.
+enum class PuffinCompressionCodec {
+  /// No compression
+  kNone,
+  /// LZ4 single compression frame with content size present
+  kLz4,
+  /// Zstandard single compression frame with content size present
+  kZstd,
+};
+
+/// \brief Get the codec name for a compression codec.
+///
+/// Returns an empty string for kNone.
+ICEBERG_EXPORT std::string_view CodecName(PuffinCompressionCodec codec);
+
+/// \brief Get the compression codec from a codec name.
+///
+/// An empty string is interpreted as kNone.
+ICEBERG_EXPORT Result<PuffinCompressionCodec> PuffinCompressionCodecFromName(
+    std::string_view codec_name);
+
+/// \brief Returns a string representation of a PuffinCompressionCodec.
+ICEBERG_EXPORT std::string ToString(PuffinCompressionCodec codec);
+
+}  // namespace iceberg::puffin