feat: Support UUID values across row and file IO (#790)

closes #314

Author: zhjwpku

src/iceberg/avro/avro_data_util.cc

@@ -35,6 +35,7 @@
 #include "iceberg/avro/avro_schema_util_internal.h"
 #include "iceberg/metadata_columns.h"
 #include "iceberg/schema.h"
+#include "iceberg/schema_internal.h"
 #include "iceberg/schema_util.h"
 #include "iceberg/util/checked_cast.h"
 #include "iceberg/util/macros.h"
@@ -620,7 +621,9 @@ Status ExtractDatumFromArray(const ::arrow::Array& array, int64_t index,
     }
 
     case ::arrow::Type::EXTENSION: {
-      if (array.type()->name() == "arrow.uuid") {
+      const auto& extension_type =
+          internal::checked_cast<const ::arrow::ExtensionType&>(*array.type());
+      if (extension_type.extension_name() == kArrowUuidExtensionName) {
         const auto& extension_array =
             internal::checked_cast<const ::arrow::ExtensionArray&>(array);
         const auto& fixed_array =
@@ -632,7 +635,8 @@ Status ExtractDatumFromArray(const ::arrow::Array& array, int64_t index,
         return {};
       }
 
-      return NotSupported("Unsupported Arrow extension type: {}", array.type()->name());
+      return NotSupported("Unsupported Arrow extension type: {}",
+                          extension_type.extension_name());
     }
 
     case ::arrow::Type::STRUCT: {

src/iceberg/avro/avro_reader.cc

@@ -20,9 +20,11 @@
 #include "iceberg/avro/avro_reader.h"
 
 #include <memory>
+#include <vector>
 
 #include <arrow/array/builder_base.h>
 #include <arrow/c/bridge.h>
+#include <arrow/extension_type.h>
 #include <arrow/filesystem/filesystem.h>
 #include <arrow/record_batch.h>
 #include <arrow/result.h>
@@ -42,6 +44,7 @@
 #include "iceberg/metadata_columns.h"
 #include "iceberg/name_mapping.h"
 #include "iceberg/schema_internal.h"
+#include "iceberg/util/checked_cast.h"
 #include "iceberg/util/macros.h"
 
 namespace iceberg::avro {
@@ -209,6 +212,63 @@ struct ReadContext {
   std::shared_ptr<::arrow::ArrayBuilder> builder_;
 };
 
+std::shared_ptr<::arrow::DataType> StorageTypeForBuilder(
+    const std::shared_ptr<::arrow::DataType>& type);
+
+std::shared_ptr<::arrow::Field> StorageFieldForBuilder(
+    const std::shared_ptr<::arrow::Field>& field) {
+  return ::arrow::field(field->name(), StorageTypeForBuilder(field->type()),
+                        field->nullable(), field->metadata());
+}
+
+// Arrow cannot construct builders for arrow.uuid extension arrays yet, so build
+// with the UUID storage type while keeping the public schema unchanged.
+std::shared_ptr<::arrow::DataType> StorageTypeForBuilder(
+    const std::shared_ptr<::arrow::DataType>& type) {
+  switch (type->id()) {
+    case ::arrow::Type::EXTENSION: {
+      const auto& extension_type =
+          internal::checked_cast<const ::arrow::ExtensionType&>(*type);
+      if (extension_type.extension_name() == kArrowUuidExtensionName) {
+        return extension_type.storage_type();
+      }
+      return type;
+    }
+    case ::arrow::Type::STRUCT: {
+      const auto& struct_type = internal::checked_cast<const ::arrow::StructType&>(*type);
+      std::vector<std::shared_ptr<::arrow::Field>> fields;
+      fields.reserve(struct_type.num_fields());
+      for (const auto& field : struct_type.fields()) {
+        fields.emplace_back(StorageFieldForBuilder(field));
+      }
+      return ::arrow::struct_(std::move(fields));
+    }
+    case ::arrow::Type::LIST: {
+      const auto& list_type = internal::checked_cast<const ::arrow::ListType&>(*type);
+      return ::arrow::list(StorageFieldForBuilder(list_type.value_field()));
+    }
+    case ::arrow::Type::LARGE_LIST: {
+      const auto& list_type =
+          internal::checked_cast<const ::arrow::LargeListType&>(*type);
+      return ::arrow::large_list(StorageFieldForBuilder(list_type.value_field()));
+    }
+    case ::arrow::Type::FIXED_SIZE_LIST: {
+      const auto& list_type =
+          internal::checked_cast<const ::arrow::FixedSizeListType&>(*type);
+      return ::arrow::fixed_size_list(StorageFieldForBuilder(list_type.value_field()),
+                                      list_type.list_size());
+    }
+    case ::arrow::Type::MAP: {
+      const auto& map_type = internal::checked_cast<const ::arrow::MapType&>(*type);
+      return ::arrow::map(StorageTypeForBuilder(map_type.key_type()),
+                          StorageFieldForBuilder(map_type.item_field()),
+                          map_type.keys_sorted());
+    }
+    default:
+      return type;
+  }
+}
+
 }  // namespace
 
 // TODO(gang.wu): collect basic reader metrics
@@ -349,7 +409,8 @@ class AvroReader::Impl {
     context_->arrow_schema_ = import_result.MoveValueUnsafe();
 
     auto arrow_struct_type =
-        std::make_shared<::arrow::StructType>(context_->arrow_schema_->fields());
+        internal::checked_pointer_cast<::arrow::StructType>(StorageTypeForBuilder(
+            std::make_shared<::arrow::StructType>(context_->arrow_schema_->fields())));
     auto builder_result = ::arrow::MakeBuilder(arrow_struct_type);
     if (!builder_result.ok()) {
       return InvalidSchema("Failed to make the arrow builder: {}",

src/iceberg/parquet/parquet_schema_util.cc

@@ -28,6 +28,7 @@
 #include "iceberg/metadata_columns.h"
 #include "iceberg/parquet/parquet_schema_util_internal.h"
 #include "iceberg/result.h"
+#include "iceberg/schema_internal.h"
 #include "iceberg/schema_util_internal.h"
 #include "iceberg/util/checked_cast.h"
 #include "iceberg/util/formatter.h"  // IWYU pragma: keep
@@ -222,7 +223,7 @@ Status ValidateParquetSchemaEvolution(
       if (arrow_type->id() == ::arrow::Type::EXTENSION) {
         const auto& extension_type =
             internal::checked_cast<const ::arrow::ExtensionType&>(*arrow_type);
-        if (extension_type.extension_name() == "arrow.uuid") {
+        if (extension_type.extension_name() == kArrowUuidExtensionName) {
           return {};
         }
       }

src/iceberg/row/struct_like.cc

@@ -62,6 +62,12 @@ Result<Scalar> LiteralToScalar(const Literal& literal) {
       return Scalar{
           std::string_view(reinterpret_cast<const char*>(bytes.data()), bytes.size())};
     }
+    case TypeId::kUuid: {
+      const auto& uuid = std::get<Uuid>(literal.value());
+      const auto& bytes = uuid.bytes();
+      return Scalar{
+          std::string_view(reinterpret_cast<const char*>(bytes.data()), bytes.size())};
+    }
     case TypeId::kDecimal:
       return Scalar{std::get<Decimal>(literal.value())};
     default:
@@ -162,8 +168,14 @@ Result<Literal> StructLikeAccessor::GetLiteral(const StructLike& struct_like) co
       const auto& fixed_data = std::get<std::string_view>(scalar);
       return Literal::Fixed(std::vector<uint8_t>(fixed_data.cbegin(), fixed_data.cend()));
     }
-    case TypeId::kUuid:
-      // TODO(gangwu): Implement UUID type
+    case TypeId::kUuid: {
+      const auto& uuid_data = std::get<std::string_view>(scalar);
+      ICEBERG_ASSIGN_OR_RAISE(
+          auto uuid,
+          Uuid::FromBytes(std::span<const uint8_t>(
+              reinterpret_cast<const uint8_t*>(uuid_data.data()), uuid_data.size())));
+      return Literal::UUID(uuid);
+    }
     default:
       return NotSupported("Cannot convert scalar to literal of type {}",
                           type_->ToString());

src/iceberg/schema_internal.cc

@@ -37,7 +37,6 @@ namespace {
 // Constants for Arrow schema metadata
 constexpr const char* kArrowExtensionName = "ARROW:extension:name";
 constexpr const char* kArrowExtensionMetadata = "ARROW:extension:metadata";
-constexpr const char* kArrowUuidExtensionName = "arrow.uuid";
 constexpr int32_t kUnknownFieldId = -1;
 
 Status CheckArrowCompatible(const Type& type) {

src/iceberg/schema_internal.h

@@ -30,6 +30,9 @@
 
 namespace iceberg {
 
+/// \brief Canonical Arrow extension name used for Iceberg UUID values.
+inline constexpr const char* kArrowUuidExtensionName = "arrow.uuid";
+
 /// \brief Convert an Iceberg schema to an Arrow schema.
 ///
 /// \param[in] schema The Iceberg schema to convert.