fix(types): keep child field metadata when serializing ARRAY/MAP types

ArrayType::ToJson and MapType::ToJson passed a null metadata when creating
the DataType of their children, so DataType::Create could not recognise an
extension type there. An ARRAY<VARIANT> therefore serialized as its physical
ARRAY<ROW<value, metadata>>, and creating such a table failed when the schema
was read back because that struct carries the fixed child field ids 0/1.

Carry the child field metadata over instead. The metadata only ever selects
VARIANT or BLOB in DataTypeToString, so ordinary element types serialize
unchanged.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: SteNicholas

src/paimon/common/types/array_type.h

@@ -43,8 +43,10 @@ class ArrayType : public DataType {
         auto type = arrow::internal::checked_cast<arrow::ListType*>(type_.get());
         auto value_field = type->value_field();
 
+        // The element metadata is load-bearing: it is what marks an extension type such as
+        // VARIANT.
         std::shared_ptr<DataType> data_type =
-            DataType::Create(value_field->type(), value_field->nullable(), /*metadata=*/nullptr);
+            DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata());
         obj.AddMember(rapidjson::StringRef("element"),
                       RapidJsonUtil::SerializeValue(*data_type, allocator).Move(), *allocator);
         return obj;

src/paimon/common/types/data_type_test.cpp

@@ -22,7 +22,9 @@
 #include "gtest/gtest.h"
 #include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/date_time_utils.h"
+#include "paimon/common/utils/rapidjson_util.h"
 
 namespace paimon::test {
 
@@ -109,4 +111,38 @@ TEST(DataTypeTest, DataTypeToString) {
     ASSERT_THROW(dummy_data_type.DataTypeToString(unknown_type), std::invalid_argument);
 }
 
+TEST(DataTypeTest, NestedTypeSerializationUsesChildMetadata) {
+    // ARRAY and MAP must carry their children's metadata into the serialized type, because that
+    // is what marks an extension type. Dropping it serialized a VARIANT as its physical
+    // `struct<value, metadata>` ROW, whose fixed child ids 0/1 then broke reading the schema back.
+    auto to_json = [](const std::shared_ptr<arrow::Field>& field) {
+        auto data_type = DataType::Create(field->type(), field->nullable(), field->metadata());
+        rapidjson::Document doc;
+        auto value = data_type->ToJson(&doc.GetAllocator());
+        rapidjson::StringBuffer buffer;
+        rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+        value.Accept(writer);
+        return std::string(buffer.GetString());
+    };
+
+    auto variant_field = VariantTypeUtils::ToArrowField("element");
+    auto array_field = arrow::field("arr", arrow::list(variant_field));
+    ASSERT_EQ(to_json(array_field), R"({"type":"ARRAY","element":"VARIANT"})");
+
+    auto map_field =
+        arrow::field("m", arrow::map(arrow::utf8(), VariantTypeUtils::ToArrowField("value")));
+    ASSERT_EQ(to_json(map_field), R"({"type":"MAP","key":"STRING NOT NULL","value":"VARIANT"})");
+
+    // BLOB is marked the same way and was degraded to its physical BYTES the same way.
+    auto blob_array = arrow::field("b", arrow::list(BlobUtils::ToArrowField("element", true)));
+    ASSERT_EQ(to_json(blob_array), R"({"type":"ARRAY","element":"BLOB"})");
+
+    // Child metadata only ever selects an extension type, so ordinary elements are unaffected by
+    // it being carried over.
+    auto plain_child = arrow::field("element", arrow::int32(), /*nullable=*/true,
+                                    arrow::KeyValueMetadata::Make({DataField::FIELD_ID}, {"7"}));
+    ASSERT_EQ(to_json(arrow::field("a", arrow::list(plain_child))),
+              R"({"type":"ARRAY","element":"INT"})");
+}
+
 }  // namespace paimon::test

src/paimon/common/types/map_type.h

@@ -49,13 +49,15 @@ class MapType : public DataType {
             *allocator);
         auto type = arrow::internal::checked_cast<arrow::MapType*>(type_.get());
         auto key_field = type->key_field();
+        // The key and value metadata is load-bearing: it is what marks an extension type such as
+        // VARIANT.
         std::shared_ptr<DataType> key_data_type =
-            DataType::Create(key_field->type(), key_field->nullable(), /*metadata=*/nullptr);
+            DataType::Create(key_field->type(), key_field->nullable(), key_field->metadata());
         obj.AddMember(rapidjson::StringRef("key"),
                       RapidJsonUtil::SerializeValue(*key_data_type, allocator).Move(), *allocator);
         auto value_field = type->item_field();
         std::shared_ptr<DataType> value_data_type =
-            DataType::Create(value_field->type(), value_field->nullable(), /*metadata=*/nullptr);
+            DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata());
         obj.AddMember(rapidjson::StringRef("value"),
                       RapidJsonUtil::SerializeValue(*value_data_type, allocator).Move(),
                       *allocator);