Skip to content

Commit 8d88bef

Browse files
authored
fix: allow avro date reads from plain int (#728)
per https://github.com/apache/iceberg/pull/16446/changes
1 parent 5eb32df commit 8d88bef

5 files changed

Lines changed: 45 additions & 8 deletions

File tree

src/iceberg/avro/avro_data_util.cc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -395,10 +395,10 @@ Status AppendPrimitiveValueToBuilder(const ::avro::NodePtr& avro_node,
395395
}
396396

397397
case TypeId::kDate: {
398-
if (avro_node->type() != ::avro::AVRO_INT ||
399-
avro_node->logicalType().type() != ::avro::LogicalType::DATE) {
398+
if (!IsAvroDateOrPlainInt(avro_node)) {
400399
return InvalidArgument(
401-
"Expected Avro int with DATE logical type for date field, got: {}",
400+
"Expected Avro int with DATE logical type or plain int for date field, got: "
401+
"{}",
402402
ToString(avro_node));
403403
}
404404
auto* builder = internal::checked_cast<::arrow::Date32Builder*>(array_builder);

src/iceberg/avro/avro_direct_decoder.cc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -523,10 +523,10 @@ Status DecodePrimitiveValueToBuilder(const ::avro::NodePtr& avro_node,
523523
}
524524

525525
case TypeId::kDate: {
526-
if (avro_node->type() != ::avro::AVRO_INT ||
527-
avro_node->logicalType().type() != ::avro::LogicalType::DATE) {
526+
if (!IsAvroDateOrPlainInt(avro_node)) {
528527
return InvalidArgument(
529-
"Expected Avro int with DATE logical type for date field, got: {}",
528+
"Expected Avro int with DATE logical type or plain int for date field, got: "
529+
"{}",
530530
ToString(avro_node));
531531
}
532532
auto* builder = internal::checked_cast<::arrow::Date32Builder*>(array_builder);

src/iceberg/avro/avro_schema_util.cc

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,12 @@ std::string ToString(const ::avro::LogicalType::Type& logical_type) {
125125
return ToString(::avro::LogicalType(logical_type));
126126
}
127127

128+
bool IsAvroDateOrPlainInt(const ::avro::NodePtr& node) {
129+
return node->type() == ::avro::AVRO_INT &&
130+
(node->logicalType().type() == ::avro::LogicalType::DATE ||
131+
node->logicalType().type() == ::avro::LogicalType::NONE);
132+
}
133+
128134
Status ToAvroNodeVisitor::Visit(const BooleanType& type, ::avro::NodePtr* node) {
129135
*node = std::make_shared<::avro::NodePrimitive>(::avro::AVRO_BOOL);
130136
return {};
@@ -550,8 +556,7 @@ Status ValidateAvroSchemaEvolution(const Type& expected_type,
550556
}
551557
break;
552558
case TypeId::kDate:
553-
if (avro_node->type() == ::avro::AVRO_INT &&
554-
HasLogicalType(avro_node, ::avro::LogicalType::DATE)) {
559+
if (IsAvroDateOrPlainInt(avro_node)) {
555560
return {};
556561
}
557562
break;

src/iceberg/avro/avro_schema_util_internal.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,14 @@ std::string ToString(const ::avro::NodePtr& node);
147147
std::string ToString(const ::avro::LogicalType& logical_type);
148148
std::string ToString(const ::avro::LogicalType::Type& logical_type);
149149

150+
/// \brief Check if an Avro node can be read as an Iceberg date.
151+
///
152+
/// Iceberg dates are encoded as Avro ints with the date logical type. Readers
153+
/// also accept plain ints where the expected Iceberg type is date.
154+
/// \param node The Avro node to check.
155+
/// \return True if the node is an Avro date or a plain Avro int.
156+
bool IsAvroDateOrPlainInt(const ::avro::NodePtr& node);
157+
150158
/// \brief Check if an Avro node has a map logical type.
151159
/// \param node The Avro node to check.
152160
/// \return True if the node has a map logical type, false otherwise.

src/iceberg/test/avro_schema_test.cc

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -950,6 +950,30 @@ TEST(AvroSchemaProjectionTest, ProjectSchemaEvolutionIntToLong) {
950950
ASSERT_EQ(std::get<1>(projection.fields[0].from), 0);
951951
}
952952

953+
TEST(AvroSchemaProjectionTest, ProjectDateFromPlainInt) {
954+
Schema expected_schema({
955+
SchemaField::MakeRequired(/*field_id=*/1, "day", iceberg::date()),
956+
});
957+
958+
std::string avro_schema_json = R"({
959+
"type": "record",
960+
"name": "iceberg_schema",
961+
"fields": [
962+
{"name": "day", "type": "int", "field-id": 1}
963+
]
964+
})";
965+
auto avro_schema = ::avro::compileJsonSchemaFromString(avro_schema_json);
966+
967+
auto projection_result =
968+
Project(expected_schema, avro_schema.root(), /*prune_source=*/false);
969+
ASSERT_THAT(projection_result, IsOk());
970+
971+
const auto& projection = *projection_result;
972+
ASSERT_EQ(projection.fields.size(), 1);
973+
ASSERT_EQ(projection.fields[0].kind, FieldProjection::Kind::kProjected);
974+
ASSERT_EQ(std::get<1>(projection.fields[0].from), 0);
975+
}
976+
953977
TEST(AvroSchemaProjectionTest, ProjectSchemaEvolutionFloatToDouble) {
954978
// Create iceberg schema expecting a double
955979
Schema expected_schema({

0 commit comments

Comments
 (0)