Skip to content

Commit 7ae534c

Browse files
committed
fix(manifest): encode day-transform partition fields with Avro date logical type
DayTransform.Apply() returns int32 (days since epoch) but the Avro encoding of a day-partition column must carry the 'date' logical type so that Trino, Spark, and other Iceberg engines can read manifests. Root fix: PartitionSpec.PartitionType() now overrides DayTransform's ResultType (Int32) to DateType for the partition struct it produces. The existing DateType branch in partitionTypeToAvroSchema then emits DateNode (int + date logical type) automatically — no coupling between the Avro layer and transform internals. This also fixes DataFileStatistics.ToDataFile() in table/internal/utils.go, which builds fieldIDToLogicalType by switching on ResultType: the case DateType branch now fires correctly for day-partition fields. Tests verify that day-partition fields decode as time.Time (confirming the date logical type is applied) while plain Int32 fields decode as int32.
1 parent efe96b6 commit 7ae534c

2 files changed

Lines changed: 80 additions & 0 deletions

File tree

partitions.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,15 @@ func (ps *PartitionSpec) PartitionType(schema *Schema) *StructType {
499499
continue
500500
}
501501
resultType := field.Transform.ResultType(sourceType)
502+
// DayTransform.ResultType returns Int32 (days since epoch), but the
503+
// Avro encoding of a day-partition column must carry the "date" logical
504+
// type so that Trino, Spark, and other engines can read manifests.
505+
// Override to DateType here so downstream schema conversion picks it up
506+
// automatically via the existing DateType branch, without coupling the
507+
// Avro layer to transform internals.
508+
if _, ok := field.Transform.(DayTransform); ok {
509+
resultType = PrimitiveTypes.Date
510+
}
502511
nestedFields = append(nestedFields, NestedField{
503512
ID: field.FieldID,
504513
Name: field.Name,

schema_conversions_test.go

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package iceberg
2020
import (
2121
"fmt"
2222
"testing"
23+
"time"
2324

2425
"github.com/apache/iceberg-go/internal"
2526
"github.com/stretchr/testify/assert"
@@ -196,3 +197,73 @@ func TestPartitionTypeToAvroSchemaDuplicateNamedTypes(t *testing.T) {
196197
})
197198
}
198199
}
200+
201+
// TestDayTransformPartitionAvroDateEncoding verifies that a day(ts) partition
202+
// field is encoded with the Avro "date" logical type, not as a plain integer.
203+
// This is required for interoperability with Trino, Spark, and other Iceberg
204+
// engines that reject manifests where day-partition columns lack the date type.
205+
//
206+
// The fix lives in PartitionSpec.PartitionType: it overrides DayTransform's
207+
// ResultType (Int32) to DateType so the existing DateType branch in
208+
// partitionTypeToAvroSchema emits DateNode automatically.
209+
func TestDayTransformPartitionAvroDateEncoding(t *testing.T) {
210+
schema := NewSchema(0,
211+
NestedField{ID: 1, Name: "ts", Type: TimestampTzType{}, Required: true},
212+
)
213+
spec := NewPartitionSpecID(0,
214+
PartitionField{FieldID: 1000, SourceIDs: []int{1}, Name: "ts_day", Transform: DayTransform{}},
215+
)
216+
217+
// PartitionType now returns DateType for day-transform fields.
218+
partitionType := spec.PartitionType(schema)
219+
require.Equal(t, PrimitiveTypes.Date, partitionType.FieldList[0].Type,
220+
"PartitionType must map DayTransform result to DateType")
221+
222+
avroSchema, err := partitionTypeToAvroSchema(partitionType)
223+
require.NoError(t, err)
224+
225+
// Encode a day value: 19000 days since epoch = 2022-01-08.
226+
encoded, err := avroSchema.Encode(map[string]any{"ts_day": int32(19000)})
227+
require.NoError(t, err)
228+
229+
// twmb/avro decodes an Avro "date" field as time.Time, not int32.
230+
// This confirms the date logical type is present — a plain int field
231+
// decodes as int32 (verified in TestNonDayInt32PartitionAvroIntEncoding).
232+
var decoded map[string]any
233+
_, err = avroSchema.Decode(encoded, &decoded)
234+
require.NoError(t, err)
235+
236+
got, ok := decoded["ts_day"]
237+
require.True(t, ok)
238+
_, isTime := got.(time.Time)
239+
assert.True(t, isTime, "day-partition field must decode as time.Time (date logical type), got %T", got)
240+
assert.Equal(t, time.Date(2022, time.January, 8, 0, 0, 0, 0, time.UTC), got)
241+
}
242+
243+
// TestNonDayInt32PartitionAvroIntEncoding verifies that an Int32 partition
244+
// field not produced by DayTransform keeps the plain int Avro encoding and
245+
// decodes as int32, not time.Time.
246+
func TestNonDayInt32PartitionAvroIntEncoding(t *testing.T) {
247+
schema := NewSchema(0,
248+
NestedField{ID: 1, Name: "bucket_id", Type: Int32Type{}, Required: true},
249+
)
250+
spec := NewPartitionSpecID(0,
251+
PartitionField{FieldID: 1000, SourceIDs: []int{1}, Name: "bucket_id_identity", Transform: IdentityTransform{}},
252+
)
253+
254+
partitionType := spec.PartitionType(schema)
255+
avroSchema, err := partitionTypeToAvroSchema(partitionType)
256+
require.NoError(t, err)
257+
258+
encoded, err := avroSchema.Encode(map[string]any{"bucket_id_identity": int32(42)})
259+
require.NoError(t, err)
260+
261+
var decoded map[string]any
262+
_, err = avroSchema.Decode(encoded, &decoded)
263+
require.NoError(t, err)
264+
265+
got, ok := decoded["bucket_id_identity"]
266+
require.True(t, ok)
267+
assert.IsType(t, int32(0), got, "plain Int32 partition must decode as int32, not time.Time")
268+
assert.Equal(t, int32(42), got)
269+
}

0 commit comments

Comments
 (0)