Skip to content

Commit 32cda28

Browse files
artbcfkevinjqliu
andauthored
fix(manifest): encode day-transform partition fields with Avro date logical type (#915)
`DayTransform.Apply()` returns an `int32` value representing days since the epoch. For manifest partition data, Iceberg implementations commonly write `day(...)` partition fields as Avro `int` with the `date` logical type, while readers should be able to accept both plain Avro `int` and Avro `date`. Previously, `DayTransform.ResultType()` returned `Int32`, which caused `PartitionSpec.PartitionType()` to model day-transform partition fields as plain `Int32`. As a result, Iceberg Go wrote day-partition fields as plain Avro `int`, instead of Avro `date`, diverging from the writer behavior of other Iceberg implementations. This PR changes `DayTransform.ResultType()` to return `DateType`. That lets the existing partition type and Avro schema conversion paths naturally emit Avro `date` for day-transform partition fields. Tests added: - Verify that `day(ts)` partition fields are encoded with the Avro `date` logical type. - Verify that non-day `Int32` partition fields still use plain Avro `int` encoding. --------- Co-authored-by: Kevin Liu <kevinjqliu@users.noreply.github.com>
1 parent 9af9dc5 commit 32cda28

2 files changed

Lines changed: 70 additions & 1 deletion

File tree

schema_conversions_test.go

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package iceberg
2020
import (
2121
"fmt"
2222
"testing"
23+
"time"
2324

2425
"github.com/apache/iceberg-go/internal"
2526
"github.com/stretchr/testify/assert"
@@ -196,3 +197,71 @@ func TestPartitionTypeToAvroSchemaDuplicateNamedTypes(t *testing.T) {
196197
})
197198
}
198199
}
200+
201+
// TestDayTransformPartitionAvroDateEncoding verifies that a day(ts) partition
202+
// field is encoded with the Avro "date" logical type, not as a plain integer.
203+
//
204+
// The fix lives in PartitionSpec.PartitionType: it overrides DayTransform's
205+
// ResultType (Int32) to DateType so the existing DateType branch in
206+
// partitionTypeToAvroSchema emits DateNode automatically.
207+
func TestDayTransformPartitionAvroDateEncoding(t *testing.T) {
208+
schema := NewSchema(0,
209+
NestedField{ID: 1, Name: "ts", Type: TimestampTzType{}, Required: true},
210+
)
211+
spec := NewPartitionSpecID(0,
212+
PartitionField{FieldID: 1000, SourceIDs: []int{1}, Name: "ts_day", Transform: DayTransform{}},
213+
)
214+
215+
// PartitionType now returns DateType for day-transform fields.
216+
partitionType := spec.PartitionType(schema)
217+
require.Equal(t, PrimitiveTypes.Date, partitionType.FieldList[0].Type,
218+
"PartitionType must map DayTransform result to DateType")
219+
220+
avroSchema, err := partitionTypeToAvroSchema(partitionType)
221+
require.NoError(t, err)
222+
223+
// Encode a day value: 19000 days since epoch = 2022-01-08.
224+
encoded, err := avroSchema.Encode(map[string]any{"ts_day": int32(19000)})
225+
require.NoError(t, err)
226+
227+
// twmb/avro decodes an Avro "date" field as time.Time, not int32.
228+
// This confirms the date logical type is present — a plain int field
229+
// decodes as int32 (verified in TestNonDayInt32PartitionAvroIntEncoding).
230+
var decoded map[string]any
231+
_, err = avroSchema.Decode(encoded, &decoded)
232+
require.NoError(t, err)
233+
234+
got, ok := decoded["ts_day"]
235+
require.True(t, ok)
236+
_, isTime := got.(time.Time)
237+
assert.True(t, isTime, "day-partition field must decode as time.Time (date logical type), got %T", got)
238+
assert.Equal(t, time.Date(2022, time.January, 8, 0, 0, 0, 0, time.UTC), got)
239+
}
240+
241+
// TestNonDayInt32PartitionAvroIntEncoding verifies that an Int32 partition
242+
// field not produced by DayTransform keeps the plain int Avro encoding and
243+
// decodes as int32, not time.Time.
244+
func TestNonDayInt32PartitionAvroIntEncoding(t *testing.T) {
245+
schema := NewSchema(0,
246+
NestedField{ID: 1, Name: "bucket_id", Type: Int32Type{}, Required: true},
247+
)
248+
spec := NewPartitionSpecID(0,
249+
PartitionField{FieldID: 1000, SourceIDs: []int{1}, Name: "bucket_id_identity", Transform: IdentityTransform{}},
250+
)
251+
252+
partitionType := spec.PartitionType(schema)
253+
avroSchema, err := partitionTypeToAvroSchema(partitionType)
254+
require.NoError(t, err)
255+
256+
encoded, err := avroSchema.Encode(map[string]any{"bucket_id_identity": int32(42)})
257+
require.NoError(t, err)
258+
259+
var decoded map[string]any
260+
_, err = avroSchema.Decode(encoded, &decoded)
261+
require.NoError(t, err)
262+
263+
got, ok := decoded["bucket_id_identity"]
264+
require.True(t, ok)
265+
assert.IsType(t, int32(0), got, "plain Int32 partition must decode as int32, not time.Time")
266+
assert.Equal(t, int32(42), got)
267+
}

transforms.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -786,7 +786,7 @@ func (t DayTransform) MarshalText() ([]byte, error) {
786786
func (DayTransform) String() string { return "day" }
787787

788788
func (t DayTransform) CanTransform(sourceType Type) bool { return canTransformTime(t, sourceType) }
789-
func (DayTransform) ResultType(Type) Type { return PrimitiveTypes.Int32 }
789+
func (DayTransform) ResultType(Type) Type { return PrimitiveTypes.Date }
790790
func (DayTransform) PreservesOrder() bool { return true }
791791

792792
func (DayTransform) Equals(other Transform) bool {

0 commit comments

Comments
 (0)