Skip to content

Commit f51fb4a

Browse files
huan233uschuan233usc
committed
feat(evolution): support v3 column default values in UpdateSchema (4/4)
AddColumn / AddRequiredColumn now accept an optional default value, used as both the initial-default and write-default of the new column; a non-null default also lets a required column be added (or an added column be made required) without AllowIncompatibleChanges(). UpdateColumnDefault sets or clears the write-default. Defaults are cast to the column type (rejecting uncastable or out-of-range values) and preserved across rename / doc / type updates and nested field-id reassignment. Part 4 of the v3 column-default-values work (POC #731), built on #746.
1 parent 93577b3 commit f51fb4a

5 files changed

Lines changed: 535 additions & 48 deletions

File tree

‎src/iceberg/test/resources/TableMetadataV3Valid.json‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
{
2+
"format-version": 3,
3+
"table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
4+
"location": "s3://bucket/test/location",
5+
"last-sequence-number": 34,
6+
"next-row-id": 0,
7+
"last-updated-ms": 1602638573590,
8+
"last-column-id": 3,
9+
"current-schema-id": 1,
10+
"schemas": [
11+
{
12+
"type": "struct",
13+
"schema-id": 0,
14+
"fields": [
15+
{
16+
"id": 1,
17+
"name": "x",
18+
"required": true,
19+
"type": "long"
20+
}
21+
]
22+
},
23+
{
24+
"type": "struct",
25+
"schema-id": 1,
26+
"identifier-field-ids": [
27+
1,
28+
2
29+
],
30+
"fields": [
31+
{
32+
"id": 1,
33+
"name": "x",
34+
"required": true,
35+
"type": "long"
36+
},
37+
{
38+
"id": 2,
39+
"name": "y",
40+
"required": true,
41+
"type": "long",
42+
"doc": "comment"
43+
},
44+
{
45+
"id": 3,
46+
"name": "z",
47+
"required": true,
48+
"type": "long"
49+
}
50+
]
51+
}
52+
],
53+
"default-spec-id": 0,
54+
"partition-specs": [
55+
{
56+
"spec-id": 0,
57+
"fields": [
58+
{
59+
"name": "x",
60+
"transform": "identity",
61+
"source-id": 1,
62+
"field-id": 1000
63+
}
64+
]
65+
}
66+
],
67+
"last-partition-id": 1000,
68+
"default-sort-order-id": 3,
69+
"sort-orders": [
70+
{
71+
"order-id": 3,
72+
"fields": [
73+
{
74+
"transform": "identity",
75+
"source-id": 2,
76+
"direction": "asc",
77+
"null-order": "nulls-first"
78+
},
79+
{
80+
"transform": "bucket[4]",
81+
"source-id": 3,
82+
"direction": "desc",
83+
"null-order": "nulls-last"
84+
}
85+
]
86+
}
87+
],
88+
"properties": {},
89+
"current-snapshot-id": 3055729675574597004,
90+
"snapshots": [
91+
{
92+
"snapshot-id": 3051729675574597004,
93+
"timestamp-ms": 1515100955770,
94+
"sequence-number": 0,
95+
"summary": {
96+
"operation": "append"
97+
},
98+
"manifest-list": "s3://a/b/1.avro"
99+
},
100+
{
101+
"snapshot-id": 3055729675574597004,
102+
"parent-snapshot-id": 3051729675574597004,
103+
"timestamp-ms": 1555100955770,
104+
"sequence-number": 1,
105+
"summary": {
106+
"operation": "append"
107+
},
108+
"manifest-list": "s3://a/b/2.avro",
109+
"schema-id": 1
110+
}
111+
],
112+
"snapshot-log": [
113+
{
114+
"snapshot-id": 3051729675574597004,
115+
"timestamp-ms": 1515100955770
116+
},
117+
{
118+
"snapshot-id": 3055729675574597004,
119+
"timestamp-ms": 1555100955770
120+
}
121+
],
122+
"metadata-log": []
123+
}

‎src/iceberg/test/update_schema_test.cc‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,13 @@
1919

2020
#include "iceberg/update/update_schema.h"
2121

22+
#include <limits>
2223
#include <memory>
2324
#include <vector>
2425

2526
#include <gtest/gtest.h>
2627

28+
#include "iceberg/expression/literal.h"
2729
#include "iceberg/schema.h"
2830
#include "iceberg/schema_field.h"
2931
#include "iceberg/test/matchers.h"
@@ -82,6 +84,232 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnWithAllowIncompatible) {
8284
EXPECT_EQ(new_field.doc(), "A required string column");
8385
}
8486

87+
/// Default values require a v3 table for Apply() to validate successfully.
88+
class UpdateSchemaDefaultValueTest : public UpdateSchemaTest {
89+
protected:
90+
std::string MetadataResource() const override { return "TableMetadataV3Valid.json"; }
91+
};
92+
93+
TEST_F(UpdateSchemaTest, AddColumnWithDefaultValueRequiresV3) {
94+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
95+
update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42));
96+
97+
auto result = update->Apply();
98+
EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema));
99+
EXPECT_THAT(result, HasErrorMessage("is not supported until v3"));
100+
}
101+
102+
TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDefaultValue) {
103+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
104+
update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42));
105+
106+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
107+
ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col"));
108+
ASSERT_TRUE(new_field_opt.has_value());
109+
110+
const auto& new_field = new_field_opt->get();
111+
ASSERT_NE(new_field.initial_default(), nullptr);
112+
EXPECT_EQ(*new_field.initial_default(), Literal::Int(42));
113+
ASSERT_NE(new_field.write_default(), nullptr);
114+
EXPECT_EQ(*new_field.write_default(), Literal::Int(42));
115+
}
116+
117+
TEST_F(UpdateSchemaDefaultValueTest, AddRequiredColumnWithDefaultValue) {
118+
// A required column with a default does not need AllowIncompatibleChanges():
119+
// old rows read the initial-default instead of null.
120+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
121+
update->AddRequiredColumn("required_col", string(), "A required string column",
122+
Literal::String("n/a"));
123+
124+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
125+
ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt,
126+
result.schema->FindFieldByName("required_col"));
127+
ASSERT_TRUE(new_field_opt.has_value());
128+
129+
const auto& new_field = new_field_opt->get();
130+
EXPECT_FALSE(new_field.optional());
131+
ASSERT_NE(new_field.initial_default(), nullptr);
132+
EXPECT_EQ(*new_field.initial_default(), Literal::String("n/a"));
133+
ASSERT_NE(new_field.write_default(), nullptr);
134+
EXPECT_EQ(*new_field.write_default(), Literal::String("n/a"));
135+
}
136+
137+
TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithMismatchedDefaultValueFails) {
138+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
139+
update->AddColumn("new_col", int32(), "An integer column", Literal::String("oops"));
140+
141+
auto result = update->Apply();
142+
EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed));
143+
EXPECT_THAT(result, HasErrorMessage("Cannot cast default value"));
144+
}
145+
146+
TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithNarrowingDefaultValueFails) {
147+
// CastTo signals narrowing with AboveMax/BelowMin sentinels; they must not be
148+
// stored as defaults.
149+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
150+
update->AddColumn("new_col", int32(), "An integer column",
151+
Literal::Long(std::numeric_limits<int64_t>::max()));
152+
153+
auto result = update->Apply();
154+
EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed));
155+
EXPECT_THAT(result, HasErrorMessage("Cannot cast default value"));
156+
}
157+
158+
TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefault) {
159+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
160+
update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42))
161+
.UpdateColumnDefault("new_col", Literal::Int(7));
162+
163+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
164+
ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col"));
165+
ASSERT_TRUE(new_field_opt.has_value());
166+
167+
const auto& new_field = new_field_opt->get();
168+
// initial-default is fixed at column addition; write-default is updated.
169+
ASSERT_NE(new_field.initial_default(), nullptr);
170+
EXPECT_EQ(*new_field.initial_default(), Literal::Int(42));
171+
ASSERT_NE(new_field.write_default(), nullptr);
172+
EXPECT_EQ(*new_field.write_default(), Literal::Int(7));
173+
}
174+
175+
TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultOnExistingColumn) {
176+
// Updating the write-default of a pre-existing column must survive Apply().
177+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
178+
update->UpdateColumnDefault("x", Literal::Long(0));
179+
180+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
181+
ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x"));
182+
ASSERT_TRUE(field_opt.has_value());
183+
184+
const auto& field = field_opt->get();
185+
EXPECT_EQ(field.initial_default(), nullptr);
186+
ASSERT_NE(field.write_default(), nullptr);
187+
EXPECT_EQ(*field.write_default(), Literal::Long(0));
188+
}
189+
190+
TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultClearsWithNullopt) {
191+
// Passing std::nullopt removes the write-default (Java parity with null).
192+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
193+
update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42))
194+
.UpdateColumnDefault("new_col", std::nullopt);
195+
196+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
197+
ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col"));
198+
ASSERT_TRUE(field_opt.has_value());
199+
200+
const auto& field = field_opt->get();
201+
// initial-default stays; write-default is cleared.
202+
ASSERT_NE(field.initial_default(), nullptr);
203+
EXPECT_EQ(*field.initial_default(), Literal::Int(42));
204+
EXPECT_EQ(field.write_default(), nullptr);
205+
}
206+
207+
TEST_F(UpdateSchemaDefaultValueTest, AddNestedColumnPreservesNestedDefaults) {
208+
// The added column's type gets fresh field ids; defaults on its nested fields must
209+
// survive the reassignment.
210+
auto nested_type = std::make_shared<StructType>(std::vector<SchemaField>{
211+
SchemaField(/*field_id=*/100, "inner", int32(), /*optional=*/false, /*doc=*/{},
212+
std::make_shared<const Literal>(Literal::Int(5)),
213+
std::make_shared<const Literal>(Literal::Int(9)))});
214+
215+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
216+
update->AddColumn("outer", nested_type, "A nested column");
217+
218+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
219+
ICEBERG_UNWRAP_OR_FAIL(auto outer_opt, result.schema->FindFieldByName("outer"));
220+
ASSERT_TRUE(outer_opt.has_value());
221+
222+
const auto& outer_struct =
223+
internal::checked_cast<const StructType&>(*outer_opt->get().type());
224+
ASSERT_EQ(outer_struct.fields().size(), 1);
225+
const SchemaField& inner = outer_struct.fields()[0];
226+
ASSERT_NE(inner.initial_default(), nullptr);
227+
EXPECT_EQ(*inner.initial_default(), Literal::Int(5));
228+
ASSERT_NE(inner.write_default(), nullptr);
229+
EXPECT_EQ(*inner.write_default(), Literal::Int(9));
230+
}
231+
232+
TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultCastsToColumnType) {
233+
// An int default for a long column is cast to the column type, not rejected.
234+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
235+
update->UpdateColumnDefault("x", Literal::Int(5));
236+
237+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
238+
ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x"));
239+
ASSERT_TRUE(field_opt.has_value());
240+
241+
const auto& field = field_opt->get();
242+
ASSERT_NE(field.write_default(), nullptr);
243+
EXPECT_EQ(*field.write_default(), Literal::Long(5));
244+
}
245+
246+
TEST_F(UpdateSchemaDefaultValueTest, RequireColumnAddedWithDefault) {
247+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
248+
update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42))
249+
.RequireColumn("new_col");
250+
251+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
252+
ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col"));
253+
ASSERT_TRUE(new_field_opt.has_value());
254+
EXPECT_FALSE(new_field_opt->get().optional());
255+
}
256+
257+
TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDocPreservesDefaultValues) {
258+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
259+
update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42))
260+
.UpdateColumnDoc("new_col", "updated doc");
261+
262+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
263+
ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col"));
264+
ASSERT_TRUE(field_opt.has_value());
265+
266+
const auto& field = field_opt->get();
267+
EXPECT_EQ(field.doc(), "updated doc");
268+
ASSERT_NE(field.initial_default(), nullptr);
269+
EXPECT_EQ(*field.initial_default(), Literal::Int(42));
270+
ASSERT_NE(field.write_default(), nullptr);
271+
EXPECT_EQ(*field.write_default(), Literal::Int(42));
272+
}
273+
274+
TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDefaultValues) {
275+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
276+
update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42))
277+
.UpdateColumn("new_col", int64());
278+
279+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
280+
ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col"));
281+
ASSERT_TRUE(field_opt.has_value());
282+
283+
const auto& field = field_opt->get();
284+
EXPECT_EQ(field.type(), int64());
285+
ASSERT_NE(field.initial_default(), nullptr);
286+
EXPECT_EQ(*field.initial_default(), Literal::Long(42));
287+
ASSERT_NE(field.write_default(), nullptr);
288+
EXPECT_EQ(*field.write_default(), Literal::Long(42));
289+
}
290+
291+
TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDecimalDefault) {
292+
// decimal(9,2) -> decimal(18,2) is an allowed precision widening. Literal::CastTo
293+
// does not cast between decimal types, so the default must still be promoted (the
294+
// unscaled value is unchanged).
295+
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
296+
update
297+
->AddColumn("new_col", decimal(9, 2), "A decimal column",
298+
Literal::Decimal(1234, 9, 2))
299+
.UpdateColumn("new_col", decimal(18, 2));
300+
301+
ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply());
302+
ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col"));
303+
ASSERT_TRUE(field_opt.has_value());
304+
305+
const auto& field = field_opt->get();
306+
EXPECT_EQ(field.type()->ToString(), decimal(18, 2)->ToString());
307+
ASSERT_NE(field.initial_default(), nullptr);
308+
EXPECT_EQ(*field.initial_default(), Literal::Decimal(1234, 18, 2));
309+
ASSERT_NE(field.write_default(), nullptr);
310+
EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2));
311+
}
312+
85313
TEST_F(UpdateSchemaTest, AddMultipleColumns) {
86314
ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema());
87315
update->AddColumn("col1", int32(), "First column")

0 commit comments

Comments
 (0)