Skip to content

Commit b88ac6c

Browse files
adriangbclaude
adriangb
and
claude
committed
refactor: deprecate TableSchema::new(), migrate callers to the builder
`TableSchema::new(file_schema, partition_cols)` is now deprecated: it can only express a file schema plus partition columns, so once additional optional column groups exist (e.g. virtual columns) it is a partial constructor sitting alongside the complete `TableSchemaBuilder`. Routing everything through the builder gives a single construction path. Migrate all in-tree callers: - empty partition columns -> `TableSchema::from_file_schema(file_schema)` - with partition columns -> `TableSchema::builder(file_schema) .with_table_partition_cols(cols).build()` `from_file_schema` is kept as a convenience for the no-partition-column case. Documented in the 55.0.0 upgrade guide. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent e46be6b commit b88ac6c

17 files changed

Lines changed: 143 additions & 103 deletions

File tree

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -321,14 +321,15 @@ impl ListingTable {
321321

322322
/// Creates a file source for this table
323323
fn create_file_source(&self) -> Arc<dyn FileSource> {
324-
let table_schema = TableSchema::new(
325-
Arc::clone(&self.file_schema),
326-
self.options
327-
.table_partition_cols
328-
.iter()
329-
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), false)))
330-
.collect(),
331-
);
324+
let table_schema = TableSchema::builder(Arc::clone(&self.file_schema))
325+
.with_table_partition_cols(
326+
self.options
327+
.table_partition_cols
328+
.iter()
329+
.map(|(col, field)| Arc::new(Field::new(col, field.clone(), false)))
330+
.collect::<Vec<_>>(),
331+
)
332+
.build();
332333

333334
self.options.format.file_source(table_schema)
334335
}

‎datafusion/core/src/datasource/file_format/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ pub(crate) mod test_util {
6767
.await?
6868
};
6969

70-
let table_schema = TableSchema::new(file_schema.clone(), vec![]);
70+
let table_schema = TableSchema::from_file_schema(file_schema.clone());
7171

7272
let statistics = format
7373
.infer_stats(state, &store, file_schema.clone(), &meta)

‎datafusion/core/src/datasource/physical_plan/avro.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -223,10 +223,13 @@ mod tests {
223223
partitioned_file.partition_values = vec![ScalarValue::from("2021-10-26")];
224224

225225
let projection = Some(vec![0, 1, file_schema.fields().len(), 2]);
226-
let table_schema = TableSchema::new(
227-
file_schema.clone(),
228-
vec![Arc::new(Field::new("date", DataType::Utf8, false))],
229-
);
226+
let table_schema = TableSchema::builder(file_schema.clone())
227+
.with_table_partition_cols(vec![Arc::new(Field::new(
228+
"date",
229+
DataType::Utf8,
230+
false,
231+
))])
232+
.build();
230233
let source = Arc::new(AvroSource::new(table_schema.clone()));
231234
let conf = FileScanConfigBuilder::new(object_store_url, source)
232235
// select specific columns of the files as well as the partitioning

‎datafusion/core/src/datasource/physical_plan/csv.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -400,10 +400,13 @@ mod tests {
400400
quote: b'"',
401401
..Default::default()
402402
};
403-
let table_schema = TableSchema::new(
404-
Arc::clone(&file_schema),
405-
vec![Arc::new(Field::new("date", DataType::Utf8, false))],
406-
);
403+
let table_schema = TableSchema::builder(Arc::clone(&file_schema))
404+
.with_table_partition_cols(vec![Arc::new(Field::new(
405+
"date",
406+
DataType::Utf8,
407+
false,
408+
))])
409+
.build();
407410
let source =
408411
Arc::new(CsvSource::new(table_schema.clone()).with_csv_options(options));
409412
let config =

‎datafusion/core/src/datasource/physical_plan/parquet.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1642,9 +1642,8 @@ mod tests {
16421642
),
16431643
]);
16441644

1645-
let table_schema = TableSchema::new(
1646-
Arc::clone(&schema),
1647-
vec![
1645+
let table_schema = TableSchema::builder(Arc::clone(&schema))
1646+
.with_table_partition_cols(vec![
16481647
Arc::new(Field::new("year", DataType::Utf8, false)),
16491648
Arc::new(Field::new("month", DataType::UInt8, false)),
16501649
Arc::new(Field::new(
@@ -1655,8 +1654,8 @@ mod tests {
16551654
),
16561655
false,
16571656
)),
1658-
],
1659-
);
1657+
])
1658+
.build();
16601659
let source = Arc::new(ParquetSource::new(table_schema.clone()));
16611660
let config = FileScanConfigBuilder::new(object_store_url, source)
16621661
.with_file(partitioned_file)

‎datafusion/core/tests/physical_optimizer/projection_pushdown.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1574,10 +1574,13 @@ fn partitioned_data_source() -> Arc<DataSourceExec> {
15741574
quote: b'"',
15751575
..Default::default()
15761576
};
1577-
let table_schema = TableSchema::new(
1578-
Arc::clone(&file_schema),
1579-
vec![Arc::new(Field::new("partition_col", DataType::Utf8, true))],
1580-
);
1577+
let table_schema = TableSchema::builder(Arc::clone(&file_schema))
1578+
.with_table_partition_cols(vec![Arc::new(Field::new(
1579+
"partition_col",
1580+
DataType::Utf8,
1581+
true,
1582+
))])
1583+
.build();
15811584
let config = FileScanConfigBuilder::new(
15821585
ObjectStoreUrl::parse("test:///").unwrap(),
15831586
Arc::new(CsvSource::new(table_schema).with_csv_options(options)),

‎datafusion/core/tests/physical_optimizer/pushdown_utils.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ pub struct TestSource {
111111

112112
impl TestSource {
113113
pub fn new(schema: SchemaRef, support: bool, batches: Vec<RecordBatch>) -> Self {
114-
let table_schema = datafusion_datasource::TableSchema::new(schema, vec![]);
114+
let table_schema = datafusion_datasource::TableSchema::from_file_schema(schema);
115115
Self {
116116
support,
117117
metrics: ExecutionPlanMetricsSet::new(),

‎datafusion/datasource-arrow/src/file_format.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,10 +197,9 @@ impl FileFormat for ArrowFormat {
197197
.object_meta
198198
.location;
199199

200-
let table_schema = TableSchema::new(
201-
Arc::clone(conf.file_schema()),
202-
conf.table_partition_cols().to_vec(),
203-
);
200+
let table_schema = TableSchema::builder(Arc::clone(conf.file_schema()))
201+
.with_table_partition_cols(conf.table_partition_cols().clone())
202+
.build();
204203

205204
let mut source: Arc<dyn FileSource> =
206205
match is_object_in_arrow_ipc_file_format(object_store, object_location).await

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1887,10 +1887,13 @@ mod test {
18871887
Field::new("a", DataType::Int32, false),
18881888
]));
18891889

1890-
let table_schema_for_opener = TableSchema::new(
1891-
file_schema.clone(),
1892-
vec![Arc::new(Field::new("part", DataType::Int32, false))],
1893-
);
1890+
let table_schema_for_opener = TableSchema::builder(file_schema.clone())
1891+
.with_table_partition_cols(vec![Arc::new(Field::new(
1892+
"part",
1893+
DataType::Int32,
1894+
false,
1895+
))])
1896+
.build();
18941897
let make_opener = |predicate| {
18951898
ParquetMorselizerBuilder::new()
18961899
.with_store(Arc::clone(&store))
@@ -1956,10 +1959,13 @@ mod test {
19561959
Field::new("a", DataType::Int32, false),
19571960
Field::new("b", DataType::Float32, true),
19581961
]));
1959-
let table_schema_for_opener = TableSchema::new(
1960-
file_schema.clone(),
1961-
vec![Arc::new(Field::new("part", DataType::Int32, false))],
1962-
);
1962+
let table_schema_for_opener = TableSchema::builder(file_schema.clone())
1963+
.with_table_partition_cols(vec![Arc::new(Field::new(
1964+
"part",
1965+
DataType::Int32,
1966+
false,
1967+
))])
1968+
.build();
19631969
let make_opener = |predicate| {
19641970
ParquetMorselizerBuilder::new()
19651971
.with_store(Arc::clone(&store))
@@ -2028,10 +2034,13 @@ mod test {
20282034
Field::new("a", DataType::Int32, false),
20292035
]));
20302036

2031-
let table_schema_for_opener = TableSchema::new(
2032-
file_schema.clone(),
2033-
vec![Arc::new(Field::new("part", DataType::Int32, false))],
2034-
);
2037+
let table_schema_for_opener = TableSchema::builder(file_schema.clone())
2038+
.with_table_partition_cols(vec![Arc::new(Field::new(
2039+
"part",
2040+
DataType::Int32,
2041+
false,
2042+
))])
2043+
.build();
20352044
let make_opener = |predicate| {
20362045
ParquetMorselizerBuilder::new()
20372046
.with_store(Arc::clone(&store))
@@ -2109,10 +2118,13 @@ mod test {
21092118
Field::new("part", DataType::Int32, false),
21102119
]));
21112120

2112-
let table_schema_for_opener = TableSchema::new(
2113-
file_schema.clone(),
2114-
vec![Arc::new(Field::new("part", DataType::Int32, false))],
2115-
);
2121+
let table_schema_for_opener = TableSchema::builder(file_schema.clone())
2122+
.with_table_partition_cols(vec![Arc::new(Field::new(
2123+
"part",
2124+
DataType::Int32,
2125+
false,
2126+
))])
2127+
.build();
21162128
let make_opener = |predicate| {
21172129
ParquetMorselizerBuilder::new()
21182130
.with_store(Arc::clone(&store))

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1287,7 +1287,9 @@ mod tests {
12871287
let file_schema =
12881288
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
12891289
let partition_b = Arc::new(Field::new("b", DataType::Int32, true));
1290-
let table_schema = TableSchema::new(file_schema, vec![partition_b]);
1290+
let table_schema = TableSchema::builder(file_schema)
1291+
.with_table_partition_cols(vec![partition_b])
1292+
.build();
12911293
let source = ParquetSource::new(table_schema);
12921294

12931295
// EquivalenceProperties is built on the *full* table schema so

0 commit comments

Comments
 (0)