Skip to content

Commit df2781c

Browse files
committed
Skip files outside partition structure in hive-partitioned listing tables
When a hive-partitioned listing table contains files in the root directory (not inside any partition_col=value/ path), these files have no partition values. Previously they were included with empty partition_values, causing "Unable to get field named" errors when queries reference partition columns. Now try_into_partitioned_file returns None for files that don't match the partition structure, and the caller skips them.
1 parent fb8be01 commit df2781c

2 files changed

Lines changed: 7 additions & 9 deletions

File tree

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

datafusion/catalog-listing/src/helpers.rs

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -352,10 +352,7 @@ fn try_into_partitioned_file(
352352
let parsed = parse_partitions_for_path(table_path, &object_meta.location, cols);
353353

354354
let Some(parsed) = parsed else {
355-
debug!(
356-
"Skipping file outside partition structure: {}",
357-
object_meta.location
358-
);
355+
// parse_partitions_for_path already logs a debug message
359356
return Ok(None);
360357
};
361358

@@ -592,7 +589,7 @@ mod tests {
592589
let partition_cols = vec![("year_month".to_string(), DataType::Utf8)];
593590
let meta = ObjectMeta {
594591
location: Path::from("bucket/mytable/year_month=2024-01/data.parquet"),
595-
last_modified: chrono::Utc::now(),
592+
last_modified: chrono::DateTime::from(std::time::SystemTime::UNIX_EPOCH),
596593
size: 100,
597594
e_tag: None,
598595
version: None,
@@ -615,7 +612,7 @@ mod tests {
615612
let partition_cols = vec![("year_month".to_string(), DataType::Utf8)];
616613
let meta = ObjectMeta {
617614
location: Path::from("bucket/mytable/data.parquet"),
618-
last_modified: chrono::Utc::now(),
615+
last_modified: chrono::DateTime::from(std::time::SystemTime::UNIX_EPOCH),
619616
size: 100,
620617
e_tag: None,
621618
version: None,
@@ -635,7 +632,7 @@ mod tests {
635632
let partition_cols = vec![("year_month".to_string(), DataType::Utf8)];
636633
let meta = ObjectMeta {
637634
location: Path::from("bucket/mytable/wrong_col=2024-01/data.parquet"),
638-
last_modified: chrono::Utc::now(),
635+
last_modified: chrono::DateTime::from(std::time::SystemTime::UNIX_EPOCH),
639636
size: 100,
640637
e_tag: None,
641638
version: None,
@@ -658,7 +655,7 @@ mod tests {
658655
];
659656
let meta = ObjectMeta {
660657
location: Path::from("bucket/mytable/year=2024/month=01/data.parquet"),
661-
last_modified: chrono::Utc::now(),
658+
last_modified: chrono::DateTime::from(std::time::SystemTime::UNIX_EPOCH),
662659
size: 100,
663660
e_tag: None,
664661
version: None,
@@ -688,7 +685,7 @@ mod tests {
688685
];
689686
let meta = ObjectMeta {
690687
location: Path::from("bucket/mytable/year=2024/data.parquet"),
691-
last_modified: chrono::Utc::now(),
688+
last_modified: chrono::DateTime::from(std::time::SystemTime::UNIX_EPOCH),
692689
size: 100,
693690
e_tag: None,
694691
version: None,

0 commit comments

Comments
 (0)