Skip to content

Commit eb5fb51

Browse files
adriangbadriangb
committed
handle upstream changes
1 parent be1228c commit eb5fb51

10 files changed

Lines changed: 85 additions & 93 deletions

File tree

‎Cargo.lock‎

Lines changed: 42 additions & 30 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -272,15 +272,15 @@ inherits = "release"
272272
debug = true
273273
strip = false
274274

275-
# [patch.crates-io]
276-
# parquet = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
277-
# arrow = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
278-
# arrow-array = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
279-
# arrow-buffer = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
280-
# arrow-cast = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
281-
# arrow-data = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
282-
# arrow-ipc = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
283-
# arrow-ord = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
284-
# arrow-schema = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
285-
# arrow-select = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
286-
# arrow-string = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
275+
[patch.crates-io]
276+
parquet = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
277+
arrow = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
278+
arrow-array = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
279+
arrow-buffer = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
280+
arrow-cast = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
281+
arrow-data = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
282+
arrow-ipc = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
283+
arrow-ord = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
284+
arrow-schema = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
285+
arrow-select = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }
286+
arrow-string = { git = "https://github.com/pydantic/arrow-rs.git", branch = "fix-cache-nested" }

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8339,7 +8339,7 @@ mod tests {
83398339
)))),
83408340
];
83418341

8342-
let check_array = |array| {
8342+
let check_array = |array: Arc<dyn Array>| {
83438343
let is_null = is_null(&array).unwrap();
83448344
assert_eq!(is_null, BooleanArray::from(vec![true, false, false]));
83458345

@@ -8868,7 +8868,7 @@ mod tests {
88688868
.unwrap(),
88698869
ScalarValue::try_new_null(&DataType::Map(map_field_ref, false)).unwrap(),
88708870
ScalarValue::try_new_null(&DataType::Union(
8871-
UnionFields::new(vec![42], vec![field_ref]),
8871+
UnionFields::try_new(vec![42], vec![field_ref]).unwrap(),
88728872
UnionMode::Dense,
88738873
))
88748874
.unwrap(),
@@ -8971,13 +8971,14 @@ mod tests {
89718971
}
89728972

89738973
// Test union type
8974-
let union_fields = UnionFields::new(
8974+
let union_fields = UnionFields::try_new(
89758975
vec![0, 1],
89768976
vec![
89778977
Field::new("i32", DataType::Int32, false),
89788978
Field::new("f64", DataType::Float64, false),
89798979
],
8980-
);
8980+
)
8981+
.unwrap();
89818982
let union_result = ScalarValue::new_default(&DataType::Union(
89828983
union_fields.clone(),
89838984
UnionMode::Sparse,

‎datafusion/datasource-avro/src/avro_to_arrow/schema.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ fn schema_to_field_with_props(
118118
.map(|s| schema_to_field_with_props(s, None, has_nullable, None))
119119
.collect::<Result<Vec<Field>>>()?;
120120
let type_ids = 0_i8..fields.len() as i8;
121-
DataType::Union(UnionFields::new(type_ids, fields), UnionMode::Dense)
121+
DataType::Union(UnionFields::try_new(type_ids, fields)?, UnionMode::Dense)
122122
}
123123
}
124124
AvroSchema::Record(RecordSchema { fields, .. }) => {

‎datafusion/datasource-parquet/src/row_filter.rs‎

Lines changed: 0 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -427,40 +427,6 @@ mod test {
427427
use datafusion_physical_plan::metrics::{Count, Time};
428428

429429
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
430-
use parquet::arrow::parquet_to_arrow_schema;
431-
use parquet::file::reader::{FileReader, SerializedFileReader};
432-
433-
// List predicates used by the decoder should be accepted for pushdown
434-
#[test]
435-
fn test_filter_candidate_builder_supports_list_types() {
436-
let testdata = datafusion_common::test_util::parquet_test_data();
437-
let file = std::fs::File::open(format!("{testdata}/list_columns.parquet"))
438-
.expect("opening file");
439-
440-
let reader = SerializedFileReader::new(file).expect("creating reader");
441-
442-
let metadata = reader.metadata();
443-
444-
let table_schema =
445-
parquet_to_arrow_schema(metadata.file_metadata().schema_descr(), None)
446-
.expect("parsing schema");
447-
448-
let expr = col("int64_list").is_not_null();
449-
let expr = logical2physical(&expr, &table_schema);
450-
451-
let table_schema = Arc::new(table_schema.clone());
452-
453-
let list_index = table_schema
454-
.index_of("int64_list")
455-
.expect("list column should exist");
456-
457-
let candidate = FilterCandidateBuilder::new(expr, table_schema)
458-
.build(metadata)
459-
.expect("building candidate")
460-
.expect("list pushdown should be supported");
461-
462-
assert_eq!(candidate.projection.leaf_indices, vec![list_index]);
463-
}
464430

465431
#[test]
466432
fn test_filter_type_coercion() {

‎datafusion/functions/src/core/union_extract.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,13 +189,13 @@ mod tests {
189189
fn test_scalar_value() -> Result<()> {
190190
let fun = UnionExtractFun::new();
191191

192-
let fields = UnionFields::new(
192+
let fields = UnionFields::try_new(
193193
vec![1, 3],
194194
vec![
195195
Field::new("str", DataType::Utf8, false),
196196
Field::new("int", DataType::Int32, false),
197197
],
198-
);
198+
)?;
199199

200200
let args = vec![
201201
ColumnarValue::Scalar(ScalarValue::Union(

‎datafusion/physical-plan/src/filter.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1557,13 +1557,13 @@ mod tests {
15571557
#[test]
15581558
fn test_equivalence_properties_union_type() -> Result<()> {
15591559
let union_type = DataType::Union(
1560-
UnionFields::new(
1560+
UnionFields::try_new(
15611561
vec![0, 1],
15621562
vec![
15631563
Field::new("f1", DataType::Int32, true),
15641564
Field::new("f2", DataType::Utf8, true),
15651565
],
1566-
),
1566+
)?,
15671567
UnionMode::Sparse,
15681568
);
15691569

‎datafusion/proto-common/src/from_proto/mod.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,10 @@ impl TryFrom<&protobuf::arrow_type::ArrowTypeEnum> for DataType {
310310
false => union.type_ids.iter().map(|i| *i as i8).collect(),
311311
};
312312

313-
DataType::Union(UnionFields::new(type_ids, union_fields), union_mode)
313+
DataType::Union(
314+
UnionFields::try_new(type_ids, union_fields).unwrap(),
315+
union_mode,
316+
)
314317
}
315318
arrow_type::ArrowTypeEnum::Dictionary(dict) => {
316319
let key_datatype = dict.as_ref().key.as_deref().required("key")?;
@@ -602,7 +605,14 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {
602605
.collect::<Option<Vec<_>>>();
603606
let fields = fields.ok_or_else(|| Error::required("UnionField"))?;
604607
let fields = parse_proto_fields_to_fields(&fields)?;
605-
let fields = UnionFields::new(ids, fields);
608+
let fields = match UnionFields::try_new(ids, fields) {
609+
Ok(f) => f,
610+
Err(e) => {
611+
return Err(Error::General(format!(
612+
"Invalid UnionFields in ScalarValue::Union: {e}"
613+
)));
614+
}
615+
};
606616
let v_id = val.value_id as i8;
607617
let val = match &val.value {
608618
None => None,

‎datafusion/proto/tests/cases/roundtrip_logical_plan.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1780,18 +1780,19 @@ fn round_trip_datatype() {
17801780
),
17811781
])),
17821782
DataType::Union(
1783-
UnionFields::new(
1783+
UnionFields::try_new(
17841784
vec![7, 5, 3],
17851785
vec![
17861786
Field::new("nullable", DataType::Boolean, false),
17871787
Field::new("name", DataType::Utf8, false),
17881788
Field::new("datatype", DataType::Binary, false),
17891789
],
1790-
),
1790+
)
1791+
.unwrap(),
17911792
UnionMode::Sparse,
17921793
),
17931794
DataType::Union(
1794-
UnionFields::new(
1795+
UnionFields::try_new(
17951796
vec![5, 8, 1],
17961797
vec![
17971798
Field::new("nullable", DataType::Boolean, false),
@@ -1807,7 +1808,8 @@ fn round_trip_datatype() {
18071808
true,
18081809
),
18091810
],
1810-
),
1811+
)
1812+
.unwrap(),
18111813
UnionMode::Dense,
18121814
),
18131815
DataType::Dictionary(

0 commit comments

Comments
 (0)