Skip to content

Commit 2fca2bb

Browse files
Fix partitioned hash join test inputs
1 parent 0aa76ec commit 2fca2bb

1 file changed

Lines changed: 26 additions & 9 deletions

File tree

  • datafusion/physical-plan/src/joins/hash_join

datafusion/physical-plan/src/joins/hash_join/exec.rs

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2513,6 +2513,16 @@ mod tests {
25132513
Ok((join, dynamic_filter))
25142514
}
25152515

2516+
fn repartition_by(
2517+
input: Arc<dyn ExecutionPlan>,
2518+
column_name: &str,
2519+
partition_count: usize,
2520+
) -> Result<Arc<dyn ExecutionPlan>> {
2521+
let expr = Arc::new(Column::new_with_schema(column_name, &input.schema())?);
2522+
RepartitionExec::try_new(input, Partitioning::Hash(vec![expr], partition_count))
2523+
.map(|exec| Arc::new(exec) as _)
2524+
}
2525+
25162526
async fn join_collect(
25172527
left: Arc<dyn ExecutionPlan>,
25182528
right: Arc<dyn ExecutionPlan>,
@@ -5747,7 +5757,6 @@ mod tests {
57475757
Arc::new(Column::new_with_schema("b1", &left_batch.schema())?) as _,
57485758
Arc::new(Column::new_with_schema("b2", &right_batch.schema())?) as _,
57495759
)];
5750-
57515760
let join_types = vec![
57525761
JoinType::Inner,
57535762
JoinType::Left,
@@ -5760,6 +5769,11 @@ mod tests {
57605769
];
57615770

57625771
for join_type in join_types {
5772+
let left =
5773+
repartition_by(Arc::clone(&left) as Arc<dyn ExecutionPlan>, "b1", 1)?;
5774+
let right =
5775+
repartition_by(Arc::clone(&right) as Arc<dyn ExecutionPlan>, "b2", 1)?;
5776+
57635777
let runtime = RuntimeEnvBuilder::new()
57645778
.with_memory_limit(100, 1.0)
57655779
.build_arc()?;
@@ -5781,19 +5795,19 @@ mod tests {
57815795
false,
57825796
)?;
57835797

5784-
let stream = join.execute(1, task_ctx)?;
5798+
let stream = join.execute(0, task_ctx)?;
57855799
let err = common::collect(stream).await.unwrap_err();
57865800

5787-
// Asserting that stream-level reservation attempting to overallocate
5788-
assert_contains!(
5789-
err.to_string(),
5790-
"Resources exhausted: Additional allocation failed for HashJoinInput[1] with top memory consumers (across reservations) as:\n HashJoinInput[1]"
5791-
);
5792-
5801+
// Assert that the stream-level reservation attempts to
5802+
// overallocate. Other reservations can appear in the top-consumer
5803+
// report before HashJoinInput[0], so avoid depending on its exact
5804+
// ordering.
57935805
assert_contains!(
57945806
err.to_string(),
5795-
"Failed to allocate additional 120.0 B for HashJoinInput[1]"
5807+
"Resources exhausted: Additional allocation failed for HashJoinInput[0] with top memory consumers"
57965808
);
5809+
assert_contains!(err.to_string(), "HashJoinInput[0]");
5810+
assert_contains!(err.to_string(), "Failed to allocate additional");
57975811
}
57985812

57995813
Ok(())
@@ -6092,6 +6106,8 @@ mod tests {
60926106
Arc::clone(&parent_left_schema),
60936107
None,
60946108
)?;
6109+
let child_left = repartition_by(child_left, "child_key", 4)?;
6110+
let child_right = repartition_by(child_right, "child_right_key", 4)?;
60956111

60966112
let child_on = vec![(
60976113
Arc::new(Column::new_with_schema("child_key", &child_left_schema)?) as _,
@@ -6108,6 +6124,7 @@ mod tests {
61086124
PartitionMode::Partitioned,
61096125
)?;
61106126
let child_join: Arc<dyn ExecutionPlan> = Arc::new(child_join);
6127+
let parent_left = repartition_by(parent_left, "parent_key", 4)?;
61116128

61126129
let parent_on = vec![(
61136130
Arc::new(Column::new_with_schema("parent_key", &parent_left_schema)?) as _,

0 commit comments

Comments
 (0)