Skip to content

Commit 2bf1db5

Browse files
authored
feat: fix NTILE distribution logic (apache#22051)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Closes apache#22049 . ## Rationale for this change Root cause — datafusion/functions-window/src/ntile.rs:170-176 used the linear-interpolation formula i * n / num_rows to assign bucket numbers. That formula spreads the larger buckets evenly through the partition, but the SQL standard requires the first num_rows mod n buckets to be the larger ones. For NTILE(4) over 10 rows it produced bucket sizes 3,2,3,2 instead of 3,3,2,2 Fix — replaced the formula with the front-loaded distribution: compute base = num_rows / n and remainder = num_rows % n, treat the first remainder * (base+1) rows as the "large bucket" region (each bucket of size base+1), and the remainder as size-base buckets. The n > num_rows case (existing test NTILE(9223377) over 3 rows) still works because base = 0 forces every row into the "large" arm where each bucket holds one row. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent 015162a commit 2bf1db5

2 files changed

Lines changed: 96 additions & 16 deletions

File tree

datafusion/functions-window/src/ntile.rs

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@
1717

1818
//! `ntile` window function implementation
1919
20-
use crate::utils::{
21-
get_scalar_value_from_args, get_signed_integer, get_unsigned_integer,
22-
};
20+
use crate::utils::{get_scalar_value_from_args, get_unsigned_integer};
2321
use arrow::datatypes::FieldRef;
2422
use datafusion_common::arrow::array::{ArrayRef, UInt64Array};
2523
use datafusion_common::arrow::datatypes::{DataType, Field};
@@ -120,6 +118,7 @@ impl WindowUDFImpl for Ntile {
120118
&self,
121119
partition_evaluator_args: PartitionEvaluatorArgs,
122120
) -> Result<Box<dyn PartitionEvaluator>> {
121+
// check the n value is provided, guard against NTILE()
123122
let scalar_n =
124123
get_scalar_value_from_args(partition_evaluator_args.input_exprs(), 0)?
125124
.ok_or_else(|| {
@@ -130,16 +129,17 @@ impl WindowUDFImpl for Ntile {
130129
return exec_err!("NTILE requires a positive integer, but finds NULL");
131130
}
132131

133-
if scalar_n.is_unsigned() {
134-
let n = get_unsigned_integer(&scalar_n)?;
135-
Ok(Box::new(NtileEvaluator { n }))
136-
} else {
137-
let n: i64 = get_signed_integer(&scalar_n)?;
138-
if n <= 0 {
139-
return exec_err!("NTILE requires a positive integer");
140-
}
141-
Ok(Box::new(NtileEvaluator { n: n as u64 }))
132+
// Works for both signed and unsigned inputs: ScalarValue::cast_to uses
133+
// safe=false, so negative signed values fail the cast to UInt64, and
134+
// routing through UInt64 also accepts values greater than i64::MAX.
135+
let n = get_unsigned_integer(&scalar_n)
136+
.map_err(|_| exec_datafusion_err!("NTILE requires a positive integer"))?;
137+
138+
if n == 0 {
139+
return exec_err!("NTILE requires a positive integer");
142140
}
141+
142+
Ok(Box::new(NtileEvaluator { n }))
143143
}
144144
fn field(&self, field_args: WindowUDFFieldArgs) -> Result<FieldRef> {
145145
let nullable = false;
@@ -167,12 +167,27 @@ impl PartitionEvaluator for NtileEvaluator {
167167
_values: &[ArrayRef],
168168
num_rows: usize,
169169
) -> Result<ArrayRef> {
170+
// SQL NTILE distributes rows "as equally as possible": with `base = num_rows / n`
171+
// and `remainder = num_rows % n`, the first `remainder` buckets each contain
172+
// `base + 1` rows and the rest contain `base` rows. The previous formula
173+
// `i * n / num_rows` does not preserve those bucket sizes (e.g. NTILE(4) over
174+
// 10 rows yielded sizes 3,2,3,2 instead of 3,3,2,2).
170175
let num_rows = num_rows as u64;
171-
let mut vec: Vec<u64> = Vec::new();
172-
let n = u64::min(self.n, num_rows);
176+
let n = self.n;
177+
let mut vec: Vec<u64> = Vec::with_capacity(num_rows as usize);
178+
let base = num_rows / n;
179+
let remainder = num_rows % n;
180+
let large_bucket_size = base + 1;
181+
let large_rows = remainder * large_bucket_size;
173182
for i in 0..num_rows {
174-
let res = i * n / num_rows;
175-
vec.push(res + 1)
183+
let bucket = if i < large_rows {
184+
i / large_bucket_size + 1
185+
} else {
186+
// base > 0 here: i >= large_rows is only reachable when remainder < n,
187+
// which forces base >= 1 (otherwise large_rows would equal num_rows).
188+
remainder + (i - large_rows) / base + 1
189+
};
190+
vec.push(bucket);
176191
}
177192
Ok(Arc::new(UInt64Array::from(vec)))
178193
}

datafusion/sqllogictest/test_files/window.slt

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3814,6 +3814,71 @@ Simpsons Lisa 710 1
38143814
Simpsons Marge 990 1
38153815
Simpsons Bart 2010 1
38163816

3817+
# Regression: when num_rows is not a multiple of n, the SQL standard requires the
3818+
# first `num_rows mod n` buckets to be the larger ones. Earlier the formula
3819+
# `i * n / num_rows` produced sizes 3,2,3,2 for NTILE(4) over 10 rows; the
3820+
# correct sizes are 3,3,2,2.
3821+
query I
3822+
SELECT NTILE(4) OVER (ORDER BY column1)
3823+
FROM (VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10));
3824+
----
3825+
1
3826+
1
3827+
1
3828+
2
3829+
2
3830+
2
3831+
3
3832+
3
3833+
4
3834+
4
3835+
3836+
# 7 rows / 3 buckets -> sizes 3,2,2
3837+
query I
3838+
SELECT NTILE(3) OVER (ORDER BY column1)
3839+
FROM (VALUES (1),(2),(3),(4),(5),(6),(7));
3840+
----
3841+
1
3842+
1
3843+
1
3844+
2
3845+
2
3846+
3
3847+
3
3848+
3849+
# Same regression with PARTITION BY: every 10-row partition must produce 3,3,2,2.
3850+
query III
3851+
SELECT a, b,
3852+
NTILE(4) OVER (PARTITION BY a ORDER BY b) AS ntile_4
3853+
FROM (VALUES
3854+
(0, 0), (0, 1), (0, 2), (0, 3), (0, 4),
3855+
(0, 5), (0, 6), (0, 7), (0, 8), (0, 9),
3856+
(1, 0), (1, 1), (1, 2), (1, 3), (1, 4),
3857+
(1, 5), (1, 6), (1, 7), (1, 8), (1, 9))
3858+
t(a, b)
3859+
ORDER BY a, b;
3860+
----
3861+
0 0 1
3862+
0 1 1
3863+
0 2 1
3864+
0 3 2
3865+
0 4 2
3866+
0 5 2
3867+
0 6 3
3868+
0 7 3
3869+
0 8 4
3870+
0 9 4
3871+
1 0 1
3872+
1 1 1
3873+
1 2 1
3874+
1 3 2
3875+
1 4 2
3876+
1 5 2
3877+
1 6 3
3878+
1 7 3
3879+
1 8 4
3880+
1 9 4
3881+
38173882
# incorrect number of parameters for ntile
38183883
query error DataFusion error: Execution error: NTILE requires a positive integer, but finds NULL
38193884
SELECT

0 commit comments

Comments
 (0)