Skip to content

Commit 7715379

Browse files
westonpaceclaude
andcommitted
fix: gradual feedback degradation and drop (legacy) from warning
- Feedback loop now degrades gradually when actual bpr is smaller than the current estimate (midpoint) instead of snapping immediately. Larger values are still adopted immediately to avoid OOM. - Remove "(legacy)" from user-facing v2.0 warning message. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 81b8873 commit 7715379

1 file changed

Lines changed: 14 additions & 3 deletions

File tree

rust/lance-encoding/src/decoder.rs

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1887,7 +1887,18 @@ impl StructuralBatchDecodeStream {
18871887
let num_rows = batch.num_rows() as u64;
18881888
if num_rows > 0 {
18891889
let bpr = data_size / num_rows;
1890-
bytes_per_row_feedback.store(bpr.max(1), Ordering::Relaxed);
1890+
let prev = bytes_per_row_feedback.load(Ordering::Relaxed);
1891+
let next = if prev == 0 || bpr >= prev {
1892+
// First batch or actual size is larger than estimate:
1893+
// adopt immediately to avoid OOM.
1894+
bpr
1895+
} else {
1896+
// Actual size is smaller: degrade gradually toward
1897+
// the true value to avoid over-correcting on a
1898+
// single anomalous batch.
1899+
(prev + bpr) / 2
1900+
};
1901+
bytes_per_row_feedback.store(next.max(1), Ordering::Relaxed);
18911902
}
18921903
Ok(batch)
18931904
};
@@ -1962,7 +1973,7 @@ pub struct SchedulerDecoderConfig {
19621973
pub decoder_config: DecoderConfig,
19631974
/// If set, target this many bytes per batch instead of using `batch_size` rows.
19641975
///
1965-
/// Only supported for v2.1+ (structural) files. For v2.0 (legacy) files this
1976+
/// Only supported for v2.1+ (structural) files. For v2.0 files this
19661977
/// option is ignored and a warning is logged.
19671978
pub batch_size_bytes: Option<u64>,
19681979
}
@@ -2029,7 +2040,7 @@ pub fn create_decode_stream(
20292040
.into_stream())
20302041
} else {
20312042
if batch_size_bytes.is_some() {
2032-
warn!("batch_size_bytes is not supported for v2.0 (legacy) files and will be ignored");
2043+
warn!("batch_size_bytes is not supported for v2.0 files and will be ignored");
20332044
}
20342045
let arrow_schema = ArrowSchema::from(schema);
20352046
let root_fields = arrow_schema.fields;

0 commit comments

Comments
 (0)