Skip to content

Commit 0d7e0ef

Browse files
KKouldKKould
committed
refactor(stats): load sketches alongside histograms in StatisticsMeta
1 parent a65ce17 commit 0d7e0ef

5 files changed

Lines changed: 74 additions & 192 deletions

File tree

‎src/execution/dml/analyze.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,9 +159,9 @@ impl Analyze {
159159
{
160160
let (histogram, sketch) =
161161
builder.build(histogram_buckets.unwrap_or(DEFAULT_NUM_OF_BUCKETS))?;
162-
let meta = StatisticsMeta::new(histogram);
162+
let meta = StatisticsMeta::new(histogram, sketch);
163163

164-
transaction.save_statistics_meta(cache, table_name, meta, sketch)?;
164+
transaction.save_statistics_meta(cache, table_name, meta)?;
165165
values.push(DataValue::Utf8 {
166166
value: format!("{table_name}/{index_id}"),
167167
ty: Utf8Type::Variable(None),
@@ -367,12 +367,11 @@ mod test {
367367
let table_name = "t1".to_string().into();
368368
let loader = transaction.meta_loader(kite_sql.state.meta_cache());
369369
let statistics_meta = loader.load(&table_name, 0)?.unwrap();
370-
let statistics_sketch = transaction
371-
.statistics_sketch(table_name.as_ref(), 0)?
372-
.unwrap();
373370
let expected_keys = 1
374371
+ 1
375-
+ statistics_sketch.storage_page_count(COUNT_MIN_SKETCH_STORAGE_PAGE_LEN)
372+
+ statistics_meta
373+
.sketch()
374+
.storage_page_count(COUNT_MIN_SKETCH_STORAGE_PAGE_LEN)
376375
+ statistics_meta.histogram().buckets_len();
377376
assert_eq!(keys, expected_keys);
378377

‎src/optimizer/core/histogram.rs‎

Lines changed: 23 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -354,14 +354,11 @@ impl Histogram {
354354
self.meta.buckets_len
355355
}
356356

357-
pub fn collect_count<F>(
357+
pub fn collect_count(
358358
&self,
359359
ranges: &[Range],
360-
estimate: &mut F,
361-
) -> Result<usize, DatabaseError>
362-
where
363-
F: FnMut(&DataValue) -> Result<usize, DatabaseError>,
364-
{
360+
sketch: &CountMinSketch<DataValue>,
361+
) -> Result<usize, DatabaseError> {
365362
if self.buckets.is_empty() || ranges.is_empty() {
366363
return Ok(0);
367364
}
@@ -379,7 +376,7 @@ impl Histogram {
379376
&mut bucket_i,
380377
&mut bucket_idxs,
381378
&mut count,
382-
estimate,
379+
sketch,
383380
&comparator,
384381
)?;
385382
if is_dummy {
@@ -395,19 +392,16 @@ impl Histogram {
395392
}
396393

397394
#[allow(clippy::too_many_arguments)]
398-
fn _collect_count<F>(
395+
fn _collect_count(
399396
&self,
400397
ranges: &[Range],
401398
binary_i: &mut usize,
402399
bucket_i: &mut usize,
403400
bucket_idxs: &mut Vec<usize>,
404401
count: &mut usize,
405-
estimate: &mut F,
402+
sketch: &CountMinSketch<DataValue>,
406403
comparator: &BoundComparator,
407-
) -> Result<bool, DatabaseError>
408-
where
409-
F: FnMut(&DataValue) -> Result<usize, DatabaseError>,
410-
{
404+
) -> Result<bool, DatabaseError> {
411405
let float_value = |value: &DataValue, prefix_len: usize| {
412406
let value = match value.logical_type() {
413407
LogicalType::Varchar(..) | LogicalType::Char(..) => match value {
@@ -530,7 +524,7 @@ impl Histogram {
530524
}
531525
Bound::Excluded(val) => (
532526
calc_fraction(&bucket.lower, &bucket.upper, val)?,
533-
Some(estimate(val)?),
527+
Some(sketch.estimate(val)),
534528
),
535529
Bound::Unbounded => unreachable!(),
536530
};
@@ -547,7 +541,7 @@ impl Histogram {
547541
}
548542
Bound::Excluded(val) => (
549543
calc_fraction(&bucket.lower, &bucket.upper, val)?,
550-
Some(estimate(val)?),
544+
Some(sketch.estimate(val)),
551545
),
552546
Bound::Unbounded => unreachable!(),
553547
};
@@ -564,7 +558,7 @@ impl Histogram {
564558
}
565559
Bound::Excluded(val) => (
566560
calc_fraction(&bucket.lower, &bucket.upper, val)?,
567-
Some(estimate(val)?),
561+
Some(sketch.estimate(val)),
568562
),
569563
Bound::Unbounded => unreachable!(),
570564
};
@@ -574,7 +568,7 @@ impl Histogram {
574568
}
575569
Bound::Excluded(val) => (
576570
calc_fraction(&bucket.lower, &bucket.upper, val)?,
577-
Some(estimate(val)?),
571+
Some(sketch.estimate(val)),
578572
),
579573
Bound::Unbounded => unreachable!(),
580574
};
@@ -592,7 +586,7 @@ impl Histogram {
592586
*count += cmp::max(temp_count, 0);
593587
}
594588
Range::Eq(value) => {
595-
*count += estimate(value)?;
589+
*count += sketch.estimate(value);
596590
*binary_i += 1
597591
}
598592
Range::Dummy => return Ok(true),
@@ -866,7 +860,6 @@ mod tests {
866860
builder.append(&DataValue::Null)?;
867861

868862
let (histogram, sketch) = builder.build(4)?;
869-
let mut estimate = |value: &DataValue| Ok(sketch.estimate(value));
870863

871864
let count_1 = histogram.collect_count(
872865
&[
@@ -876,7 +869,7 @@ mod tests {
876869
max: Bound::Excluded(DataValue::Int32(12)),
877870
},
878871
],
879-
&mut estimate,
872+
&sketch,
880873
)?;
881874

882875
assert_eq!(count_1, 9);
@@ -886,7 +879,7 @@ mod tests {
886879
min: Bound::Included(DataValue::Int32(4)),
887880
max: Bound::Unbounded,
888881
}],
889-
&mut estimate,
882+
&sketch,
890883
)?;
891884

892885
assert_eq!(count_2, 11);
@@ -896,7 +889,7 @@ mod tests {
896889
min: Bound::Excluded(DataValue::Int32(7)),
897890
max: Bound::Unbounded,
898891
}],
899-
&mut estimate,
892+
&sketch,
900893
)?;
901894

902895
assert_eq!(count_3, 7);
@@ -906,7 +899,7 @@ mod tests {
906899
min: Bound::Unbounded,
907900
max: Bound::Included(DataValue::Int32(11)),
908901
}],
909-
&mut estimate,
902+
&sketch,
910903
)?;
911904

912905
assert_eq!(count_4, 12);
@@ -916,7 +909,7 @@ mod tests {
916909
min: Bound::Unbounded,
917910
max: Bound::Excluded(DataValue::Int32(8)),
918911
}],
919-
&mut estimate,
912+
&sketch,
920913
)?;
921914

922915
assert_eq!(count_5, 8);
@@ -926,7 +919,7 @@ mod tests {
926919
min: Bound::Included(DataValue::Int32(2)),
927920
max: Bound::Unbounded,
928921
}],
929-
&mut estimate,
922+
&sketch,
930923
)?;
931924

932925
assert_eq!(count_6, 13);
@@ -936,7 +929,7 @@ mod tests {
936929
min: Bound::Excluded(DataValue::Int32(1)),
937930
max: Bound::Unbounded,
938931
}],
939-
&mut estimate,
932+
&sketch,
940933
)?;
941934

942935
assert_eq!(count_7, 13);
@@ -946,7 +939,7 @@ mod tests {
946939
min: Bound::Unbounded,
947940
max: Bound::Included(DataValue::Int32(12)),
948941
}],
949-
&mut estimate,
942+
&sketch,
950943
)?;
951944

952945
assert_eq!(count_8, 13);
@@ -956,7 +949,7 @@ mod tests {
956949
min: Bound::Unbounded,
957950
max: Bound::Excluded(DataValue::Int32(13)),
958951
}],
959-
&mut estimate,
952+
&sketch,
960953
)?;
961954

962955
assert_eq!(count_9, 13);
@@ -966,7 +959,7 @@ mod tests {
966959
min: Bound::Excluded(DataValue::Int32(0)),
967960
max: Bound::Excluded(DataValue::Int32(3)),
968961
}],
969-
&mut estimate,
962+
&sketch,
970963
)?;
971964

972965
assert_eq!(count_10, 2);
@@ -976,7 +969,7 @@ mod tests {
976969
min: Bound::Included(DataValue::Int32(1)),
977970
max: Bound::Included(DataValue::Int32(2)),
978971
}],
979-
&mut estimate,
972+
&sketch,
980973
)?;
981974

982975
assert_eq!(count_11, 2);

0 commit comments

Comments
 (0)