Skip to content

Commit 1748d17

Browse files
authored
Remove generic rle (#7350)
## Summary Tracking issue: #7216 We have this generic RLE code from the old compressor that doesn't make as much sense anymore. This change hardcodes the 2 schemes (int and float) instead of using generics that make things harder to trace. This is a purely cosmetic change. ## API Changes Removes olds constants and just has `FloatRLEScheme` and `IntRLEScheme`. ## Testing <!-- Please describe how this change was tested. Here are some common categories for testing in Vortex: 1. Verifying existing behavior is maintained. 2. Verifying new behavior and functionality works correctly. 3. Serialization compatibility (backwards and forwards) should be maintained or explicitly broken. --> Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
1 parent eacb959 commit 1748d17

6 files changed

Lines changed: 330 additions & 309 deletions

File tree

vortex-btrblocks/public-api.lock

Lines changed: 72 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,42 @@ pub fn vortex_btrblocks::schemes::float::ALPScheme::num_children(&self) -> usize
138138

139139
pub fn vortex_btrblocks::schemes::float::ALPScheme::scheme_name(&self) -> &'static str
140140

141+
pub struct vortex_btrblocks::schemes::float::FloatRLEScheme
142+
143+
impl core::clone::Clone for vortex_btrblocks::schemes::float::FloatRLEScheme
144+
145+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::clone(&self) -> vortex_btrblocks::schemes::float::FloatRLEScheme
146+
147+
impl core::cmp::Eq for vortex_btrblocks::schemes::float::FloatRLEScheme
148+
149+
impl core::cmp::PartialEq for vortex_btrblocks::schemes::float::FloatRLEScheme
150+
151+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::eq(&self, other: &vortex_btrblocks::schemes::float::FloatRLEScheme) -> bool
152+
153+
impl core::fmt::Debug for vortex_btrblocks::schemes::float::FloatRLEScheme
154+
155+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
156+
157+
impl core::marker::Copy for vortex_btrblocks::schemes::float::FloatRLEScheme
158+
159+
impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::FloatRLEScheme
160+
161+
impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::FloatRLEScheme
162+
163+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::ancestor_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::AncestorExclusion>
164+
165+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>
166+
167+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::descendant_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::DescendantExclusion>
168+
169+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate
170+
171+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool
172+
173+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::num_children(&self) -> usize
174+
175+
pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::scheme_name(&self) -> &'static str
176+
141177
pub struct vortex_btrblocks::schemes::float::NullDominatedSparseScheme
142178

143179
impl core::clone::Clone for vortex_btrblocks::schemes::float::NullDominatedSparseScheme
@@ -202,8 +238,6 @@ pub fn vortex_btrblocks::schemes::float::PcoScheme::matches(&self, canonical: &v
202238

203239
pub fn vortex_btrblocks::schemes::float::PcoScheme::scheme_name(&self) -> &'static str
204240

205-
pub const vortex_btrblocks::schemes::float::RLE_FLOAT_SCHEME: vortex_btrblocks::schemes::rle::RLEScheme<vortex_btrblocks::schemes::rle::FloatRLEConfig>
206-
207241
pub mod vortex_btrblocks::schemes::integer
208242

209243
pub use vortex_btrblocks::schemes::integer::IntConstantScheme
@@ -276,6 +310,42 @@ pub fn vortex_btrblocks::schemes::integer::FoRScheme::matches(&self, canonical:
276310

277311
pub fn vortex_btrblocks::schemes::integer::FoRScheme::scheme_name(&self) -> &'static str
278312

313+
pub struct vortex_btrblocks::schemes::integer::IntRLEScheme
314+
315+
impl core::clone::Clone for vortex_btrblocks::schemes::integer::IntRLEScheme
316+
317+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::clone(&self) -> vortex_btrblocks::schemes::integer::IntRLEScheme
318+
319+
impl core::cmp::Eq for vortex_btrblocks::schemes::integer::IntRLEScheme
320+
321+
impl core::cmp::PartialEq for vortex_btrblocks::schemes::integer::IntRLEScheme
322+
323+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::eq(&self, other: &vortex_btrblocks::schemes::integer::IntRLEScheme) -> bool
324+
325+
impl core::fmt::Debug for vortex_btrblocks::schemes::integer::IntRLEScheme
326+
327+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
328+
329+
impl core::marker::Copy for vortex_btrblocks::schemes::integer::IntRLEScheme
330+
331+
impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::IntRLEScheme
332+
333+
impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::IntRLEScheme
334+
335+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::ancestor_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::AncestorExclusion>
336+
337+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>
338+
339+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::descendant_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::DescendantExclusion>
340+
341+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate
342+
343+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool
344+
345+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::num_children(&self) -> usize
346+
347+
pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::scheme_name(&self) -> &'static str
348+
279349
pub struct vortex_btrblocks::schemes::integer::PcoScheme
280350

281351
impl core::clone::Clone for vortex_btrblocks::schemes::integer::PcoScheme
@@ -446,8 +516,6 @@ pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::num_children(&self) ->
446516

447517
pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::scheme_name(&self) -> &'static str
448518

449-
pub const vortex_btrblocks::schemes::integer::RLE_INTEGER_SCHEME: vortex_btrblocks::schemes::rle::RLEScheme<vortex_btrblocks::schemes::rle::IntRLEConfig>
450-
451519
pub mod vortex_btrblocks::schemes::string
452520

453521
pub use vortex_btrblocks::schemes::string::StringConstantScheme

vortex-btrblocks/src/builder.rs

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ use crate::schemes::bool;
1414
use crate::schemes::decimal;
1515
use crate::schemes::float;
1616
use crate::schemes::integer;
17-
use crate::schemes::rle;
1817
use crate::schemes::string;
1918
use crate::schemes::temporal;
2019

@@ -40,7 +39,7 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
4039
&integer::IntDictScheme,
4140
&integer::RunEndScheme,
4241
&integer::SequenceScheme,
43-
&rle::RLE_INTEGER_SCHEME,
42+
&integer::IntRLEScheme,
4443
////////////////////////////////////////////////////////////////////////////////////////////////
4544
// Float schemes.
4645
////////////////////////////////////////////////////////////////////////////////////////////////
@@ -49,7 +48,7 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
4948
&float::ALPRDScheme,
5049
&float::FloatDictScheme,
5150
&float::NullDominatedSparseScheme,
52-
&rle::RLE_FLOAT_SCHEME,
51+
&float::FloatRLEScheme,
5352
////////////////////////////////////////////////////////////////////////////////////////////////
5453
// String schemes.
5554
////////////////////////////////////////////////////////////////////////////////////////////////
@@ -162,8 +161,8 @@ impl BtrBlocksCompressorBuilder {
162161
pub fn only_cuda_compatible(self) -> Self {
163162
let builder = self.exclude_schemes([
164163
integer::SparseScheme.id(),
165-
rle::RLE_INTEGER_SCHEME.id(),
166-
rle::RLE_FLOAT_SCHEME.id(),
164+
integer::IntRLEScheme.id(),
165+
float::FloatRLEScheme.id(),
167166
float::NullDominatedSparseScheme.id(),
168167
string::StringDictScheme.id(),
169168
string::FSSTScheme.id(),

vortex-btrblocks/src/schemes/float.rs

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ use crate::CompressorContext;
2929
use crate::Scheme;
3030
use crate::SchemeExt;
3131
use crate::compress_patches;
32+
use crate::schemes::rle_ancestor_exclusions;
33+
use crate::schemes::rle_descendant_exclusions;
3234

3335
/// ALP (Adaptive Lossless floating-Point) encoding.
3436
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -55,7 +57,9 @@ pub use vortex_compressor::builtins::FloatDictScheme;
5557
pub use vortex_compressor::builtins::is_float_primitive;
5658
pub use vortex_compressor::stats::FloatStats;
5759

58-
pub use crate::schemes::rle::RLE_FLOAT_SCHEME;
60+
/// RLE scheme for float arrays.
61+
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
62+
pub struct FloatRLEScheme;
5963

6064
impl Scheme for ALPScheme {
6165
fn scheme_name(&self) -> &'static str {
@@ -269,6 +273,55 @@ impl Scheme for PcoScheme {
269273
}
270274
}
271275

276+
impl Scheme for FloatRLEScheme {
277+
fn scheme_name(&self) -> &'static str {
278+
"vortex.float.rle"
279+
}
280+
281+
fn matches(&self, canonical: &Canonical) -> bool {
282+
is_float_primitive(canonical)
283+
}
284+
285+
/// Children: values=0, indices=1, offsets=2.
286+
fn num_children(&self) -> usize {
287+
3
288+
}
289+
290+
fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
291+
rle_descendant_exclusions()
292+
}
293+
294+
fn ancestor_exclusions(&self) -> Vec<vortex_compressor::scheme::AncestorExclusion> {
295+
rle_ancestor_exclusions()
296+
}
297+
298+
fn expected_compression_ratio(
299+
&self,
300+
data: &mut ArrayAndStats,
301+
ctx: CompressorContext,
302+
) -> CompressionEstimate {
303+
// RLE is only useful when we cascade it with another encoding.
304+
if ctx.finished_cascading() {
305+
return CompressionEstimate::Skip;
306+
}
307+
308+
if data.float_stats().average_run_length() < super::integer::RUN_LENGTH_THRESHOLD {
309+
return CompressionEstimate::Skip;
310+
}
311+
312+
CompressionEstimate::Sample
313+
}
314+
315+
fn compress(
316+
&self,
317+
compressor: &CascadingCompressor,
318+
data: &mut ArrayAndStats,
319+
ctx: CompressorContext,
320+
) -> VortexResult<ArrayRef> {
321+
super::integer::rle_compress(self, compressor, data, ctx)
322+
}
323+
}
324+
272325
#[cfg(test)]
273326
mod tests {
274327
use std::iter;
@@ -288,7 +341,7 @@ mod tests {
288341
use vortex_fastlanes::RLE;
289342

290343
use crate::BtrBlocksCompressor;
291-
use crate::schemes::rle::RLE_FLOAT_SCHEME;
344+
use crate::schemes::float::FloatRLEScheme;
292345

293346
#[test]
294347
fn test_empty() -> VortexResult<()> {
@@ -330,7 +383,7 @@ mod tests {
330383

331384
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
332385

333-
let compressor = CascadingCompressor::new(vec![&RLE_FLOAT_SCHEME]);
386+
let compressor = CascadingCompressor::new(vec![&FloatRLEScheme]);
334387
let compressed = compressor.compress(&array.into_array())?;
335388
assert!(compressed.is::<RLE>());
336389

0 commit comments

Comments
 (0)